From c49bb56bf49a3f8e966c316d203d39385f6a5c60 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 6 Oct 2026 07:51:17 +0000 Subject: [PATCH 01/12] test(agentic): cover every tool and page, fail on hidden page errors Upgrade e2e to 0.17.0 and @e2e-dev/web to 0.12.0 for init scripts and failOnSkippedFailure. An init script records uncaught errors, console.error and Supabase 5xx answers; every test fails on them. Add a journey per tool and a sweep of every page in English, Spanish and Hungarian on both viewports. Run CI per viewport on app changes, and a weekly bug bash with each persona. Fix the p > div > p nesting in Attack results that the new check caught. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/skills/e2e/SKILL.md | 11 +- .claude/skills/e2e/references/agent.md | 33 +- .claude/skills/e2e/references/debugging.md | 8 +- .claude/skills/e2e/references/mcp.md | 3 + .claude/skills/e2e/references/running.md | 21 +- .claude/skills/e2e/references/setup.md | 57 +- .../skills/e2e/references/writing-tests.md | 58 +- .claude/skills/verify-debatelab/SKILL.md | 6 +- .../verify-debatelab/features/README.md | 2 +- .github/workflows/agentic-e2e.yml | 93 +- AGENTS.md | 2 +- agentic/ai-tools.e2e.ts | 57 + agentic/case-generation.e2e.ts | 11 +- agentic/first-debate.e2e.ts | 2 +- agentic/hungarian.e2e.ts | 21 - agentic/pages.e2e.ts | 66 + agentic/practice.e2e.ts | 42 + agentic/support/capture-errors.js | 41 + agentic/support/fixtures.ts | 104 +- agentic/workspace.e2e.ts | 56 + docs/testing.md | 28 +- e2e.config.ts | 22 +- package-lock.json | 1174 +---------------- package.json | 4 +- .../features/attack/AttackResults.tsx | 4 +- 25 files changed, 640 insertions(+), 1286 deletions(-) create mode 100644 agentic/ai-tools.e2e.ts delete mode 100644 agentic/hungarian.e2e.ts create mode 100644 agentic/pages.e2e.ts create mode 100644 agentic/practice.e2e.ts create mode 100644 agentic/support/capture-errors.js create mode 100644 agentic/workspace.e2e.ts diff --git a/.claude/skills/e2e/SKILL.md b/.claude/skills/e2e/SKILL.md index d280768f1..62ca3b62f 100644 --- a/.claude/skills/e2e/SKILL.md +++ b/.claude/skills/e2e/SKILL.md @@ -3,14 +3,17 @@ name: e2e description: Agentic end-to-end tests with e2e, the e2e runner. Covers scaffolding e2e.config.ts, picking the Playwright browser engine or the agent-device mobile engine, starting the app under test from the config, driving flows with agent.act, judging with agent.assert, agent.waitFor, and agent.extract, pinning values with screen, app, browser, and expect, shaping the agent (context, system prompt, tools, personas), the replay cache, the e2e CLI, reading .e2e/report.json, and bug bashes (parallel explore runs proven with repro tests). Use when a project depends on e2e, when asked for end-to-end, browser, mobile, or agentic UI tests, to bug bash or hunt for bugs, or when an e2e run fails. --- -> **DebateLab rules override this skill.** Vendored from `e2e@0.15.2` +> **DebateLab rules override this skill.** Vendored from `e2e@0.17.0` > (`node_modules/e2e/skills/e2e`, Apache-2.0, TesterArmy). Refresh it with > `rm -rf .claude/skills/e2e && cp -r node_modules/e2e/skills/e2e .claude/skills/e2e` > after an upgrade, then re-add this block. > > - Tests live in `agentic/*.e2e.ts` and import `test`, `expect`, and the > `student` fixture from `agentic/support/fixtures.ts`. The config is -> `e2e.config.ts`. +> `e2e.config.ts`. Tag a tool journey `journey`; a new route joins the page +> sweep in `agentic/pages.e2e.ts`. The fixtures fail a test on page errors, +> `console.error`, Supabase 5xx answers and overflow; fix the app, never the +> check. > - Run everything through the verification stack, never bare `npx e2e`: > `.claude/skills/verify-debatelab/bin/verify.sh up`, then > `verify.sh e2e run|explore|list ...`, then `verify.sh down`. The wrapper @@ -30,8 +33,8 @@ one goal; `agent.assert`, `agent.waitFor`, and `agent.extract` judge the screen. `screen`, `app`, `browser`, and `expect` make exact interactions and checks. The replay cache reruns verified actions and checks their recorded end state without a model call; agent judgments still run live. UI targets use -`@e2e-dev/web` for browsers or `@e2e-dev/mobile` for iOS simulators and -Android emulators. A test that takes only `app` can check an API with `fetch` +`@e2e-dev/web` for browsers or `@e2e-dev/mobile` for iOS simulators, +Android emulators, and connected phones. A test that takes only `app` can check an API with `fetch` and `expect` (topic `writing-tests`). Model sign-in commands are in [setup](references/setup.md#subscriptions-and-api-keys). diff --git a/.claude/skills/e2e/references/agent.md b/.claude/skills/e2e/references/agent.md index 3d190d459..128740499 100644 --- a/.claude/skills/e2e/references/agent.md +++ b/.claude/skills/e2e/references/agent.md @@ -6,8 +6,13 @@ model-call budget, no shared transcript. No agent step, no model calls. ## Configure a model -Put an AI SDK model under `agents.default`. Vercel AI Gateway reads -`AI_GATEWAY_API_KEY` or, without it, a Vercel OIDC token: +Put an AI SDK model under `agents.default`. You can use all AI SDK providers +that support tool calls and the language model specification v2 or later +(https://ai-sdk.dev/providers). Examples: `@ai-sdk/openai`, +`@ai-sdk/anthropic`, `@ai-sdk/google`, `@ai-sdk/amazon-bedrock`, +`@openrouter/ai-sdk-provider`, `ollama-ai-provider-v2`. For a server with +`/v1/chat/completions`, use `@ai-sdk/openai-compatible`. Vercel AI Gateway +reads `AI_GATEWAY_API_KEY` or, without it, a Vercel OIDC token: ```ts import type { E2EConfig } from 'e2e'; @@ -32,7 +37,8 @@ any provider. - `model` drives `agent.act`. Judgments use `judge` when set, else `model`. - A missing model for the built-in agent raises one run-level `MODEL_UNAVAILABLE` when the first test acquires `agent`, exit 2; auth - failures surface on the first model call as `MODEL_PROVIDER_FAILED`. + failures surface on the first model call as `MODEL_PROVIDER_FAILED`, or as + `CONFIG_LOAD_FAILED` for a provider that reads its key when the config loads. - `context` is what the app calls things, sent to every model call, judges included; `system` is how the acting agent works, read only by the act loop. Both live on the agents entry; `agentContext` on a test or group @@ -50,8 +56,8 @@ Tests use `agents.default` unless selected otherwise: - `{ agent: 'name' }` on an `agent.*` call overrides the test's choice. For signed-in personas, pair an agent with a `session` in a describe block -repeated per persona; the cache records per agent step, so a specialised -agent replays too. +repeated per persona; the cache keys every entry by the agent and its +context, so each persona records and replays its own steps. ## act: one goal @@ -236,14 +242,19 @@ matches. Misses and hand-offs use the model; `agent.assert`, `agent.assert`, `agent.waitFor`), so an unchecked `act` never replays; a plain-value `expect`, `expect.poll`, `agent.extract`, another `act`, or the attempt passing confirms nothing. -- A replay needs the app on the recorded path (unless the recording opens - with a navigation), re-finds each control by role, name, test id, - placeholder, and input purpose, and passes alone only when the recorded - end path and the controls seen during the step are back; otherwise the - agent takes over mid-step. `step.cache.reason` says why: `no-entry`, +- A replay needs the app on the recorded route (origin, path, and query, + ids and tokens aside; unless the recording opens with a navigation), + re-finds each control by role, name, test id, placeholder, input purpose, + and its named row (an unnamed control with twins and no named container + hands off), and passes alone only when the recorded end route is back, + every control that appeared (with its checked or selected state) is + there, every one that went away is gone, at least one of those changed + during the replay, and no new alert showed; otherwise the agent takes + over mid-step. Entries are keyed per agent and per `agentContext`. `step.cache.reason` says why: `no-entry`, `wrong-context`, `target-not-found`, `target-ambiguous`, `end-mismatch`, and so on. -- A step recording no actions creates no entry; one whose `unique()` value +- A step recording no actions, or changing nothing on screen or in the + route, creates no entry; one whose `unique()` value equals, is spelled inside, or is the encoded form of another param's value is not recorded either (`step.cache.notRecorded`: `param-collision`). - `e2e init` gitignores `.e2e/cache/`; remove that line to commit entries diff --git a/.claude/skills/e2e/references/debugging.md b/.claude/skills/e2e/references/debugging.md index abb9176b3..d3c46295a 100644 --- a/.claude/skills/e2e/references/debugging.md +++ b/.claude/skills/e2e/references/debugging.md @@ -28,7 +28,7 @@ jq '.run.results[] | select(.selected and .status != "passed") | .attempts[-1] (recorded as `skipped`). 4. Artifacts, under `.e2e/artifacts/`: `failure/screen.txt` and the engine's screenshot per failed attempt; a Playwright trace per - traced attempt (`npx playwright show-trace `); downloads; with + traced attempt (`npx playwright-core@1.63.0 show-trace `; `pnpm dlx` under pnpm); downloads; with `--video` the recording (`video/video.webm` in a local browser, each later page `video/video-part.webm` with its own `startedAt`; `video/video.mp4` on a device; a provider's file or link); with @@ -46,7 +46,7 @@ jq '.run.results[] | select(.selected and .status != "passed") | .attempts[-1] | `COLLECTION_ERROR` | `async` describe body, `test.setup` inside `describe`, an option forbidden in a serial group, registration outside collection | Restructure per `writing-tests` | | `HOOK_FAILED` | `beforeAll` or `afterAll` threw; its scope's tests skip | Fix the hook; the report carries its error | | `UNSUPPORTED_ARTIFACT` | A test's or target's `trace` or `video` on an engine that cannot record | Drop it there, or set it at the config root or CLI (such targets skip with a notice) | -| `BROWSER_INSTALL_FAILED`, `LAUNCH_TIMEOUT` | Browser download failed; engine init or attempt start exceeded `launchTimeout` | Run the quoted `npx playwright install ` (`--with-deps` on bare Linux); raise the root `launchTimeout` (60 s default) | +| `BROWSER_INSTALL_FAILED`, `LAUNCH_TIMEOUT` | Browser download failed; engine init or attempt start exceeded `launchTimeout` | Run the quoted `npx @e2e-dev/web install ` or `pnpm exec e2e-web install ` (`--with-deps` on bare Linux); raise the root `launchTimeout` (60 s default) | | `APP_UNREACHABLE` | `app.command` never answered `readyUrl` within `startupTimeout`; on a device, a message naming the iOS automation runner: the runner failed, not the app | Read the quoted log lines; check the port, `app.url`, `app.command.env`, `app.command.startupTimeout`. Runner: rerun, else `npx agent-device daemon stop` and reboot the simulator | | `APP_ALREADY_RUNNING` | Something already serves `url` when `command` should start | Stop it, or `reuseExisting: true` locally | | `APP_URL_REQUIRED`, `APP_NOT_OPEN` | A navigation on a target without `app.url`; a `screen` call before `app.open()` | Add `app.url` to the target; open the app first | @@ -58,14 +58,14 @@ jq '.run.results[] | select(.selected and .status != "passed") | .attempts[-1] | `MODEL_OUTPUT_INVALID` | An `extract` or `assert` answer failed the schema after one repair round | Simplify the schema or question; pick a stronger model | | `ACTION_FAILED` | Element not actionable (covered, disabled, detached), or an operation timed out | `expect` the condition first; close overlays; check `actionTimeout` | | `TEST_TIMEOUT` | The attempt exceeded `timeout` (120 s) | Split the test, or raise `timeout` | -| `STEP_NOT_AWAITED` | The body returned while a step still ran: a call without `await` | `await` the call the code frame names (every `app`, `agent`, `screen`, `expect` call) | +| `STEP_NOT_AWAITED` | The body returned while a step or `expect.poll` still ran, or an `afterEach` hook or fixture teardown returned while an `expect.poll` still ran: a call without `await` | `await` the call the code frame names (every `app`, `agent`, `screen`, `expect`, `expect.poll` call) | | `MODEL_UNAVAILABLE` | `agents..model` holds no AI SDK instance: reported once under `run.errors` at the first `agent` fixture; the run stops. A model-less custom executor fails only `waitFor` and `extract` this way | Construct one, e.g. `gateway('openai/gpt-6-luna-fast')` from `ai`, and export its key (`AI_GATEWAY_API_KEY`) | | `MODEL_PROVIDER_FAILED` | Network, 5xx, rate limit, no credits, or no response in 120 s, after the transport retries | Check the credential (the key; for keyless `gateway()` the Vercel CLI login and `.vercel/project.json`) and quota; retry (exit 3) | | `REPLAY_STALE` | `--strict-cache` or `cache.strict`, and a committed recording no longer replays; `step.cache.reason` says why | Rerun read-write with the knob the message names off; commit the changed entry under the directory it names (a custom `cache.store` is written by that run) | | `AUTOMATION_UNSUPPORTED` | The step needs an interaction the engine's toolset lacks (drag on a device, say); blocked, exit 1 | Do that step with `screen` actions, or run the test on a target that supports it | | `STEP_TIMEOUT`, `STEP_BUDGET_EXHAUSTED` | Goal too big or ambiguous, or a slow provider | Split the goal, use on-screen wording, add vocabulary via `agents..context` or `agentContext`; raise `maxSteps` or `maxModelCalls` (budget), the config `timeout` (`act`), `judgmentTimeout` (judgments); `--debug` | | `CONTEXT_OVERFLOW` | Screen plus step history did not fit the model's context window (`act` after one shrink-and-retry; a judgment on the first overflow) | Lower `agents..maxObservationBytes`, split the step, or pick a larger-window model | -| `POLICY_DENIED` | A `file:`, `data:`, or `javascript:` URL; a password `Secret` into a non-password sink; reading a secure field, `toHaveValue`, `toHaveText`, `toContainText`, `toHaveAttribute` included; `app.screenshot()` after a secret fill | http(s) only; passwords into password inputs only; assert the outcome, not the value; screenshot before filling secrets | +| `POLICY_DENIED` | A URL whose scheme is not `http:` or `https:` (`file:`, `view-source:`, `data:`); a password `Secret` into a non-password sink; reading a secure field, `toHaveValue`, `toHaveText`, `toContainText`, `toHaveAttribute` included; `app.screenshot()` after a secret fill | http(s) or `about:blank` only; passwords into password inputs only; assert the outcome, not the value; screenshot before filling secrets | | `UNSUPPORTED_CAPABILITY` | A fixture the engine lacks (`browser` on a device), `schema` or `vision` on `act`, an action the surface lacks | Declare `requires: ['browser']`; drop the option or action | | `SESSION_UNAVAILABLE`, `SESSION_CONTRACT` | `session: 'x'` with no setup saving `x`; a setup that skipped a declared name. `SESSION_MISMATCH`, `SESSION_EXPIRED`, `SESSION_INVALID`: the stored session is another run's or app's, expired, or corrupt | Add or fix the `test.setup`; rerun it | | `ONLY_IN_CI` | `test.only` reached CI | Remove it | diff --git a/.claude/skills/e2e/references/mcp.md b/.claude/skills/e2e/references/mcp.md index 4ce194a05..9875d27d6 100644 --- a/.claude/skills/e2e/references/mcp.md +++ b/.claude/skills/e2e/references/mcp.md @@ -20,6 +20,9 @@ Or declare it in the client's project config (`.mcp.json` for Claude Code, { "mcpServers": { "e2e": { "command": "npx", "args": ["e2e", "mcp"] } } } ``` +No model is needed: without the optional `ai` package, sessions open and the +deterministic tools work. + Flags: `--config ` names the default config file, `--target ` fixes the target every session opens on, `--headless` hides the browser or simulator (sessions are headed by default outside CI), `--max-sessions ` diff --git a/.claude/skills/e2e/references/running.md b/.claude/skills/e2e/references/running.md index e7c788d2b..f02e95a2a 100644 --- a/.claude/skills/e2e/references/running.md +++ b/.claude/skills/e2e/references/running.md @@ -11,7 +11,7 @@ npx e2e guide [topic] # print this skill; topics: setup, writing-te # running, explore, debugging, mcp, bug-bash npx e2e cache ls|clear|stats # inspect or empty the replay cache npx e2e login|logout|models [provider] # e2e/oauth subscription logins: openai, - # github-copilot, spacexai + # github-copilot, opencode-console, spacexai npx e2e mcp [--target ] # MCP server for a coding agent (topic mcp) npx e2e feedback -m [opts] # report a problem with e2e itself npx e2e telemetry [disable|enable] # anonymous usage telemetry: status or switch @@ -27,7 +27,7 @@ npx e2e telemetry [disable|enable] # anonymous usage telemetry: status or switch | `--tag ` | Any of the tags, comma-separated or repeated; all of them with `--tag-mode all`. An empty `--target`, `--tag`, or `--agent` value is a usage error, exit 2. | | `--exclude-tag ` | Drop tests carrying any of these tags, however selected. | | `--grep `, `--grep-invert ` | Keep, or drop, tests whose title (describe titles and test title joined by spaces, `checkout pays`; not file or tags) matches a regular expression. Bare pattern or `'/pattern/i'`; repeat for alternatives. | -| `--last-failed` | The tests the previous run (`/report.json`) did not pass, plus every test in a failed `beforeAll` or `afterAll` scope. No report is `NO_LAST_RUN`, exit 2. | +| `--last-failed` | The tests the previous run (`/report.json`) did not pass or never ran (`--max-failures` included), plus every test in a failed `beforeAll` or `afterAll` scope. A test another filter leaves out stays owed (`run.carried` in the report) until a rerun runs it. No report is `NO_LAST_RUN`, exit 2. | | `--shard ` | One contiguous slice of the selected tests (`--shard 2/3`), cut after every other filter; serial groups stay whole, each shard brings its own setup tests. | | `--headed` | Visible browser or simulator when the engine supports it. | | `--agent ` | Run unpinned tests as these `agents.` entries (default `agents.default`), comma-separated or repeated; several names run each such test once per agent. | @@ -87,7 +87,8 @@ its own tools. `summary.md`, `failures/`, `ai-trace.json`, `sessions/`, and `artifacts/` (screenshots, Playwright traces, videos, `--debug` transcripts, downloads). `artifacts/` is cleared once a run's tests start; a run stopping before -leaves the last run's files. The report records every artifact path, a +leaves the last run's files, and a `--last-failed` rerun keeps the files +the report it reruns names and writes its own under `artifacts/rerun-/`. The report records every artifact path, a hosted service's video by URL. - `list` (default): setup steps, one line per file and target, a `Failed @@ -102,11 +103,14 @@ hosted service's video by URL. and `run.results[]`, one per test and target: `titlePath`, `file`, `source`, `tags` (`[]` when none), `agent`, `repeat` (0 unless `--repeat-each`), `selected`, `status`, `attempts[]` of `steps[]`, - `artifacts[]`, `error`. + `artifacts[]`, `error`. `run.summary` counts the selected results: + `passed`, `failed` (timed out included), `interrupted` (stopped by the + run: a signal, `--max-failures`, a run-level error; not a failure), `flaky`, and `skipped` add up + to `selected`; `discovered - selected` were left out. - `junit`: `junit.xml` for CI summaries; `--reporter list,junit` keeps the terminal output. - `markdown` (`--reporter list,markdown`): `summary.md` plus one page per - failed or flaky test under `failures/`. The summary holds counts and + failed or flaky test under `failures/` (an interrupted test gets none). The summary holds counts and spend, a block per failed test (error, facts, failing step, whether every attempt failed alike, last model turns, screen location and closest nodes, the line to look at, evidence paths), the flaky tests folded @@ -142,8 +146,9 @@ interrupted workers keeps the failures' or the error's code. Exit 2 is deterministic, never retry it; only exit 3 is worth a job-level retry. The first SIGINT or SIGTERM interrupts and writes the report if any test -had started; the second forces engine teardown; the third kills the app -process groups and exits 130 at once. +had started; the second forces engine teardown and still writes the +report, plus `junit.xml` and `summary.md` when those reporters are on; the +third kills the app process groups and exits 130 at once. ## Continuous integration @@ -174,7 +179,7 @@ jobs: node-version: 26 # any Node >= 22.12 cache: pnpm - run: pnpm install --frozen-lockfile - - run: npx playwright install chromium --with-deps + - run: pnpm exec e2e-web install chromium --with-deps - run: npx e2e run --reporter list,junit env: AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }} diff --git a/.claude/skills/e2e/references/setup.md b/.claude/skills/e2e/references/setup.md index 7c42c1e01..8d754df9c 100644 --- a/.claude/skills/e2e/references/setup.md +++ b/.claude/skills/e2e/references/setup.md @@ -6,13 +6,11 @@ - ES modules: `.ts` config, tests, helpers, and workspace packages exporting `.ts` source load as ESM regardless of the nearest `package.json` `type` (CommonJS packages need no change); never `require` or `module.exports`. -- Browser tests: `@e2e-dev/web` plus `playwright` (`>=1.63.0 <2`), a peer the - engine does not install: an existing Playwright keeps its version and - browser cache, one out of range fails install as an unmet peer (npm's - `ERESOLVE`): upgrade `playwright` within the range. Missing - browsers download on first boot; in CI run `npx playwright install chromium - --with-deps`. Mobile tests: `@e2e-dev/mobile`, pinning `agent-device` - exactly; the pin moves with each engine release. +- Browser tests: `@e2e-dev/web`, pinning `playwright-core` exactly; do not + add `playwright` for it. Missing browsers download on first boot; in CI run + `npx @e2e-dev/web install chromium --with-deps` (pnpm: `pnpm exec e2e-web + install chromium --with-deps`). Mobile tests: `@e2e-dev/mobile`, pinning + `agent-device` exactly; each pin moves with its engine release. ## Scaffold @@ -41,7 +39,7 @@ MCP entries. Without the wizard (`ai`, Vercel AI SDK v7, only for `agent.*` steps): ```bash -npm install --save-dev e2e @e2e-dev/web playwright ai@^7 +npm install --save-dev e2e @e2e-dev/web ai@^7 ``` ## Subscriptions and API keys @@ -53,6 +51,7 @@ key, or a local endpoint. Authenticate: | --- | --- | | ChatGPT Plus or Pro | `npx e2e login openai` | | GitHub Copilot | `npx e2e login github-copilot` (GitHub CLI signed in, or your own `--client-id`) | +| OpenCode Console (OpenCode Zen and OpenCode Go) | `npx e2e login opencode-console` (approve the device code, pick the workspace) | | SuperGrok or X Premium+ | `npx e2e login spacexai` | | Vercel AI Gateway | Set `AI_GATEWAY_API_KEY`, or sign in to the Vercel CLI and `npx vercel link`; without the key `gateway()` uses a Vercel OIDC token | | OpenRouter | Set `OPENROUTER_API_KEY` | @@ -63,6 +62,22 @@ set `model: chatgpt('gpt-6-luna')` from `e2e/oauth/chatgpt`, run `npx e2e login openai`. `npx e2e models` lists the ids each login serves. Use API keys in CI. +Switching to Copilot: install `ai`, `@ai-sdk/openai-compatible`, and +`@ai-sdk/openai`, set `model: copilot('')` from `e2e/oauth/copilot`, run +`npx e2e login github-copilot`. `copilot()` calls a model over chat completions, or +over Copilot's Responses API when the plan serves that model only there, choosing +per model from the plan's listing. `npx e2e models github-copilot` marks the models +it cannot call at all: those served only over an API `copilot()` does not speak, and +those the plan has not enabled. + +Switching to OpenCode Console: install `ai`, `@ai-sdk/openai-compatible`, +`@ai-sdk/openai`, `@ai-sdk/anthropic`, and `@ai-sdk/google`, set +`model: opencodeConsole('')` from `e2e/oauth/opencode-console`, run +`npx e2e login opencode-console`. A bare id is an OpenCode Zen model; a `go/` id +(`go/deepseek-v4.1-flash`) is an OpenCode Go model and needs the workspace's Go +subscription. `npx e2e models opencode-console` lists the ids, tagged Zen or Go. In CI, +set a Console service account key as `OPENCODE_API_KEY`. + ## The config `e2e.config.ts` sits at the project root and default-exports an object literal @@ -168,6 +183,8 @@ start a script that brings them up and serves the app. | `headers` | Sent to the app's site only (Vercel's `x-vercel-protection-bypass`, ngrok's `ngrok-skip-browser-warning`), `agent.act` included; disables the browser HTTP cache and service workers. | | `basicAuth` | `{ username, password }` for a `401` challenge; `password` may be `secrets.get('name')`, resolved per attempt and redacted like any secret, the base64 `Authorization` credential too. | | `userAgent` | The `User-Agent` every attempt sends and `navigator.userAgent` reports. | +| `locale`, `timezoneId` | The language (`'de-DE'`: `navigator.language`, `Intl`, `Accept-Language`) and IANA time zone (`'Europe/Berlin'`) every attempt runs in. | +| `initScripts` | Scripts every document runs before the page's own: source, `{ path }`, or a function with no closures. | | `testIdAttribute` | What `getByTestId` reads; default `data-testid`. | | `screencast` | `{ size?, quality? }` for the engine's own video: frame size (default the viewport's), JPEG quality 0 to 100. | @@ -176,7 +193,7 @@ start a script that brings them up and serves the app. the handle (a reference, not the value) in `app.command.env`, a template literal, or `context`; read those from `process.env`. - `reconnectEndpoint` or an attempt-scoped provider rides one persistent - context without `headers`, `basicAuth`, `userAgent`, `app.clearState()`, or + context without `headers`, `basicAuth`, `userAgent`, `locale`, `timezoneId`, `app.clearState()`, or session state. Recovery never repeats a dispatched operation; exhausting the budget is `OPERATION_TIMEOUT`. Without `reconnectEndpoint` a dropped connection is reacquired at the next attempt. @@ -246,16 +263,20 @@ For an app started elsewhere, point `app.url` at it, literally or via | --- | --- | | `AI_GATEWAY_API_KEY`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`, ... | Read by provider packages, not the runner. | | `E2E_USER__USERNAME`, `E2E_USER__PASSWORD` | Override `credentials.`; `` is the name uppercased, other characters `_`. | -| `E2E_SECRET_` | Overrides `secrets.`, same rule. | +| `E2E_SECRET_` | Overrides `secrets.`, same rule. Two entries of one namespace mapping to one variable are `INVALID_CONFIG`. | | `CI` | CI defaults; list in topic `running`. | | `E2E_TELEMETRY_DISABLED`, `DO_NOT_TRACK` | Disable anonymous telemetry, as does `e2e telemetry disable`; `E2E_TELEMETRY_DEBUG=1` prints events instead of sending. | ## Mobile targets -`@e2e-dev/mobile` drives iOS simulators and Android emulators through -[agent-device](https://github.com/callstack/agent-device); needs Xcode with a -simulator runtime or the Android SDK with an emulator; run -`npx agent-device doctor` once. +`@e2e-dev/mobile` drives iOS simulators, Android emulators, and connected +phones through [agent-device](https://github.com/callstack/agent-device); +needs Xcode with a simulator runtime or the Android SDK with an emulator (a +phone needs Xcode with Developer Mode and runner signing, or adb with USB +debugging authorized); run `npx agent-device doctor` once. Name a phone in `device` by the name +`npx agent-device devices` lists, not its UDID or serial. On an iPhone, device +settings (permissions, `clearState`, network, location, appearance, +biometrics, keychain) and the clipboard are simulator-only. ```ts import type { E2EConfig } from 'e2e'; @@ -285,8 +306,14 @@ export default { `app.launchArguments` and `app.permissions` ride every fresh launch: arguments reach the app process (iOS) or `am start` (Android), permissions are set first. +- Expo development build on an iOS simulator: `app.launchArguments: + ['--initialUrl', 'http://localhost:8081', '-EXDevMenuShowsAtLaunch', 'NO', + '-EXDevMenuIsOnboardingFinished', 'YES', '-EXDevMenuShowFloatingActionButton', + 'NO']` loads that dev server on every fresh launch, dev menu off; + `app.command` (`npx expo start --port 8081`) with `readyUrl: + 'http://localhost:8081/status'` starts it. - One worker per device. No `device`: every booted simulator or emulator of - the platform is the pool, up to `workers` (none booted: agent-device boots + the platform, and every connected phone, is the pool, up to `workers` (none booted: agent-device boots one); one `device`: one worker whatever `workers` says; a list (`device: ['iPhone 17', 'iPhone 17 Pro']`): an explicit pool. Devices boot in `prepare`, before the run's clock. Two sessions on one device fight over diff --git a/.claude/skills/e2e/references/writing-tests.md b/.claude/skills/e2e/references/writing-tests.md index f7601c2e0..9a2b719c2 100644 --- a/.claude/skills/e2e/references/writing-tests.md +++ b/.claude/skills/e2e/references/writing-tests.md @@ -168,7 +168,9 @@ substring; a `RegExp` matches as written. scoping under another locator. Any other `filter` key (`hasNot`, `hasNotText`) or an empty `filter({})` is `INVALID_LOCATOR`. - `visible: true` drops nodes the page hides (a closed drawer) before the - exactly-one rule. + exactly-one rule. Visibility is what renders, as `toBeVisible()` reads it: + an `aria-hidden` spinner that paints is visible, so `toBeHidden()` waits + for it to go. - On the web, queries reach open shadow roots and closed roots attached with `attachShadow`, not declarative closed roots. `browser.locator(css)`, `frameLocator`, and `filter({ hasText })` stop at a closed root; query the @@ -215,7 +217,7 @@ text, both `[]` at zero. Text is the rendered text, whitespace collapsed: on the web what `innerText` reads (`text-transform` applies, `display: none` drops out, `
` is a space); `toHaveText` reads the same. `isChecked()` is `false`, not an error, on a node with no checked state, so query checkable -controls by role. `waitFor({ state?: 'visible' | 'hidden', timeout? })` waits +controls by role. `waitFor({ state?: 'attached' | 'detached' | 'visible' | 'hidden', timeout? })` waits within `actionTimeout`, else `LOCATOR_NOT_FOUND`. For a value that has to settle use `expect`, not a read. Reading a password field's value or attributes is `POLICY_DENIED`, as is `toHaveAttribute` on one, negated too. @@ -224,13 +226,20 @@ attributes is `POLICY_DENIED`, as is `toHaveAttribute` on one, negated too. `expect(locator)` polls up to `config.assertionTimeout` (5 s) or `{ timeout }`; `.not` inverts and passes once the negation has held 1 s -continuously, so it never returns in under a second. `expect(value, +continuously, or for the whole `timeout` when that is shorter, so it never +returns sooner. `expect(value, message?)` is synchronous. `expect.poll(read, { timeout?, interval?, message? })` re-reads until a value matcher passes (`assertionTimeout` and 100 ms by default, stopping with the -attempt); a throwing read keeps polling, and it is not a report step. +attempt); a throwing read keeps polling, and it is not a report step. A +poll that the body or a hook returns without awaiting is cancelled and fails +that phase with `STEP_NOT_AWAITED`. `expect.soft(x)` keeps a failure instead of throwing; the attempt fails -after the body with every soft failure listed. +after the body with every soft failure listed. If the body calls `test.skip`, +the test stays skipped and the failures remain in `secondaryErrors`. The CLI +shows `Skipped After Failure`, also when a retry skips after an earlier +failure. Set `failOnSkippedFailure: true` in the config to fail the run with +exit code 1 in either case. The default is `false`; clean skips stay green. `expect.any(Class)`, `expect.anything()`, `expect.objectContaining(obj)`, `expect.arrayContaining(arr)`, `expect.stringContaining(s)`, and `expect.stringMatching(s | RegExp)` stand in for values inside `toEqual`, @@ -263,6 +272,12 @@ presence, `toHaveAttribute(name, value)` the value; `toBeAttached` waits for a match, hidden or not; `toHaveClass` compares the whole normalized class list or tests a RegExp. A failed matcher is `ASSERTION_FAILED`, exit code 1. +Playwright's `{ checked: false }`, `{ enabled: false }`, `{ visible: false }`, +and `{ attached: false }` flip their matchers, and `{ ignoreCase: true }` +works on `toHaveText`, `toContainText`, `toHaveAccessibleName`, +`toHaveAttribute(name, value)`, and `expect(browser).toHaveURL`. Any other option, `indeterminate` or +`useInnerText` included, is `INVALID_ARGUMENT`, in JavaScript too. + ## Sign-in sessions Sign in once in a setup test, save the state under a name, and let other @@ -317,7 +332,9 @@ credentials: { A function is read at fill time and redacted only from that fill on. - `E2E_USER__USERNAME` and `E2E_USER__PASSWORD` override either field per run, even over a function; `` is the credential name - uppercased, every character outside `[A-Z0-9]` as `_`. + uppercased, every character outside `[A-Z0-9]` as `_`. Two credentials (or + two secrets) mapping to one variable, `svc-a` and `svc_a`, are + `INVALID_CONFIG`. - `credentials.user('admin').password` is a `Secret` with no plaintext accessor, named `admin.password` to the agent and in reports; `secrets.get()` never returns it (separate namespaces). Only `fill()` and `agent.act` params accept it, stringifying it @@ -328,6 +345,11 @@ credentials: { `E2E_SECRET_STRIPE_KEY`; `secrets.get('stripe-key')` is the same kind of handle, fills any editable input, and is redacted by name everywhere the runner writes. +- A registered value passed as a plain string (`process.env.STRIPE_KEY` in a + title, a URL, a locator, an `agent.act` instruction or param) is redacted + too: reports, step labels, the executor, and the model see + `` (and so does the test id, for a value in a title), + so the agent cannot type it. Pass the handle. ## The browser fixture (browser only) @@ -349,14 +371,20 @@ the app opens itself (`target="_blank"`, `window.open`) is not followed: - `evaluate(fn | source, arg?)`: runs a function or source string in the page, JSON in and out, no closures; a throw in the page is `EVALUATE_FAILED`. -- `route(pattern, handler)`, `unroute(pattern)`: intercept requests; - `route.request` has `url`, `method`, `headers`, `postData`. The handler - calls exactly one of `fulfill({ status?, headers?, json | body })`, - `continue()`, or `abort()`; none or two fails the next step with - `ACTION_FAILED`. -- `waitForResponse(pattern, { timeout? })`: resolves with - `{ url, status, headers, json(), text() }`; `text()` and `json()` reject - with `ACTION_FAILED` when the body could not be read. +- `addInitScript(source | { path } | fn)`, `addInitScript(fn, arg)`: runs + before the page's own scripts; call it before `app.open`. `arg` is JSON. +- `route(pattern, handler)`, `unroute(pattern)`: intercept requests, newest + route first; `route.request` has `url`, `method`, `headers`, `postData`. + The handler calls exactly one of + `fulfill({ status?, headers?, contentType?, json | body | path })`, + `continue({ url?, method?, headers?, postData? })` (straight to the + network), `fallback()` (the route registered before it), or `abort()`; + none or two fails the next step with `ACTION_FAILED`, an unsupported + option with `INVALID_ARGUMENT`. +- `waitForResponse(pattern, { timeout? })`: resolves once the headers + arrive, with `{ url, status, headers, json(), text() }`; `text()` and + `json()` wait for the body (up to the action timeout) and reject with + `ACTION_FAILED` when it could not be read. - `cookies()`, `setCookies([...])`: a target is an http(s) URL or a domain. - `setViewport({ width, height })`: resize. - `onDialog('accept' | 'dismiss' | handler)`: awaited; resolves to an async @@ -385,5 +413,5 @@ A `route`, `unroute`, or `waitForResponse` pattern is a glob string or plus value matchers, in a `test.extend` fixture that reads `browser.cookies()` when the API needs the session. - No sleeps or polling loops; a matcher with a longer `timeout` instead. -- `await` every step call, else `STEP_NOT_AWAITED` at the line of the call. +- `await` every step call and `expect.poll`, else `STEP_NOT_AWAITED` at the line of the call. - Assert the fact a model produced with `toContain`, not its exact sentence. diff --git a/.claude/skills/verify-debatelab/SKILL.md b/.claude/skills/verify-debatelab/SKILL.md index e650f142d..9e15855fb 100644 --- a/.claude/skills/verify-debatelab/SKILL.md +++ b/.claude/skills/verify-debatelab/SKILL.md @@ -111,13 +111,15 @@ Selectors: use accessible names and roles as the feature files list them (`getBy ```bash $H e2e run # the suite, desktop and mobile targets -$H e2e run agentic/hungarian.e2e.ts --target mobile # one file, one viewport +$H e2e run agentic/pages.e2e.ts --target mobile # one file, one viewport +$H e2e run --tag journey --grep Sparring # one tool journey $H e2e explore "Use Brainstorm and Attack on one motion and report anything off" --session student --agent coach $H e2e cache ls # recorded agent flows in agentic/replay ``` - `e2e` runs `doctor` first and refuses a broken stack, like `drive`. It turns e2e telemetry off and prints `EVIDENCE `, which holds `summary.md`, `report.json`, and per-attempt screenshots, traces, and failure videos. -- The testing agent has its own OpenRouter models: free `qwen/qwen3.8-27b:free` acts (test steps and explorations) and `meta/muse-spark-1.3-contributor` judges, under two cents per suite run. `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` override them. The key comes from `OPENROUTER_API_KEY`, `LIVE_OPENROUTER_API_KEY`, or the live keys file, and is needed in fake mode too. A recorded `agent.act` replays with no model call; every judgment and every exploration step is a call. Without a key, agent tests skip. A `temporarily rate-limited upstream` error from the free actor is infrastructure: rerun with `--last-failed`. For a long or important exploration, `E2E_AGENT_MODEL=openai/gpt-6-luna` is faster and more reliable for cents (the Muse Spark judge cannot act). +- The suite is tool journeys (tag `journey`) and a page sweep (tag `pages`) of every page in English, Spanish and Hungarian. Every test fails on an uncaught error, `console.error` (React warnings included), a Supabase 5xx, horizontal overflow, or a request to a hosted project: `agentic/support/capture-errors.js` records them in the page. A failure that names `errors the page logged` is a real app error to fix, not test noise. +- The testing agent has its own OpenRouter models: free `qwen/qwen3.8-27b:free` acts (test steps and explorations) and `meta/muse-spark-1.3-contributor` judges, `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` override them. Record new or changed flows with `E2E_AGENT_MODEL=openai/gpt-6-luna`: the model is not part of the replay key, so CI replays them with the free actor. The key comes from `OPENROUTER_API_KEY`, `LIVE_OPENROUTER_API_KEY`, or the live keys file, and is needed in fake mode too. A recorded `agent.act` replays with no model call; every judgment and every exploration step is a call. Without a key, agent tests skip. A `temporarily rate-limited upstream` error from the free actor is infrastructure: rerun with `--last-failed`. For a long or important exploration, `E2E_AGENT_MODEL=openai/gpt-6-luna` is faster and more reliable for cents (the Muse Spark judge cannot act). - `--session student` starts an exploration signed in as a fresh demo user, minted by `agentic/sessions.e2e.ts`. `--agent` picks a persona: `first-timer`, `coach`, `edge-input`, or `hungarian`. `--target mobile` explores the phone-sized viewport. - Explorations report `issue` (functional) and `warning` (cosmetic) findings, and exit 1 when there is an issue. Treat a finding as a lead: the stack serves Vite dev, so a route that is still compiling can look dead, and fake mode answers every motion with free-transport text. Prove a finding with a repro test tagged `repro` before you report it as a bug. - Several explorations can run at once (a bug bash), because each gets its own evidence directory. They share this one stack, so give each a different area. diff --git a/.claude/skills/verify-debatelab/features/README.md b/.claude/skills/verify-debatelab/features/README.md index e2bd114fe..6fb8e1f13 100644 --- a/.claude/skills/verify-debatelab/features/README.md +++ b/.claude/skills/verify-debatelab/features/README.md @@ -51,4 +51,4 @@ Each feature file starts with an H1 title and one paragraph describing the user- - Billing: expired account blocked, promo code restores access (`recipes/billing.spec.ts`); signed, forged, and replayed Stripe webhooks (`e2e/full-stack/billing.spec.ts`). - Admin announcements and the non-admin block (`e2e/full-stack/admin.spec.ts`), public share links (`e2e/full-stack/public.spec.ts`), and Workspaces with live collaboration and invites (`e2e/workspace-collaboration.spec.ts`). -Not yet mapped: theme and UI language, the public landing and SEO pages, and share-link creation (the app has no client caller for the `share` function). +Not yet mapped in recipes: theme and UI language, the public landing and SEO pages, and share-link creation (the app has no client caller for the `share` function). The agentic suite covers UI language switching (`agentic/workspace.e2e.ts`) and renders the public pages in every language (`agentic/pages.e2e.ts`). diff --git a/.github/workflows/agentic-e2e.yml b/.github/workflows/agentic-e2e.yml index 6dec72d43..1f974fe14 100644 --- a/.github/workflows/agentic-e2e.yml +++ b/.github/workflows/agentic-e2e.yml @@ -3,46 +3,66 @@ name: Agentic E2E # The e2e suite in agentic/ against the local stack in fake mode: an AI agent # drives goal-level flows and judges screens. Recorded flows replay from # agentic/replay without model calls; judgments always call the model. -# On demand it runs one exploration (a persona looking for bugs) instead. +# It runs nightly and on pull requests that touch the app or the suite, one +# job per viewport. A weekly bug bash sends each persona exploring, and on +# demand one exploration runs instead of the suite. # Not a required check. See docs/testing.md#agentic-browser-tests. on: pull_request: paths: + - "src/**" + - "public/**" + - "index.html" + - "supabase/functions/**" + - "supabase/migrations/**" + - "package-lock.json" - "agentic/**" - "e2e.config.ts" - "e2e/full-stack/stack-core.ts" + - "e2e/full-stack/fake-openrouter.mjs" - ".claude/skills/verify-debatelab/bin/verify.sh" - ".github/workflows/agentic-e2e.yml" schedule: - - cron: "47 3 * * *" + - cron: "47 3 * * *" # the suite + - cron: "13 5 * * 0" # the weekly bug bash workflow_dispatch: inputs: - explore: - description: "Exploration goal. Empty runs the test suite instead." - required: false - default: "" - persona: - description: "Agent from e2e.config.ts that explores" - type: choice - default: first-timer - options: [default, first-timer, coach, edge-input, hungarian] - target: - description: "Viewport to explore on" - type: choice - default: desktop - options: [desktop, mobile] - -permissions: - contents: read - -concurrency: - group: agentic-e2e-${{ github.ref }} - cancel-in-progress: true + # The weekly bug bash explores with every persona; a manual run explores once. + plan-explore: + if: github.event.schedule == '13 5 * * 0' || (github.event_name == 'workflow_dispatch' && inputs.explore != '') + runs-on: ubuntu-latest + outputs: + legs: ${{ steps.plan.outputs.legs }} + steps: + - id: plan + env: + GOAL: ${{ inputs.explore }} + PERSONA: ${{ inputs.persona }} + TARGET: ${{ inputs.target }} + run: | + if [ -n "$GOAL" ]; then + jq -cn '[{persona: env.PERSONA, target: env.TARGET, goal: env.GOAL}]' + else + jq -cn '[ + {persona: "first-timer", target: "desktop", goal: "Get ready for a tournament next week: start a debate, generate a case for it, then practise it in a Sparring round. Report anything broken, confusing or misleading."}, + {persona: "coach", target: "desktop", goal: "Use Generate, Brainstorm and the Library in one debate, then check that History and the Current Debate page show every saved result and source where the app says they will be."}, + {persona: "edge-input", target: "desktop", goal: "Try awkward input in the motion, argument and source fields of Generate, Attack and the Library: very long text, emoji, right-to-left text, blank fields and double submits."}, + {persona: "hungarian", target: "mobile", goal: "Switch the app to Hungarian, start a debate, and open every tool in the menu. Report untranslated, cut-off or overflowing text."} + ]' + fi | sed 's/^/legs=/' >>"$GITHUB_OUTPUT" -jobs: - agentic-e2e: + explore: + needs: plan-explore runs-on: ubuntu-latest timeout-minutes: 45 + # Findings are leads to reproduce, not verdicts: a red exploration does not + # fail the workflow. docs/testing.md has the bug-bash procedure. + continue-on-error: true + strategy: + fail-fast: false + matrix: + leg: ${{ fromJSON(needs.plan-explore.outputs.legs) }} + name: explore (${{ matrix.leg.persona }}, ${{ matrix.leg.target }}) steps: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -65,36 +85,22 @@ jobs: - name: Install Chromium run: npx playwright install --with-deps chromium - # Without the key every agent test skips, which would read as a pass. - # Same accepted risk as live-e2e.yml: pull-request code runs with it. - name: Require the testing agent's model key env: LIVE_OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }} run: test -n "$LIVE_OPENROUTER_API_KEY" || { echo "LIVE_OPENROUTER_API_KEY is not set" >&2; exit 1; } - name: Start the local stack (fake model provider) - env: - SUPABASE_INTERNAL_IMAGE_REGISTRY: public.ecr.aws run: .claude/skills/verify-debatelab/bin/verify.sh up - - name: Agentic suite - if: inputs.explore == '' - env: - LIVE_OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }} - E2E_AGENT_MODEL: ${{ vars.E2E_AGENT_MODEL }} - E2E_JUDGE_MODEL: ${{ vars.E2E_JUDGE_MODEL }} - # Bug-bash repro tests fail until their bug is fixed; they run on demand. - run: .claude/skills/verify-debatelab/bin/verify.sh e2e run --exclude-tag repro --reporter list,markdown,junit - - name: Exploration - if: inputs.explore != '' env: LIVE_OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }} E2E_AGENT_MODEL: ${{ vars.E2E_AGENT_MODEL }} E2E_JUDGE_MODEL: ${{ vars.E2E_JUDGE_MODEL }} - GOAL: ${{ inputs.explore }} - PERSONA: ${{ inputs.persona }} - TARGET: ${{ inputs.target }} + GOAL: ${{ matrix.leg.goal }} + PERSONA: ${{ matrix.leg.persona }} + TARGET: ${{ matrix.leg.target }} run: >- .claude/skills/verify-debatelab/bin/verify.sh e2e explore "$GOAL" --session student --agent "$PERSONA" --target "$TARGET" @@ -112,12 +118,11 @@ jobs: if: always() run: .claude/skills/verify-debatelab/bin/verify.sh down - # Local test data only: fake model output, throwaway demo users. - name: Upload evidence if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: agentic-e2e-${{ github.run_id }} + name: agentic-explore-${{ matrix.leg.persona }}-${{ github.run_id }} path: | .verify/artifacts/ .verify/logs/ diff --git a/AGENTS.md b/AGENTS.md index 3b9a262b6..9f82aa068 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -98,7 +98,7 @@ $H down # always, including after failures - The stack holds the same lock as `./.superset/db-gate.sh`, so only one worktree can run it at a time. - Most tools work inside a Current Debate. Recipes start one with `startDebate` or answer the motion dialog with `chooseForMotion`; see the skill's Drive section. - Neither mode proves Stripe Checkout, real Stripe cancellation, ElevenLabs voice audio, Google sign-in or export, staging, or production. -- Agentic tests (`agentic/*.e2e.ts`) are an AI agent driving goals and judging screens, with exact assertions on the result. They catch what selectors cannot: stray untranslated text, clipped layouts, off-topic model output in live mode. They also run explorations that hunt for bugs as a persona. The agent calls its own OpenRouter models (free Qwen acts, the Muse Spark contributor tier judges), even on a fake stack. A flow change that breaks an agentic test updates it in the same PR; commit new or re-recorded `agentic/replay/` entries with it. See [docs/testing.md](docs/testing.md#agentic-browser-tests). +- Agentic tests (`agentic/*.e2e.ts`) are an AI agent driving goals and judging screens, with exact assertions on the result. They cover a journey per tool and every page in English, Spanish and Hungarian, and fail on page errors, `console.error`, and Supabase 5xx answers. They catch what selectors cannot: stray untranslated text, clipped layouts, off-topic model output in live mode. They also run explorations that hunt for bugs as a persona. The agent calls its own OpenRouter models (free Qwen acts, the Muse Spark contributor tier judges), even on a fake stack. A flow change that breaks an agentic test updates it in the same PR; commit new or re-recorded `agentic/replay/` entries with it. See [docs/testing.md](docs/testing.md#agentic-browser-tests). ### Keep the skill working diff --git a/agentic/ai-tools.e2e.ts b/agentic/ai-tools.e2e.ts new file mode 100644 index 000000000..cd732236a --- /dev/null +++ b/agentic/ai-tools.e2e.ts @@ -0,0 +1,57 @@ +import { admin, agentic, aiRequestsOf, AI_TIMEOUT, debateControl, expect, fakeAnswer, LIVE, test } from "./support/fixtures"; + +// One goal per tool, pinned by the ai_requests row the Edge Function wrote, +// the fake provider's marker on screen (fake mode), and a judge reading the +// result. The recipes in verify-debatelab cover the exact response fields. +const options = { ...agentic, tags: ["journey"] }; + +test("a student finds the weaknesses in an argument", options, async ({ app, agent, screen, student }) => { + await app.open("/app/attack"); + await agent.act("Enter the argument {argument} and find its weaknesses.", { + params: { argument: "Free public transport cuts congestion because more people ride the bus." }, + }); + + await expect.poll(() => aiRequestsOf(student), { timeout: AI_TIMEOUT }).toContain("attack: success"); + if (!LIVE) await expect(fakeAnswer(screen)).toBeVisible({ timeout: AI_TIMEOUT }); + await agent.assert("The page lists weaknesses in the student's argument, each with an explanation."); +}); + +test("a student brainstorms a motion and it is filed in a new debate", options, async ({ app, agent, browser, screen, student }) => { + const motion = "This House would ban zoos"; + await app.open("/app/brainstorm"); + await agent.act("Brainstorm angles on the motion {motion}. If the app offers to start a debate with this motion, start it.", { + params: { motion }, + }); + + await expect(debateControl(browser)).toHaveAttribute("aria-label", `Current debate: ${motion}`); + if (!LIVE) await expect(fakeAnswer(screen)).toBeVisible({ timeout: AI_TIMEOUT }); + const userId = await student.userId(); + await expect.poll(async () => (await admin.from("brainstorms").select("debates(motion)").eq("user_id", userId)).data, { timeout: 15_000 }) + .toEqual([{ debates: { motion } }]); + await agent.assert("The page shows brainstormed angles or ideas for the motion, each with a title and an explanation."); +}); + +test("a student looks up factoids for a statement", options, async ({ app, agent, student }) => { + await app.open("/app/factoids"); + await agent.act("Find factoids for the statement {statement}.", { + params: { statement: "Cities with free buses have fewer cars downtown." }, + }); + + await expect.poll(() => aiRequestsOf(student), { timeout: AI_TIMEOUT }).toContain("factoids: success"); + // Fake mode has no Exa key, so every factoid is unverified and dropped. + await agent.assert(LIVE + ? "The page lists factoids about the statement, each with a source." + : "The search has finished, and the page either lists factoids or clearly tells the student that no verified factoids were found. It is not stuck loading and shows no error."); +}); + +test("a student asks the coach a question and gets a streamed answer", options, async ({ app, agent, screen, student }) => { + await app.open("/app/ask"); + await agent.act("Ask your debate coach: {question}", { + params: { question: "How do I weigh access against cost in a rebuttal?" }, + }); + + await expect.poll(() => aiRequestsOf(student), { timeout: AI_TIMEOUT }).toContain("ask: success"); + if (!LIVE) await expect(screen.getByText(/streamed answer/).first()).toBeVisible({ timeout: AI_TIMEOUT }); + await expect(screen.getByText("Response was interrupted")).toHaveCount(0); + await agent.assert("The coach has answered the student's question in the chat, and the answer is complete rather than cut off or showing an error."); +}); diff --git a/agentic/case-generation.e2e.ts b/agentic/case-generation.e2e.ts index febeb4965..581ddeb82 100644 --- a/agentic/case-generation.e2e.ts +++ b/agentic/case-generation.e2e.ts @@ -1,8 +1,8 @@ -import { admin, agentic, AI_TIMEOUT, debateControl, expect, LIVE, test } from "./support/fixtures"; +import { admin, agentic, AI_TIMEOUT, debateControl, expect, fakeAnswer, LIVE, test } from "./support/fixtures"; const MOTION = "This House would replace school exams with continuous assessment"; -test("a student starts a debate and generates a case for it", agentic, async ({ app, agent, browser, screen, student }) => { +test("a student starts a debate and generates a case for it", { ...agentic, tags: ["journey"] }, async ({ app, agent, browser, screen, student }) => { await app.open("/app"); await agent.act("Start a new debate on the motion {motion}. Then open Generate and generate a debate case for that motion.", { params: { motion: MOTION }, @@ -23,4 +23,11 @@ test("a student starts a debate and generates a case for it", agentic, async ({ // Recipes never assert on model wording; a judge can check that the case is on topic. await agent.assert(`The generated case argues about the motion "${MOTION}", not about an unrelated topic.`); } + + // The saved case reopens from History. + await app.open("/app"); + await agent.act("Open History and load the debate case you generated."); + await expect(screen.getByRole("button", "Regenerate Arguments")).toBeVisible(); + if (!LIVE) await expect(fakeAnswer(screen)).toBeVisible(); + await agent.assert("A debate case with a thesis and contentions is open again on screen."); }); diff --git a/agentic/first-debate.e2e.ts b/agentic/first-debate.e2e.ts index d85f681c0..f53d04871 100644 --- a/agentic/first-debate.e2e.ts +++ b/agentic/first-debate.e2e.ts @@ -4,7 +4,7 @@ const MOTION = "This House would make debate compulsory in secondary schools"; // The onboarding copy and step order change often. The goal stays the same: // a new student finishes the tour with their first debate started. -test("a new student starts their first debate from the onboarding tour", agentic, async ({ agent, browser, student }) => { +test("a new student starts their first debate from the onboarding tour", { ...agentic, tags: ["journey"] }, async ({ agent, browser, student }) => { await agent.act("Follow the onboarding tour. At its first step, start a debate on the motion {motion}, then go through the remaining steps until the tour is finished.", { params: { motion: MOTION }, }); diff --git a/agentic/hungarian.e2e.ts b/agentic/hungarian.e2e.ts deleted file mode 100644 index 08fe81e75..000000000 --- a/agentic/hungarian.e2e.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { agentic, expect, hungarianTest as test } from "./support/fixtures"; - -// e2e/hungarian-parity.spec.ts checks lang, headings and chosen labels, and by -// design cannot reject stray English. A judge reading each screen can. -const PAGES = ["/app", "/app/generate", "/app/brainstorm", "/app/sparring", "/app/library", "/app/history", "/app/settings"]; - -for (const path of PAGES) { - test(`${path} reads in Hungarian and fits the screen`, agentic, async ({ app, agent, browser }) => { - await app.open(path); - await expect(browser.locator("html")).toHaveAttribute("lang", "hu"); - await expect(browser.locator("main h1, main h2").first()).toBeVisible(); - await agent.assert( - "Every piece of interface text on this screen (navigation, headings, buttons, field labels, placeholders, hints and empty states) is in Hungarian. Product names such as DebateLab, AI model names (Claude Sonnet 5) and plan names (Pro), debate format codes, language names offered as choices (English, Español, Magyar), and user content do not count. An English loanword inflected with a Hungarian ending, such as \"coachingot\", is Hungarian.", - { vision: true }, - ); - await agent.assert( - "No label, heading, button, navigation or message text on this screen is cut off, overlapping other text, or running past the edge of its button or the screen. A placeholder or value that scrolls inside a text field is normal and does not count.", - { vision: "only" }, - ); - }); -} diff --git a/agentic/pages.e2e.ts b/agentic/pages.e2e.ts new file mode 100644 index 000000000..26f9e8181 --- /dev/null +++ b/agentic/pages.e2e.ts @@ -0,0 +1,66 @@ +import type { Browser } from "@e2e-dev/web"; +import type { Agent } from "e2e"; +import { agentic, expect, LANGUAGES, localeTest, visitorTest, type Language } from "./support/fixtures"; + +// Every page, in every interface language, on both viewports. Selectors pin +// what they can (lang, a heading, no raw values); the judge reads the screen +// for what they cannot: stray untranslated text, error notices, and clipped +// or overlapping text. e2e/hungarian-parity.spec.ts and authed-pages.spec.ts +// render the same pages with a stubbed session and by design cannot judge copy. +const APP_PAGES = [ + "/app", + "/app/debate", + "/app/generate", + "/app/attack", + "/app/brainstorm", + "/app/factoids", + "/app/ask", + "/app/sparring", + "/app/crossfire", + "/app/flowchart", + "/app/timer", + "/app/library", + "/app/history", + "/app/settings", +]; +const PUBLIC_PAGES = ["/", "/login", "/terms", "/privacy", "/expocit/sources"]; + +const NAMES: Record = { en: "English", es: "Spanish", hu: "Hungarian" }; + +const EXEMPT = + "Product names such as DebateLab and ExpoCit, AI model names (Claude Sonnet 5), plan names (Pro), debate format codes, language names offered as choices (English, Español, Magyar), and user content do not count."; + +async function checkPage(browser: Browser, agent: Agent, language: Language) { + await expect(browser.locator("html")).toHaveAttribute("lang", language); + await expect(browser.locator("h1, h2, h3").first()).toBeVisible(); + // Values a broken render leaks into the page as text. + await expect(browser.locator("body")).not.toContainText(/\[object Object\]|\bundefined\b|\bNaN\b/); + + const translated = language === "en" ? "" : ` Every piece of interface text (navigation, headings, buttons, field labels, placeholders, hints and empty states) is in ${NAMES[language]}. ${EXEMPT}`; + await agent.assert( + `The screen shows no error message, failed-to-load notice, or raw placeholder text such as a translation key (settings.title).${translated}`, + { vision: true }, + ); + await agent.assert( + "No label, heading, button, navigation or message text on this screen is cut off, overlapping other text, or running past the edge of its button or the screen. A placeholder or value that scrolls inside a text field, and an intentional ellipsis (…), are normal and do not count.", + { vision: "only" }, + ); +} + +for (const language of LANGUAGES) { + const test = localeTest(language); + for (const path of APP_PAGES) { + test(`${path} [${language}] reads correctly and fits the screen`, { ...agentic, tags: ["pages", language] }, async ({ app, agent, browser }) => { + await app.open(path); + await checkPage(browser, agent, language); + }); + } + + const visitor = visitorTest(language); + for (const path of PUBLIC_PAGES) { + visitor(`${path} [${language}] reads correctly and fits the screen for a visitor`, { ...agentic, tags: ["pages", language] }, async ({ agent, browser, visitor }) => { + await visitor.open(path); + await checkPage(browser, agent, language); + }); + } +} diff --git a/agentic/practice.e2e.ts b/agentic/practice.e2e.ts new file mode 100644 index 000000000..fd8f7be6c --- /dev/null +++ b/agentic/practice.e2e.ts @@ -0,0 +1,42 @@ +import { admin, agentic, AI_TIMEOUT, expect, fakeAnswer, LIVE, test } from "./support/fixtures"; + +const options = { ...agentic, tags: ["journey"] }; + +// A whole round as one goal: the opponent answers between speeches, so the +// next speech box appears only after a model call. +test("a student practises a full sparring round and gets a verdict", options, async ({ app, agent, screen, student }) => { + const motion = "This House would lower the voting age to 16"; + await app.open("/app/sparring"); + await agent.act( + "Start a Lincoln-Douglas sparring round on the motion {motion}. If the app offers to start a debate with this motion, start it. Then deliver each of your speeches by typing {speech} and submitting it, until the round ends and the verdict is shown.", + { params: { motion, speech: "Sixteen-year-olds pay taxes and should have a say in how that money is spent." } }, + ); + + await expect(screen.getByRole("heading", "Here is what happened in the room.")).toBeVisible({ timeout: AI_TIMEOUT * 2 }); + const userId = await student.userId(); + await expect.poll(async () => (await admin.from("sparring_sessions").select("status, debates(motion)").eq("user_id", userId)).data, { timeout: 15_000 }) + .toEqual([{ status: "complete", debates: { motion } }]); + await agent.assert("The round is over and the page shows the judge's verdict with feedback on the student's speeches."); +}); + +test("a student cross-examines the AI and gets a debrief", options, async ({ app, agent, screen, student }) => { + const motion = "This House would make public transport free"; + await app.open("/app/crossfire"); + await agent.act( + "Start a Crossfire in which you question the AI, on the resolution {motion} with the argument {argument}. If the app offers to start a debate with this motion, start it. Ask {question}, wait for the answer, then end the crossfire.", + { + params: { + motion, + argument: "Fares keep the poorest riders off the bus.", + question: "What evidence links free fares to fewer cars?", + }, + }, + ); + + await expect(screen.getByTestId("crossfire-debrief")).toBeVisible({ timeout: AI_TIMEOUT }); + if (!LIVE) await expect(fakeAnswer(screen)).toBeVisible(); + const userId = await student.userId(); + await expect.poll(async () => (await admin.from("crossfires").select("title, debates(motion)").eq("user_id", userId)).data, { timeout: 15_000 }) + .toEqual([{ title: motion, debates: { motion } }]); + await agent.assert("The crossfire has ended and the page shows a debrief of the exchange with feedback for the student."); +}); diff --git a/agentic/support/capture-errors.js b/agentic/support/capture-errors.js new file mode 100644 index 000000000..62aea7197 --- /dev/null +++ b/agentic/support/capture-errors.js @@ -0,0 +1,41 @@ +// Runs before the app's own scripts in every document (web({ initScripts })). +// It records what the page hides from the screen: uncaught errors, rejected +// promises, console.error, and 5xx answers from Supabase. The student fixture +// reads the list at teardown and fails the test on anything it does not expect. +// sessionStorage keeps the list across reloads and navigations in the tab. +(() => { + const KEY = "__agentic_errors"; + const record = (kind, text) => { + try { + const list = JSON.parse(sessionStorage.getItem(KEY) || "[]"); + if (list.length < 50) list.push(`${kind}: ${String(text).slice(0, 500)} [${location.pathname}]`); + sessionStorage.setItem(KEY, JSON.stringify(list)); + } catch { + // Storage can be unavailable in an opaque-origin frame; nothing to record there. + } + }; + const describe = (value) => (value instanceof Error ? `${value.name}: ${value.message}` : typeof value === "string" ? value : (() => { + try { return JSON.stringify(value); } catch { return String(value); } + })()); + + window.addEventListener("error", (event) => record("pageerror", event.error ? describe(event.error) : event.message)); + window.addEventListener("unhandledrejection", (event) => record("unhandledrejection", describe(event.reason))); + + const consoleError = console.error.bind(console); + console.error = (...args) => { + record("console.error", args.map(describe).join(" ")); + consoleError(...args); + }; + + const nativeFetch = window.fetch.bind(window); + window.fetch = async (input, init) => { + const response = await nativeFetch(input, init); + if (response.status >= 500) { + const url = new URL(typeof input === "string" || input instanceof URL ? String(input) : input.url, location.href); + if (url.pathname.startsWith("/functions/") || url.pathname.startsWith("/rest/") || url.pathname.startsWith("/auth/")) { + record("http", `${response.status} ${(init && init.method) || (input instanceof Request ? input.method : "GET")} ${url.pathname}`); + } + } + return response; + }; +})(); diff --git a/agentic/support/fixtures.ts b/agentic/support/fixtures.ts index 18f7a52db..f81732f38 100644 --- a/agentic/support/fixtures.ts +++ b/agentic/support/fixtures.ts @@ -1,6 +1,6 @@ import { test as base, type Browser } from "@e2e-dev/web"; import { expect, type Screen } from "e2e"; -import { admin, createDemoUser, deleteDemoUser, stackReady, type DemoUser } from "../../e2e/full-stack/stack-core"; +import { admin, createDemoUser, deleteDemoUser, MARK, stackReady, type DemoUser } from "../../e2e/full-stack/stack-core"; export { admin, fake, MARK } from "../../e2e/full-stack/stack-core"; export { expect }; @@ -27,14 +27,20 @@ export const agentic = { const labels = { en: { tab: "Demo code", field: "Demo code", submit: "Sign in with demo code" }, + es: { tab: "Código demo", field: "Código demo", submit: "Iniciar sesión con código demo" }, hu: { tab: "Demókód", field: "Demókód", submit: "Bejelentkezés demókóddal" }, } as const; -type Language = keyof typeof labels; +export type Language = keyof typeof labels; +export const LANGUAGES = Object.keys(labels) as Language[]; type StudentOptions = { tour?: boolean; language?: Language }; export type Student = DemoUser & { codeId: string }; -/** Persisted client state the app reads at boot. e2e has no init scripts, so the caller reloads. */ +/** + * Persisted client state the app reads at boot; the caller reloads. Seeded + * once rather than as an init script, which would reseed on every navigation + * and undo what the app writes during the test. + */ async function seedClientState(browser: Browser, { tour = false, language = "en" }: StudentOptions) { await browser.evaluate((seed: { tourPending: boolean; language: string }) => { localStorage.setItem("debate-trainer-onboarding", JSON.stringify({ @@ -58,31 +64,54 @@ async function signIn(screen: Screen, browser: Browser, code: string, language: await browser.waitForURL(/\/app$/, { timeout: 20_000 }); } +/** + * What `support/capture-errors.js` recorded in this tab: uncaught errors, + * rejected promises, console.error, and Supabase 5xx answers. Reading clears + * the list. + */ +export async function pageErrors(browser: Browser): Promise { + return browser.evaluate(() => { + const list = JSON.parse(sessionStorage.getItem("__agentic_errors") || "[]") as string[]; + sessionStorage.removeItem("__agentic_errors"); + return list; + }); +} + +/** + * Fails the test on any request to a hosted Supabase project, anything the + * page logged as an error (see `pageErrors`), and a page wider than its + * viewport, checked after the test body on whatever page it ended. + */ +async function guarded(browser: Browser, body: () => Promise) { + const remote: string[] = []; + await browser.route(/^https:\/\/[^/]+\.supabase\.co\//, async (route) => { + remote.push(route.request.url); + await route.abort(); + }); + await body(); + expect(await pageErrors(browser), "errors the page logged").toEqual([]); + const overflow = await browser.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth); + expect(overflow, "horizontal page overflow in px").toBeLessThanOrEqual(1); + expect(remote, "requests to a hosted Supabase project").toEqual([]); +} + /** * A `test` whose `student` fixture is a fresh demo-code user (Pro, daily limit - * 999, no Current Debate), signed in and deleted afterwards. Any request to a - * hosted Supabase project is aborted and fails the test, and so does a page - * wider than its viewport. + * 999, no Current Debate), signed in and deleted afterwards, under `guarded`. */ function studentTest(options: StudentOptions = {}) { const language = options.language ?? "en"; return base.extend<{ student: Student }>({ student: async ({ app, screen, browser }, provide) => { - const remote: string[] = []; - await browser.route(/^https:\/\/[^/]+\.supabase\.co\//, async (route) => { - remote.push(route.request.url); - await route.abort(); - }); const user = await createDemoUser(`agentic-${language}`); try { - await app.open("/login"); - await seedClientState(browser, options); - await browser.reload(); - await signIn(screen, browser, user.code, language); - await provide(user); - const overflow = await browser.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth); - expect(overflow, "horizontal page overflow in px").toBeLessThanOrEqual(1); - expect(remote, "requests to a hosted Supabase project").toEqual([]); + await guarded(browser, async () => { + await app.open("/login"); + await seedClientState(browser, options); + await browser.reload(); + await signIn(screen, browser, user.code, language); + await provide(user); + }); } finally { await deleteDemoUser(user); } @@ -90,12 +119,34 @@ function studentTest(options: StudentOptions = {}) { }); } +/** + * A signed-out visitor using the given interface language, under `guarded`. + * The `visitor` fixture opens a path with that language seeded. + */ +export function visitorTest(language: Language) { + return base.extend<{ visitor: { open: (path: string) => Promise } }>({ + visitor: async ({ app, browser }, provide) => { + await guarded(browser, async () => { + await provide({ + open: async (path) => { + await app.open(path); + await seedClientState(browser, { language }); + await browser.reload(); + }, + }); + }); + }, + }); +} + /** A signed-in student who has already seen the onboarding tour. */ export const test = studentTest(); /** A brand-new account: the onboarding tour starts on /app. */ export const firstRunTest = studentTest({ tour: true }); /** A signed-in student using the Hungarian interface. */ export const hungarianTest = studentTest({ language: "hu" }); +/** A signed-in student using the given interface language. */ +export const localeTest = (language: Language) => studentTest({ language }); /** The visible header Current Debate control on either viewport. */ export function debateControl(browser: Browser) { @@ -108,3 +159,18 @@ export async function debatesOf(student: Student) { if (error) throw error; return data; } + +/** + * A model answer on screen. In fake mode every answer carries the provider's + * marker; live answers vary, so live tests add a judgment instead. + */ +export function fakeAnswer(screen: Screen) { + return screen.getByText(new RegExp(MARK)).first(); +} + +/** The `ai_requests` rows a user owns, oldest first, as `feature: status`. */ +export async function aiRequestsOf(student: Student) { + const { data, error } = await admin.from("ai_requests").select("feature, status").eq("user_id", await student.userId()).order("created_at"); + if (error) throw error; + return data.map((row) => `${row.feature}: ${row.status}`); +} diff --git a/agentic/workspace.e2e.ts b/agentic/workspace.e2e.ts new file mode 100644 index 000000000..c7ff77c59 --- /dev/null +++ b/agentic/workspace.e2e.ts @@ -0,0 +1,56 @@ +import { admin, agentic, debatesOf, expect, test } from "./support/fixtures"; + +// The non-AI tools: Library sources, Flowchart, Timer, and Settings. +const options = { ...agentic, tags: ["journey"] }; + +test("a student adds a source to the debate's Library and pauses it", options, async ({ app, agent, screen, student }) => { + const motion = "This House would ban single-use plastics"; + const title = "Plastic bag levy study"; + await app.open("/app"); + await agent.act("Start a new debate on the motion {motion}. Then open the Library and add a source titled {title} with the content {content}.", { + params: { motion, title, content: "After a 5p levy, single-use bag sales at major retailers fell by more than 95 percent." }, + }); + + const toggle = screen.getByRole("switch", `Toggle ${title}`); + await expect(toggle).toBeChecked(); + const [debate] = await debatesOf(student); + const sources = async () => (await admin.from("sources").select("title, is_active, debate_id").eq("user_id", await student.userId())).data; + await expect.poll(sources).toEqual([{ title, is_active: true, debate_id: debate.id }]); + + await agent.act("Pause the source {title} so it is no longer used.", { params: { title } }); + await expect(toggle).not.toBeChecked(); + await expect.poll(sources).toEqual([{ title, is_active: false, debate_id: debate.id }]); +}); + +test("a student edits and saves a flowchart", options, async ({ app, agent, screen, student }) => { + const title = "Zoo ban flow"; + await app.open("/app/flowchart"); + await agent.act("Delete the node Argument 1, then save the flowchart with the title {title}.", { params: { title } }); + + await expect(screen.getByText("Argument 1", { exact: true })).toHaveCount(0); + await expect.poll(async () => (await admin.from("flowcharts").select("title, nodes").eq("user_id", await student.userId())).data + ?.map((row) => ({ title: row.title, hasArgument1: JSON.stringify(row.nodes).includes("Argument 1") }))) + .toEqual([{ title, hasArgument1: false }]); +}); + +test("a student times a speech and pauses the timer", options, async ({ app, agent, browser, screen }) => { + await app.open("/app/timer"); + const display = browser.locator('[role="timer"][aria-label="Timer display"]'); + await expect(display).toBeVisible(); + const initial = (await display.textContent()) ?? ""; + await agent.act("Start the speech timer, then pause it."); + + await expect(screen.getByText("PAUSED").first()).toBeVisible(); + await expect(display).not.toHaveText(initial); +}); + +test("a student switches the interface to Spanish and the AI output to Hungarian", options, async ({ app, agent, browser, student }) => { + await app.open("/app/settings"); + await agent.act("Switch the interface language to Español."); + await expect(browser.locator("html")).toHaveAttribute("lang", "es"); + + await agent.act("Set the AI output language to Magyar."); + await expect.poll(async () => (await admin.from("user_settings").select("output_language").eq("user_id", await student.userId()).single()).data) + .toEqual({ output_language: "hu" }); + await agent.assert("The Settings page is shown in Spanish, and its AI output language setting reads Magyar."); +}); diff --git a/docs/testing.md b/docs/testing.md index e0963fc24..837a079d3 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -236,34 +236,36 @@ LIVE_PROVIDERS=1 LIVE_MODEL=as-requested LIVE_OPENROUTER_API_KEY=... LIVE_EXA_AP H=.claude/skills/verify-debatelab/bin/verify.sh $H up $H e2e run # the suite on both viewports -$H e2e run agentic/case-generation.e2e.ts --target desktop # one file, one viewport +$H e2e run --target desktop --tag journey # the tool journeys on one viewport +$H e2e run agentic/pages.e2e.ts --grep '\[hu\]' # the page sweep in one language $H e2e explore "Practise a Sparring round and report anything broken" --session student --agent first-timer $H down ``` -[`agentic/`](../agentic/) holds tests for [e2e](https://e2e.tester.army/docs), TesterArmy's agentic test runner (`e2e@0.15.2`, configured in [`e2e.config.ts`](../e2e.config.ts)). In each test, an AI agent works toward a goal in plain language (`agent.act`) and judges what is on screen (`agent.assert`). Locators and database reads then pin the result exactly. The suite covers what the Playwright layers cannot: +[`agentic/`](../agentic/) holds tests for [e2e](https://e2e.tester.army/docs), TesterArmy's agentic test runner (`e2e@0.17.0` with `@e2e-dev/web@0.12.0`, configured in [`e2e.config.ts`](../e2e.config.ts)). In each test, an AI agent works toward a goal in plain language (`agent.act`) and judges what is on screen (`agent.assert`). Locators and database reads then pin the result exactly. The suite covers what the Playwright layers cannot: -- Goal-level journeys that survive copy and layout changes: a new student starts their first debate from the onboarding tour, and a student starts a debate and generates a case. -- Judgments a selector cannot make: every main page reads in Hungarian (`e2e/hungarian-parity.spec.ts` deliberately does not reject stray English), and no text is cut off or overflowing on either viewport. -- In live mode (`up --live`), whether the generated case argues the motion at all. Recipes never assert model wording. +- Journeys (tag `journey`), one goal per tool, that survive copy and layout changes: the onboarding tour starting a first debate, Generate and reopening the case from History, Attack, Brainstorm filed in a new debate, Factoids, Ask, a full Sparring round to its verdict, Crossfire to its debrief, adding and pausing a Library source, saving a Flowchart, the speech Timer, and switching interface and AI output language in Settings. Each is pinned by a locator, the rows the app wrote (`ai_requests`, `brainstorms`, `cases`, `sparring_sessions`, `crossfires`, `sources`, `flowcharts`, `user_settings`), and in fake mode the provider's `FAKE-PROVIDER` marker on screen. +- The page sweep (`pages.e2e.ts`, tag `pages`): every signed-in page and every public page, in English, Spanish and Hungarian, on both viewports. Selectors check `lang`, a heading, and that no `undefined`, `NaN` or `[object Object]` leaked into the page. The judge checks for error notices, raw translation keys, untranslated interface text (which `e2e/hungarian-parity.spec.ts` deliberately cannot reject), and clipped or overlapping text. +- Hidden failures. [`agentic/support/capture-errors.js`](../agentic/support/capture-errors.js) runs before the app in every document and records uncaught errors, rejected promises, `console.error` (which includes React's warnings and the route error boundary), and 5xx answers from Supabase. Every test fails on any of them, and on a request to a hosted Supabase project or a page wider than its viewport. +- In live mode (`up --live`), whether model output is on topic: the generated case argues the motion, and factoids carry sources. Recipes never assert model wording. - Exploration. `e2e explore` sends a persona through the app with a goal and reports findings with expected and observed behavior, reproduction steps, and screenshots. The personas are the named agents in `e2e.config.ts`: `first-timer`, `coach`, `edge-input`, and `hungarian`. How it runs: - Always through `verify.sh e2e `. The wrapper runs `doctor`, loads the stack environment, turns e2e telemetry off, and takes the testing agent's key from `OPENROUTER_API_KEY`, `LIVE_OPENROUTER_API_KEY`, or `~/.config/debatelab/live-verify.env`. It writes the report (`summary.md`, `report.json`), screenshots, traces, and failure videos to `.verify/artifacts/-e2e--/`. The config refuses a non-loopback URL and never starts a dev server, because a plain `npm run dev` reaches production. -- Each test signs in a fresh demo-code user through the Login page and deletes it afterwards. A request to a hosted Supabase project, or a page wider than its viewport, fails the test. -- There are two targets, `desktop` (1280 by 800) and `mobile` (393 by 851). e2e 0.15 sets only the viewport size: it does not emulate touch or a phone user agent, so the Playwright Pixel 5 project still owns mobile behavior. -- The testing agent has two models on OpenRouter, separate from the app's provider mode, and a fake-mode stack still needs both. The actor (`agent.act` and explorations) is `qwen/qwen3.8-27b:free`. It is free, and recorded flows replay without it, but it is slow and its upstream sometimes answers `temporarily rate-limited upstream`. Treat that as infrastructure and rerun with `--last-failed`. The judge (`agent.assert`, `waitFor`, `extract`) is `meta/muse-spark-1.3-contributor` ($0.10 and $0.20 per million input and output tokens), the same tier as the app's live pin. It answers every judgment on every run. A full suite run costs under two cents, and a 4-step exploration well under one. The judge cannot act, because Meta's endpoint accepts only `tool_choice: "auto"`. `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` override the two; `E2E_AGENT_MODEL=openai/gpt-6-luna` gives fast, reliable explorations for cents. Both default models may keep prompts, so use made-up debate content only. The agent's context explains that fake AI answers are canned free-transport text. -- Replay. When a later assertion passes, an `agent.act` flow is recorded in [`agentic/replay/`](../agentic/replay/). Later runs replay it with no model call until the app changes, at which point the agent takes over and records again. Commit new and changed entries with the test that made them. Local runs read and write the cache; CI only reads it. `agent.assert`, `waitFor`, and `extract` always call the model. -- Without a model key or the stack environment, agent tests skip, so a bare `npx e2e run` proves nothing. +- Each signed-in test signs in a fresh demo-code user through the Login page and deletes it afterwards. Public-page tests use the `visitorTest` fixture with the same checks. +- There are two targets, `desktop` (1280 by 800) and `mobile` (393 by 851). e2e does not emulate touch. The app lays out by viewport width and never reads the user agent, so the viewport is what matters; the Playwright Pixel 5 project still owns touch behavior. +- The testing agent has two models on OpenRouter, separate from the app's provider mode, and a fake-mode stack still needs both. The actor (`agent.act` and explorations) is `qwen/qwen3.8-27b:free`. It is free, and recorded flows replay without it, but it is slow and its upstream sometimes answers `temporarily rate-limited upstream`. Treat that as infrastructure and rerun with `--last-failed`. The judge (`agent.assert`, `waitFor`, `extract`) is `meta/muse-spark-1.3-contributor` ($0.10 and $0.20 per million input and output tokens), the same tier as the app's live pin. It answers every judgment on every run. COST_LINE The judge cannot act, because Meta's endpoint accepts only `tool_choice: "auto"`. `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` override the two. Both default models may keep prompts, so use made-up debate content only. The agent's context explains that fake AI answers are canned free-transport text. +- Replay. When a later check passes, an `agent.act` flow is recorded in [`agentic/replay/`](../agentic/replay/). Later runs replay it with no model call and verify that its effect happened, until the app changes and the agent takes over and records again. The model is not part of the cache key, so record new flows with a fast actor, `E2E_AGENT_MODEL=openai/gpt-6-luna $H e2e run ...` (cents per run), and CI replays them with the free one. Commit new and changed entries with the test that made them. Local runs read and write the cache; CI only reads it. Changing an instruction, its params, the test or target name, the agent's `context`, or e2e's minor version re-records. +- Without a model key or the stack environment, agent tests skip, so a bare `npx e2e run` proves nothing. `failOnSkippedFailure` fails a run in which a test skipped after a failure. -To write a test, import `test`, `expect`, and the fixtures from [`agentic/support/fixtures.ts`](../agentic/support/fixtures.ts) and pass the `agentic` options. Follow every `agent.act` with a locator assertion, because only verified flows are recorded. Prefer `agent.assert` for what a selector cannot express, and an exact `expect` for everything else. The [`e2e` skill](../.claude/skills/e2e/SKILL.md), `npx e2e guide `, and `node_modules/e2e/docs` cover the API for the installed version. +To write a test, import `test`, `expect`, and the fixtures from [`agentic/support/fixtures.ts`](../agentic/support/fixtures.ts) and pass the `agentic` options with a `journey` or `pages` tag. Follow every `agent.act` with a locator assertion, because only verified flows are recorded, and pin side effects with a database read through `admin`. Prefer `agent.assert` for what a selector cannot express, and an exact `expect` for everything else. A new user-facing tool gets a journey, and a new route joins `APP_PAGES` or `PUBLIC_PAGES` in `pages.e2e.ts`. When the judge flags vocabulary the app uses on purpose, add it to `context` in `e2e.config.ts` rather than weakening the assertion. The [`e2e` skill](../.claude/skills/e2e/SKILL.md), `npx e2e guide `, and `node_modules/e2e/docs` cover the API for the installed version. A bug bash runs several explorations in parallel, one charter per area and persona, then proves each finding with a repro test. Ask an agent to "bug bash "; the `e2e` skill's `bug-bash` topic has the procedure. A finding counts only when its repro test fails for the reported reason. Tag repro tests `{ tags: ["repro"] }` so the scheduled suite skips them until the fix lands, then remove the tag so they become regression tests. The client runs on a Vite dev server, so a route that is still compiling can look like a dead link to an explorer. Reproduce before you believe it. -[`.github/workflows/agentic-e2e.yml`](../.github/workflows/agentic-e2e.yml) runs the suite nightly at 03:47 UTC, on pull requests that change it, and on demand. On demand with a goal, it runs one exploration with the chosen persona and viewport instead. It uses the `LIVE_OPENROUTER_API_KEY` secret (the `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` repository variables override the models), fails when the key is missing, and writes the report to the job summary. It is not a required check: model judgments can flake, and a red nightly is a finding to triage. +[`.github/workflows/agentic-e2e.yml`](../.github/workflows/agentic-e2e.yml) runs the suite nightly at 03:47 UTC and on pull requests that change the app (`src/`, `public/`, Edge Functions, migrations, dependencies) or the suite, one job per viewport. Every Sunday at 05:13 UTC it runs a bug bash instead: one exploration per persona, each with its own charter. On demand with a goal, it runs one exploration with the chosen persona and viewport. It uses the `LIVE_OPENROUTER_API_KEY` secret (the `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` repository variables override the models), fails when the key is missing, and writes the report to the job summary. Explorations never fail the workflow: their findings are leads to reproduce. The suite is not a required check: model judgments can flake, and a red nightly is a finding to triage. -e2e is pre-1.0 and its minor releases have broken APIs, so it is pinned exactly. Read its changelog before upgrading, and expect the replay cache to re-record after an engine minor bump. +e2e is pre-1.0 and its minor releases have broken APIs, so it is pinned exactly. Read its changelog before upgrading, refresh the vendored skill as its header says, and expect the replay cache to re-record after a minor bump. ## Production canary diff --git a/e2e.config.ts b/e2e.config.ts index 4a5346e34..8a821391c 100644 --- a/e2e.config.ts +++ b/e2e.config.ts @@ -31,7 +31,7 @@ const context = [ "Each student works in one Current Debate at a time. The header control reads \"Start debate\" until one exists, then names the motion. Tool pages show a strip \"Working in: \".", "Tools: Generate (a full debate case: thesis, contentions, predicted opponent case, prepared counters), Attack, Brainstorm, Factoids, Ask, Sparring (a practice round against an AI opponent, judged at the end), Crossfire (cross-examination practice), Flowchart, Timer, Library (sources for the Current Debate), and History (saved results).", "A tool given a motion with no Current Debate asks \"Start a debate with this motion?\". A motion that differs from the Current Debate asks \"This is a different motion\".", - "The interface language can be English, Spanish or Hungarian. Source titles, citations, user content and model names may stay in their own language. Hungarian abbreviates AI as MI (mesterséges intelligencia), so \"MI-modell\" is Hungarian.", + "The interface language can be English, Spanish or Hungarian. Source titles, citations, user content and model names may stay in their own language. Hungarian abbreviates AI as MI (mesterséges intelligencia), so \"MI-modell\" is Hungarian, and an English loanword with a Hungarian ending, such as \"coachingot\", is Hungarian. The Spanish interface deliberately keeps the loanwords coach, coaching and sparring (\"Pregunta a tu coach\", \"Para Coaches\", \"Rondas de sparring\"); they are Spanish there.", "Long values in selects, chips and narrow fields are shortened with an ellipsis (…) on purpose.", "This is a local test stack. Google sign-in, Stripe checkout, ElevenLabs voices and Google Docs export are not configured here, so their failures are expected.", "The client runs on a Vite dev server, so the first visit to a page can take a few seconds to render.", @@ -57,26 +57,36 @@ const agent = (persona?: string) => ({ system: [...rules, ...(persona ? [persona] : [])].join("\n"), }); +// Records page errors, console.error and Supabase 5xx answers in every +// document; the student fixture fails the test on them. +const initScripts = [{ path: "agentic/support/capture-errors.js" }]; + export default { projectId: "debatelab", tests: "agentic/**/*.e2e.ts", targets: [ { name: "desktop", - engine: web({ viewport: { width: 1280, height: 800 } }), + engine: web({ viewport: { width: 1280, height: 800 }, initScripts }), app: { url: appUrl, identity: "debatelab-verify-stack", environment: "test" }, }, { - // A phone-sized viewport only: e2e 0.15 has no touch or user-agent emulation. + // A phone-sized viewport without touch emulation, which e2e does not + // offer. The app lays out by viewport width and never reads the user + // agent; the Playwright Pixel 5 project covers touch. name: "mobile", - engine: web({ viewport: { width: 393, height: 851 } }), + engine: web({ viewport: { width: 393, height: 851 }, initScripts }), app: { url: appUrl, identity: "debatelab-verify-stack", environment: "test" }, }, ], // AI steps wait on Edge Functions, and a free model can take minutes per judgment. timeout: 600_000, - // More parallel act steps only trip the free actor's upstream rate limit. - workers: process.env.CI ? 1 : 2, + // Judgments go to the paid judge, and replayed steps call no model, so three + // workers keep the page sweep short. Recording many new flows at once can + // trip the free actor's rate limit; record with a paid actor (docs/testing.md). + workers: 3, + // A test that skips after a failed soft assertion or attempt is a failure. + failOnSkippedFailure: true, // The dev server compiles lazy routes on first visit. assertionTimeout: 15_000, video: "retain-on-failure", diff --git a/package-lock.json b/package-lock.json index 4bcace633..7bccc8eef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -81,7 +81,7 @@ "zustand": "^5.0.10" }, "devDependencies": { - "@e2e-dev/web": "0.11.1", + "@e2e-dev/web": "0.12.0", "@eslint/js": "^9.32.0", "@openrouter/ai-sdk-provider": "^3.1.0", "@playwright/test": "^1.63.0", @@ -97,7 +97,7 @@ "@vitest/coverage-v8": "^4.1.8", "ai": "^7.0.127", "autoprefixer": "^10.4.21", - "e2e": "0.15.2", + "e2e": "0.17.0", "eslint": "^9.32.0", "eslint-plugin-react-hooks": "^5.2.0", "eslint-plugin-react-refresh": "^0.4.20", @@ -608,17 +608,22 @@ } }, "node_modules/@e2e-dev/web": { - "version": "0.11.1", - "resolved": "https://registry.npmjs.org/@e2e-dev/web/-/web-0.11.1.tgz", - "integrity": "sha512-ZmCTlkVei9sLRyObs4jftNewarv1tKYVE18ThOv3+96w3M5PC7vwBjYJxxPti9byzdSNF2MJYlQCXLAnguMIJQ==", + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@e2e-dev/web/-/web-0.12.0.tgz", + "integrity": "sha512-Ee5QaVZHSvjhVx5p2/kcYCxFerNr8BA8zLbja7OsvzF5Jp4Ku2mEri2KoMRol7i+DpCmfSUPSxI4KrzIPpJqdg==", "dev": true, "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "e2e-web": "dist/bin.js" + }, "engines": { "node": ">=22.12.0" }, "peerDependencies": { - "e2e": ">=0.15.0 <1", - "playwright": ">=1.63.0 <2" + "e2e": ">=0.15.0 <1" } }, "node_modules/@esbuild/aix-ppc64": { @@ -1308,19 +1313,6 @@ "yjs": "^13.6.8" } }, - "node_modules/@hono/node-server": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.3.tgz", - "integrity": "sha512-TA//nWMqPhbfdfneACk6t5a9eqbS9lABEPyKn0/xZTah3H3U2XaVg85rJFl0/Fyit0I552YDHgXGVSf3GwqbUw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=20" - }, - "peerDependencies": { - "hono": "^4" - } - }, "node_modules/@hookform/resolvers": { "version": "3.10.0", "resolved": "https://registry.npmjs.org/@hookform/resolvers/-/resolvers-3.10.0.tgz", @@ -1599,71 +1591,33 @@ "integrity": "sha512-QZqem4QuAnAyzfz+Gj5/+SLxqwCAw2qmt7732ZXodr6VDWGeYLG6w1i/vYLa55JQM9wRuBKLmXmiZ2P0LtE5rw==", "license": "MIT" }, - "node_modules/@modelcontextprotocol/sdk": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.30.1.tgz", - "integrity": "sha512-H2HxLvC3HDNybePJaLdSrU1hhUK5iQw+WvV1b01myFyI7sdVGe1u/IPTE5D9fGCiJDVtgMV/lmFkQXLmQyIFYA==", + "node_modules/@modelcontextprotocol/core": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/core/-/core-2.2.0.tgz", + "integrity": "sha512-iLhmprRmWI8EcosOA3wVvww22z02NkgqhV4fBH6f/odQBsi7xnJc0HGmi21yWO+/iKw2yzqVE127Zhna5/+JXw==", "dev": true, "license": "MIT", "dependencies": { - "@hono/node-server": "^1.19.9 || ^2.0.5", - "ajv": "^8.17.1", - "ajv-formats": "^3.0.1", - "content-type": "^1.0.5", - "cors": "^2.8.5", - "cross-spawn": "^7.0.5", - "eventsource": "^3.0.2", - "eventsource-parser": "^3.0.0", - "express": "^5.2.1", - "express-rate-limit": "^8.2.1", - "hono": "^4.11.4", - "jose": "^6.1.3", - "json-schema-typed": "^8.0.2", - "pkce-challenge": "^5.0.0", - "raw-body": "^3.0.0", - "zod": "^3.25 || ^4.0", - "zod-to-json-schema": "^3.25.1" + "zod": "^4.2.0" }, "engines": { - "node": ">=18" - }, - "peerDependencies": { - "@cfworker/json-schema": "^4.1.1", - "zod": "^3.25 || ^4.0" - }, - "peerDependenciesMeta": { - "@cfworker/json-schema": { - "optional": true - }, - "zod": { - "optional": false - } + "node": ">=20" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/ajv": { - "version": "8.20.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", - "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "node_modules/@modelcontextprotocol/server": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/server/-/server-2.2.0.tgz", + "integrity": "sha512-qFnltjus6Gk8Lx6J1w2RXmr3YPN22MNT6qD6zjAkaiWIlFIFxWClz0rXHNuJF/KSUyn5YGhkSvXkWDbQHVLFEQ==", "dev": true, "license": "MIT", "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" + "@modelcontextprotocol/core": "2.2.0", + "zod": "^4.2.0" }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" + "engines": { + "node": ">=20" } }, - "node_modules/@modelcontextprotocol/sdk/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true, - "license": "MIT" - }, "node_modules/@napi-rs/canvas": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-1.0.8.tgz", @@ -6003,47 +5957,6 @@ "dev": true, "license": "BSD-3-Clause" }, - "node_modules/accepts": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", - "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-types": "^3.0.0", - "negotiator": "^1.0.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/accepts/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/accepts/node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/acorn": { "version": "8.18.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", @@ -6139,48 +6052,6 @@ "url": "https://github.com/sponsors/epoberezkin" } }, - "node_modules/ajv-formats": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", - "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, - "node_modules/ajv-formats/node_modules/ajv": { - "version": "8.20.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", - "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.3", - "fast-uri": "^3.0.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ajv-formats/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true, - "license": "MIT" - }, "node_modules/ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", @@ -6446,62 +6317,6 @@ "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==", "license": "MIT" }, - "node_modules/body-parser": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", - "integrity": "sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==", - "dev": true, - "license": "MIT", - "dependencies": { - "bytes": "^3.1.2", - "content-type": "^2.0.0", - "debug": "^4.4.3", - "http-errors": "^2.0.1", - "iconv-lite": "^0.7.2", - "on-finished": "^2.4.1", - "qs": "^6.15.2", - "raw-body": "^3.0.2", - "type-is": "^2.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/body-parser/node_modules/content-type": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", - "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/body-parser/node_modules/iconv-lite": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", - "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/bplist-creator": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/bplist-creator/-/bplist-creator-0.1.0.tgz", @@ -6589,16 +6404,6 @@ "node": "*" } }, - "node_modules/bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/call-bind-apply-helpers": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", @@ -6613,23 +6418,6 @@ "node": ">= 0.4" } }, - "node_modules/call-bound": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", - "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bind-apply-helpers": "^1.0.2", - "get-intrinsic": "^1.3.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -6896,30 +6684,6 @@ "dev": true, "license": "MIT" }, - "node_modules/content-disposition": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz", - "integrity": "sha512-5jRCH9Z/+DRP7rkvY83B+yGIGX96OYdJmzngqnw2SBSxqCFPd0w2km3s5iawpGX8krnwSGmF0FW5Nhr0Hfai3g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/content-type": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", - "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", @@ -6940,16 +6704,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/cookie-signature": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", - "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.6.0" - } - }, "node_modules/core-js": { "version": "3.50.0", "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.50.0.tgz", @@ -6970,24 +6724,6 @@ "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", "license": "MIT" }, - "node_modules/cors": { - "version": "2.8.6", - "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", - "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", - "dev": true, - "license": "MIT", - "dependencies": { - "object-assign": "^4", - "vary": "^1" - }, - "engines": { - "node": ">= 0.10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -7375,16 +7111,6 @@ "node": ">=0.4.0" } }, - "node_modules/depd": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", - "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/dequal": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", @@ -7542,15 +7268,15 @@ } }, "node_modules/e2e": { - "version": "0.15.2", - "resolved": "https://registry.npmjs.org/e2e/-/e2e-0.15.2.tgz", - "integrity": "sha512-QBbrovhKlr++kc4J2Pt+2oJ4tEl36MgIDbaZmuMaE+mjTokfIxnSWJHAgFY6ox8rfuOkvCGYXBSFIKXyiqv7Cw==", + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/e2e/-/e2e-0.17.0.tgz", + "integrity": "sha512-Q8pgMUgp0wY7f5Zs2w0CLmFr1s95rdYyIquEFH8sjU7dYOTSVzvq/xkXCO2FVZFHdcgeO4mvFkdMg+7NyNG+LA==", "dev": true, "license": "Apache-2.0", "dependencies": { "@ai-sdk/provider": "4.0.18", "@clack/prompts": "1.8.1", - "@modelcontextprotocol/sdk": "1.30.1", + "@modelcontextprotocol/server": "2.2.0", "@vitest/expect": "5.0.2", "commander": "15.0.0", "picocolors": "1.1.1", @@ -7565,12 +7291,20 @@ "node": ">=22.12.0" }, "peerDependencies": { + "@ai-sdk/anthropic": "^4.0.0", + "@ai-sdk/google": "^4.0.0", "@ai-sdk/openai": "^4.0.0", "@ai-sdk/openai-compatible": "^3.0.0", "@ai-sdk/xai": "^5.0.0", "ai": "^7.0.0" }, "peerDependenciesMeta": { + "@ai-sdk/anthropic": { + "optional": true + }, + "@ai-sdk/google": { + "optional": true + }, "@ai-sdk/openai": { "optional": true }, @@ -7683,13 +7417,6 @@ "fsevents": "~2.3.3" } }, - "node_modules/ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", - "dev": true, - "license": "MIT" - }, "node_modules/electron-to-chromium": { "version": "1.5.420", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.420.tgz", @@ -7715,16 +7442,6 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "license": "MIT" }, - "node_modules/encodeurl": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", - "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/entities": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", @@ -7854,13 +7571,6 @@ "node": ">=6" } }, - "node_modules/escape-html": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", - "dev": true, - "license": "MIT" - }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -8108,35 +7818,12 @@ "node": ">=0.10.0" } }, - "node_modules/etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/eventemitter3": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", "license": "MIT" }, - "node_modules/eventsource": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", - "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", - "dev": true, - "license": "MIT", - "dependencies": { - "eventsource-parser": "^3.0.1" - }, - "engines": { - "node": ">=18.0.0" - } - }, "node_modules/eventsource-parser": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", @@ -8157,133 +7844,32 @@ "node": ">=12.0.0" } }, - "node_modules/express": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", - "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "dev": true, + "license": "MIT" + }, + "node_modules/fast-equals": { + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.4.2.tgz", + "integrity": "sha512-Ywe6jodPTWOTL9/k0bV7gdfP8twKL5Y8I8CZ933fAY5gBekICZSUQTbyH6ut2NZCNyB05mSUwAuEqdEIaOOlDQ==", "license": "MIT", - "dependencies": { - "accepts": "^2.0.0", - "body-parser": "^2.2.1", - "content-disposition": "^1.0.0", - "content-type": "^1.0.5", - "cookie": "^0.7.1", - "cookie-signature": "^1.2.1", - "debug": "^4.4.0", - "depd": "^2.0.0", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "etag": "^1.8.1", - "finalhandler": "^2.1.0", - "fresh": "^2.0.0", - "http-errors": "^2.0.0", - "merge-descriptors": "^2.0.0", - "mime-types": "^3.0.0", - "on-finished": "^2.4.1", - "once": "^1.4.0", - "parseurl": "^1.3.3", - "proxy-addr": "^2.0.7", - "qs": "^6.14.0", - "range-parser": "^1.2.1", - "router": "^2.2.0", - "send": "^1.1.0", - "serve-static": "^2.2.0", - "statuses": "^2.0.1", - "type-is": "^2.0.1", - "vary": "^1.1.2" - }, "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" + "node": ">=6.0.0" } }, - "node_modules/express-rate-limit": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.7.0.tgz", - "integrity": "sha512-hOwV7WOxXfjRpAM1DSJWZDXx3GhplwD8IfwuwvogD8i1Qnkgosw/H45s4ZnFAUHDAhPjlY9hLBvJhKmGMyY26g==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.4.3", - "ip-address": "^10.2.0" - }, - "engines": { - "node": ">= 16" - }, - "funding": { - "url": "https://github.com/sponsors/express-rate-limit" - }, - "peerDependencies": { - "express": ">= 4.11" - } - }, - "node_modules/express/node_modules/cookie": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", - "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/express/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/express/node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/fast-equals": { - "version": "5.4.2", - "resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-5.4.2.tgz", - "integrity": "sha512-Ywe6jodPTWOTL9/k0bV7gdfP8twKL5Y8I8CZ933fAY5gBekICZSUQTbyH6ut2NZCNyB05mSUwAuEqdEIaOOlDQ==", - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/fast-glob": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", - "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", @@ -8350,23 +7936,6 @@ "fast-string-truncated-width": "^3.0.2" } }, - "node_modules/fast-uri": { - "version": "3.1.8", - "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.8.tgz", - "integrity": "sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fastify" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fastify" - } - ], - "license": "BSD-3-Clause" - }, "node_modules/fast-wrap-ansi": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.2.2.tgz", @@ -8426,28 +7995,6 @@ "node": ">=8" } }, - "node_modules/finalhandler": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", - "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.4.0", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "on-finished": "^2.4.1", - "parseurl": "^1.3.3", - "statuses": "^2.0.1" - }, - "engines": { - "node": ">= 18.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -8503,16 +8050,6 @@ "node": ">= 6" } }, - "node_modules/forwarded": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", - "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, "node_modules/fraction.js": { "version": "5.3.4", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.3.4.tgz", @@ -8554,16 +8091,6 @@ } } }, - "node_modules/fresh": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", - "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/fs-extra": { "version": "11.4.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.0.tgz", @@ -8870,16 +8397,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/hono": { - "version": "4.13.12", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.12.tgz", - "integrity": "sha512-6E2QDAc9Ick9Sq77ZrGS/dk2WUYni91aufTw6LJKpV7w8kW5/GxVUc650FOADOmlwg3K+f7Pun6XlV+pYmW6gw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=16.9.0" - } - }, "node_modules/html-encoding-sniffer": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", @@ -8939,27 +8456,6 @@ "node": ">=8.0.0" } }, - "node_modules/http-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", - "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "depd": "~2.0.0", - "inherits": "~2.0.4", - "setprototypeof": "~1.2.0", - "statuses": "~2.0.2", - "toidentifier": "~1.0.1" - }, - "engines": { - "node": ">= 0.8" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/http-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", @@ -9123,26 +8619,6 @@ "integrity": "sha512-DRebOWuqDvxunfkNJAlc3IzWIPD5xVxwUNbHr7xKB8E6aLJxIPfNX3CoMJghcFjpv6RWQsrcJbghtEwSPoJqMA==", "license": "MIT" }, - "node_modules/ip-address": { - "version": "10.7.3", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.3.tgz", - "integrity": "sha512-A1kdq/tSb5QjvKvAMgIoEvDBIgL7qaqVP/jkvSwYYRZ9iEzvPpopxp2wQfu3SuZRHtpHNxMn8Fs0bS+gf5Xmwg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, - "node_modules/ipaddr.js": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, "node_modules/is-alphabetical": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", @@ -9293,13 +8769,6 @@ "dev": true, "license": "MIT" }, - "node_modules/is-promise": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", - "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", - "dev": true, - "license": "MIT" - }, "node_modules/is-wsl": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", @@ -9382,16 +8851,6 @@ "jiti": "bin/jiti.js" } }, - "node_modules/jose": { - "version": "6.2.12", - "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.12.tgz", - "integrity": "sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/panva" - } - }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -9501,13 +8960,6 @@ "dev": true, "license": "MIT" }, - "node_modules/json-schema-typed": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/json-schema-typed/-/json-schema-typed-8.0.2.tgz", - "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==", - "dev": true, - "license": "BSD-2-Clause" - }, "node_modules/json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", @@ -9996,33 +9448,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/media-typer": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.1.tgz", - "integrity": "sha512-yz3xRaG20c6/BOzvYoDaGtPmGscs7YivItZEEqe6GbwNfHuxu9YNmvnEkMzKldAGY4/80pRcQRZSEnhquk9XuQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/merge-descriptors": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", - "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -10642,37 +10067,6 @@ "dev": true, "license": "MIT" }, - "node_modules/negotiator": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.1.0.tgz", - "integrity": "sha512-NMPBRMJgiQHjbd8phG3Vebdx4kZ1H121rbl5IkMqeOsahptB9BKo/d7oJ3zTXqTgagn2bWlNSXkh0QUGM31RYg==", - "dev": true, - "license": "MIT", - "dependencies": { - "content-type": "^2.1.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/negotiator/node_modules/content-type": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", - "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", @@ -10763,19 +10157,6 @@ "node": ">= 6" } }, - "node_modules/object-inspect": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", - "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/obug": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.4.tgz", @@ -10790,29 +10171,6 @@ "node": ">=12.20.0" } }, - "node_modules/on-finished": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", - "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", - "dev": true, - "license": "MIT", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, "node_modules/open": { "version": "8.4.2", "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", @@ -10965,16 +10323,6 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, - "node_modules/parseurl": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", - "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -11025,17 +10373,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/path-to-regexp": { - "version": "8.4.2", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", - "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", - "dev": true, - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/pathe": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", @@ -11184,16 +10521,6 @@ "node": ">= 6" } }, - "node_modules/pkce-challenge": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", - "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=16.20.0" - } - }, "node_modules/playwright": { "version": "1.63.0", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", @@ -11801,24 +11128,6 @@ "prosemirror-transform": "^1.1.0" } }, - "node_modules/proxy-addr": { - "version": "2.0.8", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", - "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "forwarded": "0.2.0", - "ipaddr.js": "1.9.1" - }, - "engines": { - "node": ">= 0.10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", @@ -11849,23 +11158,6 @@ "node": ">=6" } }, - "node_modules/qs": { - "version": "6.16.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.0.tgz", - "integrity": "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "es-define-property": "^1.0.1", - "side-channel": "^1.1.1" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/query-selector-shadow-dom": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/query-selector-shadow-dom/-/query-selector-shadow-dom-1.0.1.tgz", @@ -11909,53 +11201,6 @@ "performance-now": "^2.1.0" } }, - "node_modules/range-parser": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", - "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/raw-body": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", - "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", - "dev": true, - "license": "MIT", - "dependencies": { - "bytes": "~3.1.2", - "http-errors": "~2.0.1", - "iconv-lite": "~0.7.0", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/raw-body/node_modules/iconv-lite": { - "version": "0.7.3", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", - "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3.0.0" - }, - "engines": { - "node": ">=0.10.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/react": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", @@ -12418,16 +11663,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/requires-port": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", @@ -12567,23 +11802,6 @@ "integrity": "sha512-UT5EDe2cu2E/6O4igUr5PSFs23nvvukicWHx6GnOPlHAiiYbzNuCRQCuiUdHJQcqKalLKlrYJnjY0ySGsXNQXQ==", "license": "MIT" }, - "node_modules/router": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", - "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.4.0", - "depd": "^2.0.0", - "is-promise": "^4.0.0", - "parseurl": "^1.3.3", - "path-to-regexp": "^8.0.0" - }, - "engines": { - "node": ">= 18" - } - }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -12672,80 +11890,6 @@ "node": ">=10" } }, - "node_modules/send": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", - "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.4.3", - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "etag": "^1.8.1", - "fresh": "^2.0.0", - "http-errors": "^2.0.1", - "mime-types": "^3.0.2", - "ms": "^2.1.3", - "on-finished": "^2.4.1", - "range-parser": "^1.2.1", - "statuses": "^2.0.2" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/send/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/send/node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/serve-static": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", - "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", - "dev": true, - "license": "MIT", - "dependencies": { - "encodeurl": "^2.0.0", - "escape-html": "^1.0.3", - "parseurl": "^1.3.3", - "send": "^1.2.0" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/set-cookie-parser": { "version": "2.7.2", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", @@ -12758,13 +11902,6 @@ "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", "license": "MIT" }, - "node_modules/setprototypeof": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", - "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", - "dev": true, - "license": "ISC" - }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -12786,82 +11923,6 @@ "node": ">=8" } }, - "node_modules/side-channel": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", - "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.4", - "side-channel-list": "^1.0.1", - "side-channel-map": "^1.0.1", - "side-channel-weakmap": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-list": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", - "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", - "dev": true, - "license": "MIT", - "dependencies": { - "es-errors": "^1.3.0", - "object-inspect": "^1.13.4" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-map": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", - "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/side-channel-weakmap": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", - "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", - "dev": true, - "license": "MIT", - "dependencies": { - "call-bound": "^1.0.2", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.5", - "object-inspect": "^1.13.3", - "side-channel-map": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/siginfo": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", @@ -12999,16 +12060,6 @@ "node": ">=0.1.14" } }, - "node_modules/statuses": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", - "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/std-env": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.2.0.tgz", @@ -13420,16 +12471,6 @@ "node": ">=8.0" } }, - "node_modules/toidentifier": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", - "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.6" - } - }, "node_modules/tough-cookie": { "version": "4.1.4", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.4.tgz", @@ -14002,66 +13043,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/type-is": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", - "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==", - "dev": true, - "license": "MIT", - "dependencies": { - "content-type": "^2.0.0", - "media-typer": "^1.1.0", - "mime-types": "^3.0.0" - }, - "engines": { - "node": ">= 18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/type-is/node_modules/content-type": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.1.0.tgz", - "integrity": "sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/type-is/node_modules/mime-db": { - "version": "1.54.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", - "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/type-is/node_modules/mime-types": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", - "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "mime-db": "^1.54.0" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -14218,16 +13199,6 @@ "node": ">= 10.0.0" } }, - "node_modules/unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/untildify": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", @@ -14370,16 +13341,6 @@ "uuid": "dist/esm/bin/uuid" } }, - "node_modules/vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, "node_modules/vfile": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", @@ -14787,13 +13748,6 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, - "license": "ISC" - }, "node_modules/ws": { "version": "8.21.3", "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", @@ -14991,16 +13945,6 @@ "url": "https://github.com/sponsors/colinhacks" } }, - "node_modules/zod-to-json-schema": { - "version": "3.25.2", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", - "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", - "dev": true, - "license": "ISC", - "peerDependencies": { - "zod": "^3.25.28 || ^4" - } - }, "node_modules/zustand": { "version": "5.0.15", "resolved": "https://registry.npmjs.org/zustand/-/zustand-5.0.15.tgz", diff --git a/package.json b/package.json index 55004e120..6ec536adf 100644 --- a/package.json +++ b/package.json @@ -109,7 +109,7 @@ "zustand": "^5.0.10" }, "devDependencies": { - "@e2e-dev/web": "0.11.1", + "@e2e-dev/web": "0.12.0", "@eslint/js": "^9.32.0", "@openrouter/ai-sdk-provider": "^3.1.0", "@playwright/test": "^1.63.0", @@ -125,7 +125,7 @@ "@vitest/coverage-v8": "^4.1.8", "ai": "^7.0.127", "autoprefixer": "^10.4.21", - "e2e": "0.15.2", + "e2e": "0.17.0", "eslint": "^9.32.0", "eslint-plugin-react-hooks": "^5.2.0", "eslint-plugin-react-refresh": "^0.4.20", diff --git a/src/components/features/attack/AttackResults.tsx b/src/components/features/attack/AttackResults.tsx index 95ea660b1..9f76fe24a 100644 --- a/src/components/features/attack/AttackResults.tsx +++ b/src/components/features/attack/AttackResults.tsx @@ -79,7 +79,7 @@ function WeaknessCard({ weakness }: { weakness: Weakness }) { {t('attackResults.improvedVersion')} -

{weakness.improvedVersion}

+
{weakness.improvedVersion}
); @@ -266,7 +266,7 @@ export function AttackResults({ result, discoveredSources, onDiscuss, onPractice {fallacy.name} {t('attackResults.in')} {fallacy.location} -

{fallacy.explanation}

+
{fallacy.explanation}
))} From 296a8e6fc60d7de7d9598b3c21238aa145a22127 Mon Sep 17 00:00:00 2001 From: Adam Date: Tue, 6 Oct 2026 09:16:26 +0000 Subject: [PATCH 02/12] fix: show form validation, name Settings selects, and record the agentic flows The agentic suite's page-error check found that empty submits on Generate, Attack, Brainstorm and Factoids threw an unhandled ZodError and showed no message: @hookform/resolvers 3 cannot read zod 4 errors. Upgrade it to 5.4.0, the newest release without the @typeschema peer chain npm rejects, and type FactoidsForm's input and output apart. agentic/forms.e2e.ts pins all four. The replay cache could not record Settings because its three selects had no accessible name. Associate their labels, and find the output language select by name in the two specs that used a CSS container. OpenRouter withdrew the free Qwen actor, so every cache miss and exploration failed. Default the actor to openai/gpt-6-luna. Split CI into page-sweep and journey legs per viewport, fix the truncated workflow, and commit the re-recorded replay cache. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/skills/verify-debatelab/SKILL.md | 2 +- .claude/skills/verify-debatelab/bin/verify.sh | 2 +- .../verify-debatelab/recipes/account.spec.ts | 2 +- .github/workflows/agentic-e2e.yml | 118 ++++++++++++- AGENTS.md | 2 +- agentic/ai-tools.e2e.ts | 2 +- agentic/forms.e2e.ts | 22 +++ ...6e45a7530884b190c73d434b9ac32628d2e14.json | 162 ++++++++++++++++++ ...638a2f99b41bd44866a41d64915470d42a880.json | 115 +++++++++++++ ...4f4290153c469cdcd41e0b0b33c7ee216a78a.json | 50 ++++++ ...31fea764059779def23ffec087cecc00a0950.json | 90 ++++++++++ ...f03e9257953f151152980f9f33713c14bf2c3.json | 103 +++++++++++ ...506ecd8b0b7d98a4006cfda221eda5edb068.json} | 40 ++++- ...5fe76f672e02d9bd0633ed5a4d9f36794f5b2.json | 50 ++++++ ...cd42ee762f88ed3d578f31d3211625788e805.json | 115 +++++++++++++ ...7036fd9cc0ac6dbdd9fc8d128176a57f60dfd.json | 76 ++++++++ ...b3838cb6588a49a20fef44fcd8b7bc6ebb795.json | 50 ++++++ ...5f77d79a388ee8d3d2aee0a6d364c01cea935.json | 162 ++++++++++++++++++ ...340448eafbee4174149a9cd7c2643da67f437.json | 50 ++++++ ...91e6b10f370683c1bdd64349ee6ebd73846e3.json | 100 +++++++++++ ...58d2bc05aff8f07aad051c9ef2cd457f9799.json} | 40 ++++- ...33f4b3906e33910768c64dc5c2d643baa1ea1.json | 77 +++++++++ ...69481bd13e73771983299bdea0f35a4ad1af1.json | 50 ++++++ ...5a31518cb6002dad033bbe5549c162f9df6e.json} | 43 ++++- ...172658e44af4d98c438f7b776fc1fe1b45112.json | 156 +++++++++++++++++ ...bb2d80a6075794717cc7c0c2cd0610bb9fe37.json | 159 +++++++++++++++++ ...c45338886c52fc41ea6a7c0f6fd1435dfbfec.json | 102 +++++++++++ ...f500077ddc18df231c58d9c29724467af6ac.json} | 43 ++++- ...281513bd8832a317f7d3d65ac987fec417dc4.json | 106 ++++++++++++ ...bd27735ac713ce264dcac4e6a90447fcc16d1.json | 135 +++++++++++++++ ...94cbf70ccb746bdcfcc35af4c0a7d2cf6795e.json | 76 ++++++++ ...0b21f11103285e5b28beed208c736f63fd98c.json | 77 +++++++++ ...e2ea67f8b44f1a1a6a400067d474f1b21cb5f.json | 92 ++++++++++ ...eaf8543bff300298b021cd7b406ac08305ded.json | 50 ++++++ ...46d1fac813a9d7ac70a1742898dc3abdf4a99.json | 92 ++++++++++ ...ebdc847f876ca5ec6d2ade3d672c29aedcaf9.json | 77 +++++++++ ...a7fba9df68b2746461d556a193ac4ed89d357.json | 162 ++++++++++++++++++ agentic/workspace.e2e.ts | 7 +- docs/testing.md | 6 +- e2e.config.ts | 19 +- e2e/full-stack/account-data.spec.ts | 2 +- package-lock.json | 19 +- package.json | 2 +- .../features/factoids/FactoidsForm.tsx | 9 +- src/pages/SettingsPage.tsx | 12 +- 45 files changed, 2870 insertions(+), 56 deletions(-) create mode 100644 agentic/forms.e2e.ts create mode 100644 agentic/replay/0243b55678ad97a2e58938635726e45a7530884b190c73d434b9ac32628d2e14.json create mode 100644 agentic/replay/0b6684c4cf8f85ea83d9d2c90b3638a2f99b41bd44866a41d64915470d42a880.json create mode 100644 agentic/replay/1059abc82a45e5a709509256ec74f4290153c469cdcd41e0b0b33c7ee216a78a.json create mode 100644 agentic/replay/126b78cf76461974e31c525473531fea764059779def23ffec087cecc00a0950.json create mode 100644 agentic/replay/2296281fbea5e96beb19e409eaaf03e9257953f151152980f9f33713c14bf2c3.json rename agentic/replay/{c7adcf13daafc4d7b7fdb639f07aee1d7a7d8028597c925bd41fdeff99dc3848.json => 23755b48030e2ba88ae7593c0e65506ecd8b0b7d98a4006cfda221eda5edb068.json} (68%) create mode 100644 agentic/replay/32ddc883c4362b19704486dd14d5fe76f672e02d9bd0633ed5a4d9f36794f5b2.json create mode 100644 agentic/replay/495abc7af387dc9131381dcd246cd42ee762f88ed3d578f31d3211625788e805.json create mode 100644 agentic/replay/4b19b5a96c8088da6cfdc7035b87036fd9cc0ac6dbdd9fc8d128176a57f60dfd.json create mode 100644 agentic/replay/4cfb98f5cbf1e6675d625a3ff77b3838cb6588a49a20fef44fcd8b7bc6ebb795.json create mode 100644 agentic/replay/678251266bef215e1d2533056045f77d79a388ee8d3d2aee0a6d364c01cea935.json create mode 100644 agentic/replay/67db0f98d129349c3fc659ab0da340448eafbee4174149a9cd7c2643da67f437.json create mode 100644 agentic/replay/6944dfabce5ae31228a20c2c28d91e6b10f370683c1bdd64349ee6ebd73846e3.json rename agentic/replay/{b4819cfeff66096f89bd15a116b3d35d58a8b3bbea01d479ba86a4b32575157f.json => 729490fb7619dcddf35c0c60cedc58d2bc05aff8f07aad051c9ef2cd457f9799.json} (68%) create mode 100644 agentic/replay/79a9ee2259d8d16afd564cc66fa33f4b3906e33910768c64dc5c2d643baa1ea1.json create mode 100644 agentic/replay/a789bd72d0b74614dceff992bc569481bd13e73771983299bdea0f35a4ad1af1.json rename agentic/replay/{c194c806fb663c3248e8b760c4857321ddf1b33facc8effb258d09fea6fcf918.json => a7ab22475ff220f06530d1b208935a31518cb6002dad033bbe5549c162f9df6e.json} (62%) create mode 100644 agentic/replay/b8ec74fc06b59f927c0c925c8e8172658e44af4d98c438f7b776fc1fe1b45112.json create mode 100644 agentic/replay/c6cf102fb2172d14552e85bf19abb2d80a6075794717cc7c0c2cd0610bb9fe37.json create mode 100644 agentic/replay/cc6903d8aafb702ef5a496a05b9c45338886c52fc41ea6a7c0f6fd1435dfbfec.json rename agentic/replay/{185f3f3b58f5231404f8aece6743cb319e69c2bd78482e64b82cab4207b3f413.json => cec2ee4bacb9750f2fa98e3124caf500077ddc18df231c58d9c29724467af6ac.json} (62%) create mode 100644 agentic/replay/d15dccac4ac18fdabdb7d4c086d281513bd8832a317f7d3d65ac987fec417dc4.json create mode 100644 agentic/replay/d2b866f651427eaefa7414a35ebbd27735ac713ce264dcac4e6a90447fcc16d1.json create mode 100644 agentic/replay/d636e78bd57e1346e9ad878bdd394cbf70ccb746bdcfcc35af4c0a7d2cf6795e.json create mode 100644 agentic/replay/d73284119fd8669a81b6ad8f2680b21f11103285e5b28beed208c736f63fd98c.json create mode 100644 agentic/replay/d837eeb2b0665a23180f65fb205e2ea67f8b44f1a1a6a400067d474f1b21cb5f.json create mode 100644 agentic/replay/da8918880a1bd2016fba3916020eaf8543bff300298b021cd7b406ac08305ded.json create mode 100644 agentic/replay/e0c912b52a6de1dadb5287bb54346d1fac813a9d7ac70a1742898dc3abdf4a99.json create mode 100644 agentic/replay/e753f1e71c4d4ed90310f668e44ebdc847f876ca5ec6d2ade3d672c29aedcaf9.json create mode 100644 agentic/replay/f6d6a30f85648dfe2446aa2888ca7fba9df68b2746461d556a193ac4ed89d357.json diff --git a/.claude/skills/verify-debatelab/SKILL.md b/.claude/skills/verify-debatelab/SKILL.md index 9e15855fb..33608fbb0 100644 --- a/.claude/skills/verify-debatelab/SKILL.md +++ b/.claude/skills/verify-debatelab/SKILL.md @@ -119,7 +119,7 @@ $H e2e cache ls # recorded agent flo - `e2e` runs `doctor` first and refuses a broken stack, like `drive`. It turns e2e telemetry off and prints `EVIDENCE `, which holds `summary.md`, `report.json`, and per-attempt screenshots, traces, and failure videos. - The suite is tool journeys (tag `journey`) and a page sweep (tag `pages`) of every page in English, Spanish and Hungarian. Every test fails on an uncaught error, `console.error` (React warnings included), a Supabase 5xx, horizontal overflow, or a request to a hosted project: `agentic/support/capture-errors.js` records them in the page. A failure that names `errors the page logged` is a real app error to fix, not test noise. -- The testing agent has its own OpenRouter models: free `qwen/qwen3.8-27b:free` acts (test steps and explorations) and `meta/muse-spark-1.3-contributor` judges, `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` override them. Record new or changed flows with `E2E_AGENT_MODEL=openai/gpt-6-luna`: the model is not part of the replay key, so CI replays them with the free actor. The key comes from `OPENROUTER_API_KEY`, `LIVE_OPENROUTER_API_KEY`, or the live keys file, and is needed in fake mode too. A recorded `agent.act` replays with no model call; every judgment and every exploration step is a call. Without a key, agent tests skip. A `temporarily rate-limited upstream` error from the free actor is infrastructure: rerun with `--last-failed`. For a long or important exploration, `E2E_AGENT_MODEL=openai/gpt-6-luna` is faster and more reliable for cents (the Muse Spark judge cannot act). +- The testing agent has its own OpenRouter models: `openai/gpt-6-luna` acts (test steps on a cache miss, and explorations) and `meta/muse-spark-1.3-contributor` judges. `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` override them; the model is not part of the replay key. The key comes from `OPENROUTER_API_KEY`, `LIVE_OPENROUTER_API_KEY`, or the live keys file, and is needed in fake mode too. A recorded `agent.act` replays with no model call; every judgment and every exploration step is a call. Without a key, agent tests skip. An OpenRouter error such as `[Meta] The requested model was not found` is infrastructure: rerun with `--last-failed`. The Muse Spark judge cannot act. - `--session student` starts an exploration signed in as a fresh demo user, minted by `agentic/sessions.e2e.ts`. `--agent` picks a persona: `first-timer`, `coach`, `edge-input`, or `hungarian`. `--target mobile` explores the phone-sized viewport. - Explorations report `issue` (functional) and `warning` (cosmetic) findings, and exit 1 when there is an issue. Treat a finding as a lead: the stack serves Vite dev, so a route that is still compiling can look dead, and fake mode answers every motion with free-transport text. Prove a finding with a repro test tagged `repro` before you report it as a bug. - Several explorations can run at once (a bug bash), because each gets its own evidence directory. They share this one stack, so give each a different area. diff --git a/.claude/skills/verify-debatelab/bin/verify.sh b/.claude/skills/verify-debatelab/bin/verify.sh index 5e1dd31ac..cbc82ce8a 100755 --- a/.claude/skills/verify-debatelab/bin/verify.sh +++ b/.claude/skills/verify-debatelab/bin/verify.sh @@ -295,7 +295,7 @@ cmd_e2e() { [ -n "$key" ] && export OPENROUTER_API_KEY="$key" fi if [ -n "${OPENROUTER_API_KEY:-}" ]; then - say "info testing agent: OpenRouter actor ${E2E_AGENT_MODEL:-qwen/qwen3.8-27b:free}, judge ${E2E_JUDGE_MODEL:-meta/muse-spark-1.3-contributor} (replayed steps make no model call)" + say "info testing agent: OpenRouter actor ${E2E_AGENT_MODEL:-openai/gpt-6-luna}, judge ${E2E_JUDGE_MODEL:-meta/muse-spark-1.3-contributor} (replayed steps make no model call)" else say "info no OpenRouter key: tests with agent steps will skip" fi diff --git a/.claude/skills/verify-debatelab/recipes/account.spec.ts b/.claude/skills/verify-debatelab/recipes/account.spec.ts index f5a588dc2..25074afab 100644 --- a/.claude/skills/verify-debatelab/recipes/account.spec.ts +++ b/.claude/skills/verify-debatelab/recipes/account.spec.ts @@ -7,7 +7,7 @@ import { admin, AI_TIMEOUT, aiResponse, expect, expectProviderCalls, signIn, tes test("acct-language: the output language follows the account to a new device and reaches the model", async ({ signedIn: page, user, browser, evidence }) => { await page.goto("/app/settings"); - await page.locator("div.space-y-2", { has: page.getByText("AI Output Language", { exact: true }) }).getByRole("combobox").click(); + await page.getByRole("combobox", { name: "AI Output Language" }).click(); await page.getByRole("option", { name: "Magyar" }).click(); await expect.poll(async () => { const { data } = await admin.from("user_settings").select("output_language").eq("user_id", await user.userId()).single(); diff --git a/.github/workflows/agentic-e2e.yml b/.github/workflows/agentic-e2e.yml index 1f974fe14..59ce2aacf 100644 --- a/.github/workflows/agentic-e2e.yml +++ b/.github/workflows/agentic-e2e.yml @@ -3,9 +3,9 @@ name: Agentic E2E # The e2e suite in agentic/ against the local stack in fake mode: an AI agent # drives goal-level flows and judges screens. Recorded flows replay from # agentic/replay without model calls; judgments always call the model. -# It runs nightly and on pull requests that touch the app or the suite, one -# job per viewport. A weekly bug bash sends each persona exploring, and on -# demand one exploration runs instead of the suite. +# It runs nightly and on pull requests that touch the app or the suite. A +# weekly bug bash sends each persona exploring, and on demand one exploration +# runs instead of the suite. # Not a required check. See docs/testing.md#agentic-browser-tests. on: pull_request: @@ -27,7 +27,115 @@ on: - cron: "13 5 * * 0" # the weekly bug bash workflow_dispatch: inputs: - # The weekly bug bash explores with every persona; a manual run explores once. + explore: + description: "Exploration goal. Empty runs the test suite instead." + required: false + default: "" + persona: + description: "Agent from e2e.config.ts that explores" + type: choice + default: first-timer + options: [default, first-timer, coach, edge-input, hungarian] + target: + description: "Viewport to explore on" + type: choice + default: desktop + options: [desktop, mobile] + +permissions: + contents: read + +concurrency: + group: agentic-e2e-${{ github.ref }}-${{ github.event.schedule || inputs.persona || 'suite' }} + cancel-in-progress: true + +env: + SUPABASE_INTERNAL_IMAGE_REGISTRY: public.ecr.aws + +jobs: + suite: + if: github.event.schedule != '13 5 * * 0' && (github.event_name != 'workflow_dispatch' || inputs.explore == '') + runs-on: ubuntu-latest + timeout-minutes: 45 + # Four legs keep each well under the timeout: the page sweep makes about + # 60 judgments per viewport, the journeys and the rest about 15. + strategy: + fail-fast: false + matrix: + target: [desktop, mobile] + part: [pages, journeys] + name: suite (${{ matrix.target }}, ${{ matrix.part }}) + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + + - name: Set up Supabase CLI + uses: supabase/setup-cli@46f7f98c7f948ad727d22c1e67fab04c223a0520 # v3.0.0 + with: + version: 2.116.0 + + - name: Install dependencies + run: npm ci + + - name: Install Chromium + run: npx playwright install --with-deps chromium + + # Without the key every agent test skips, which would read as a pass. + # Same accepted risk as live-e2e.yml: pull-request code runs with it. + - name: Require the testing agent's model key + env: + LIVE_OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }} + run: test -n "$LIVE_OPENROUTER_API_KEY" || { echo "LIVE_OPENROUTER_API_KEY is not set" >&2; exit 1; } + + - name: Start the local stack (fake model provider) + run: .claude/skills/verify-debatelab/bin/verify.sh up + + - name: Agentic suite + env: + LIVE_OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }} + E2E_AGENT_MODEL: ${{ vars.E2E_AGENT_MODEL }} + E2E_JUDGE_MODEL: ${{ vars.E2E_JUDGE_MODEL }} + TARGET: ${{ matrix.target }} + PART: ${{ matrix.part }} + # Bug-bash repro tests fail until their bug is fixed; they run on demand. + # The journeys leg runs everything that is not the page sweep, so an + # untagged test still runs. + run: | + if [ "$PART" = pages ]; then filter=(--tag pages --exclude-tag repro); else filter=(--exclude-tag pages,repro); fi + .claude/skills/verify-debatelab/bin/verify.sh e2e run --target "$TARGET" "${filter[@]}" --reporter list,markdown,junit + + - name: Publish the report + if: always() + run: | + for summary in .verify/artifacts/*-e2e-*/summary.md; do + [ -f "$summary" ] && cat "$summary" >>"$GITHUB_STEP_SUMMARY" + done + true + + - name: Stop the local stack + if: always() + run: .claude/skills/verify-debatelab/bin/verify.sh down + + # Local test data only: fake model output, throwaway demo users. + - name: Upload evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: agentic-e2e-${{ matrix.target }}-${{ matrix.part }}-${{ github.run_id }} + path: | + .verify/artifacts/ + .verify/logs/ + if-no-files-found: ignore + retention-days: 7 + + # The weekly bug bash explores with every persona; a manual run explores once. plan-explore: if: github.event.schedule == '13 5 * * 0' || (github.event_name == 'workflow_dispatch' && inputs.explore != '') runs-on: ubuntu-latest @@ -122,7 +230,7 @@ on: if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: agentic-explore-${{ matrix.leg.persona }}-${{ github.run_id }} + name: agentic-explore-${{ matrix.leg.persona }}-${{ matrix.leg.target }}-${{ github.run_id }} path: | .verify/artifacts/ .verify/logs/ diff --git a/AGENTS.md b/AGENTS.md index 9f82aa068..002acc357 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -98,7 +98,7 @@ $H down # always, including after failures - The stack holds the same lock as `./.superset/db-gate.sh`, so only one worktree can run it at a time. - Most tools work inside a Current Debate. Recipes start one with `startDebate` or answer the motion dialog with `chooseForMotion`; see the skill's Drive section. - Neither mode proves Stripe Checkout, real Stripe cancellation, ElevenLabs voice audio, Google sign-in or export, staging, or production. -- Agentic tests (`agentic/*.e2e.ts`) are an AI agent driving goals and judging screens, with exact assertions on the result. They cover a journey per tool and every page in English, Spanish and Hungarian, and fail on page errors, `console.error`, and Supabase 5xx answers. They catch what selectors cannot: stray untranslated text, clipped layouts, off-topic model output in live mode. They also run explorations that hunt for bugs as a persona. The agent calls its own OpenRouter models (free Qwen acts, the Muse Spark contributor tier judges), even on a fake stack. A flow change that breaks an agentic test updates it in the same PR; commit new or re-recorded `agentic/replay/` entries with it. See [docs/testing.md](docs/testing.md#agentic-browser-tests). +- Agentic tests (`agentic/*.e2e.ts`) are an AI agent driving goals and judging screens, with exact assertions on the result. They cover a journey per tool and every page in English, Spanish and Hungarian, and fail on page errors, `console.error`, and Supabase 5xx answers. They catch what selectors cannot: stray untranslated text, clipped layouts, off-topic model output in live mode. They also run explorations that hunt for bugs as a persona. The agent calls its own OpenRouter models (GPT-6 Luna acts, the Muse Spark contributor tier judges), even on a fake stack. A flow change that breaks an agentic test updates it in the same PR; commit new or re-recorded `agentic/replay/` entries with it. See [docs/testing.md](docs/testing.md#agentic-browser-tests). ### Keep the skill working diff --git a/agentic/ai-tools.e2e.ts b/agentic/ai-tools.e2e.ts index cd732236a..c55024592 100644 --- a/agentic/ai-tools.e2e.ts +++ b/agentic/ai-tools.e2e.ts @@ -33,7 +33,7 @@ test("a student brainstorms a motion and it is filed in a new debate", options, test("a student looks up factoids for a statement", options, async ({ app, agent, student }) => { await app.open("/app/factoids"); - await agent.act("Find factoids for the statement {statement}.", { + await agent.act("Run a factoid search for the statement {statement}. The goal is done when the search finishes, even if it finds no factoids.", { params: { statement: "Cities with free buses have fewer cars downtown." }, }); diff --git a/agentic/forms.e2e.ts b/agentic/forms.e2e.ts new file mode 100644 index 000000000..1be0e3931 --- /dev/null +++ b/agentic/forms.e2e.ts @@ -0,0 +1,22 @@ +import { agentic, expect, test } from "./support/fixtures"; + +// Submitting an AI tool's form empty shows the field's message and starts +// nothing. A resolver that cannot read zod 4's errors threw them as an +// unhandled rejection instead, which the fixture's page-error check catches. +const FORMS = [ + { path: "/app/generate", message: "Motion is required" }, + { path: "/app/attack", message: "Case text is required" }, + { path: "/app/brainstorm", message: "Motion is required" }, + { path: "/app/factoids", message: "Claim is required" }, +]; + +for (const { path, message } of FORMS) { + test(`${path} explains an empty submit`, { ...agentic, tags: ["forms"] }, async ({ app, browser, screen }) => { + await app.open(path); + const submit = browser.locator('main form button[type="submit"]').first(); + await expect(submit).toBeEnabled(); + await submit.tap(); + await expect(screen.getByText(message).first()).toBeVisible(); + await expect(screen.getByRole("dialog")).toHaveCount(0); + }); +} diff --git a/agentic/replay/0243b55678ad97a2e58938635726e45a7530884b190c73d434b9ac32628d2e14.json b/agentic/replay/0243b55678ad97a2e58938635726e45a7530884b190c73d434b9ac32628d2e14.json new file mode 100644 index 000000000..3bc225bc6 --- /dev/null +++ b/agentic/replay/0243b55678ad97a2e58938635726e45a7530884b190c73d434b9ac32628d2e14.json @@ -0,0 +1,162 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:39:18.692Z", + "payload": { + "actions": [ + { + "name": "type", + "summary": "type \"This House would lower the voting age t…\" into textbox \"Motion\"", + "target": { + "role": "textbox", + "name": "Motion", + "placeholder": "This house would require social media companies to verify the age of every user." + }, + "value": "This House would lower the voting age to 16" + }, + { + "name": "tap", + "summary": "tap combobox \"Round format\"", + "target": { + "role": "combobox", + "name": "Round format" + } + }, + { + "name": "tap", + "summary": "tap option \"Lincoln-Douglas · 5 speeches\"", + "target": { + "role": "option", + "name": "Lincoln-Douglas · 5 speeches" + } + }, + { + "name": "tap", + "summary": "tap button \"Start round\"", + "target": { + "role": "button", + "name": "Start round" + } + }, + { + "name": "tap", + "summary": "tap button \"Start debate\" in \"Start a debate with this motion?\"", + "target": { + "role": "button", + "name": "Start debate", + "within": "Start a debate with this motion?" + } + }, + { + "name": "type", + "summary": "type \"Sixteen-year-olds pay taxes and should …\" into textbox \"Draft for Affirmative constructive\"", + "target": { + "role": "textbox", + "name": "Draft for Affirmative constructive", + "placeholder": "Build your claim, warrant, and impact..." + }, + "value": "Sixteen-year-olds pay taxes and should have a say in how that money is spent." + }, + { + "name": "tap", + "summary": "tap button \"Submit speech\"", + "target": { + "role": "button", + "name": "Submit speech" + } + }, + { + "name": "type", + "summary": "type \"Sixteen-year-olds pay taxes and should …\" into textbox \"Draft for First affirmative rebuttal\"", + "target": { + "role": "textbox", + "name": "Draft for First affirmative rebuttal", + "placeholder": "Build your claim, warrant, and impact..." + }, + "value": "Sixteen-year-olds pay taxes and should have a say in how that money is spent." + }, + { + "name": "tap", + "summary": "tap button \"Submit speech\"", + "target": { + "role": "button", + "name": "Submit speech" + } + }, + { + "name": "type", + "summary": "type \"Sixteen-year-olds pay taxes and should …\" into textbox \"Draft for Second affirmative rebuttal\"", + "target": { + "role": "textbox", + "name": "Draft for Second affirmative rebuttal", + "placeholder": "Build your claim, warrant, and impact..." + }, + "value": "Sixteen-year-olds pay taxes and should have a say in how that money is spent." + }, + { + "name": "tap", + "summary": "tap button \"Submit speech\"", + "target": { + "role": "button", + "name": "Submit speech" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/practice.e2e.ts::a%20student%20practises%20a%20full%20sparring%20round%20and%20gets%20a%20verdict", + "targetId": "desktop", + "instructionDigest": "a24c487d613a9127f852667430286bc158068e0277fba5886fddf022b363a331", + "paramsDigest": "4e8b3a690950bfa3fe355c4e6cc27a7f47f902d50bbdfd4629ce085b25e8184c", + "callIndex": 0, + "agent": "default" + }, + "summary": "Started a Lincoln-Douglas round on “This House would lower the voting age to 16,” accepted the prompt to start the debate, and submitted the requested speech on all three of your turns. The round ended with the verdict shown: You won the ballot (80–72).", + "startPath": "/app/sparring", + "endPath": "/app/sparring", + "endAnchors": [ + { + "text": "Current debate" + }, + { + "text": "Round review" + }, + { + "role": "heading", + "name": "Here is what happened in the room." + }, + { + "text": "This House would lower the voting age to 16" + }, + { + "role": "button", + "name": "Export DOCX" + }, + { + "role": "button", + "name": "PDF" + }, + { + "role": "button", + "name": "Start another round" + }, + { + "text": "Final result" + } + ], + "goneAnchors": [ + { + "role": "status", + "name": "Opening Sparring" + }, + { + "role": "button", + "name": "Start debate", + "testId": "current-debate-desktop" + } + ], + "endWaitMs": 14199 + } +} diff --git a/agentic/replay/0b6684c4cf8f85ea83d9d2c90b3638a2f99b41bd44866a41d64915470d42a880.json b/agentic/replay/0b6684c4cf8f85ea83d9d2c90b3638a2f99b41bd44866a41d64915470d42a880.json new file mode 100644 index 000000000..1e2760a08 --- /dev/null +++ b/agentic/replay/0b6684c4cf8f85ea83d9d2c90b3638a2f99b41bd44866a41d64915470d42a880.json @@ -0,0 +1,115 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:42:37.456Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap button \"Argument Argument 1 Your first argument…\"", + "target": { + "role": "button", + "name": "Argument Argument 1 Your first argument strong", + "testId": "rf__node-c1" + } + }, + { + "name": "tap", + "summary": "tap button \"Delete\" (1 of 2)", + "target": { + "role": "button", + "name": "Delete", + "position": { + "index": 0, + "of": 2 + } + } + }, + { + "name": "tap", + "summary": "tap button \"Delete node\"", + "target": { + "role": "button", + "name": "Delete node" + } + }, + { + "name": "tap", + "summary": "tap button \"Save\"", + "target": { + "role": "button", + "name": "Save" + } + }, + { + "name": "type", + "summary": "type \"Zoo ban flow\" into textbox \"Node title...\" in \"Save Flowchart\"", + "target": { + "role": "textbox", + "name": "Node title...", + "placeholder": "Node title...", + "within": "Save Flowchart" + }, + "value": "Zoo ban flow" + }, + { + "name": "tap", + "summary": "tap button \"Save Flowchart\"", + "target": { + "role": "button", + "name": "Save Flowchart" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/workspace.e2e.ts::a%20student%20edits%20and%20saves%20a%20flowchart", + "targetId": "mobile", + "instructionDigest": "f9b2512fe4780b8389cd48765f698834cc973af6a8e64ba553502c62ef3fe327", + "paramsDigest": "d8eb2b16cb9f438cdf20460c83ea28bd5d56bbe5cac29e3300059124989825cb", + "callIndex": 0, + "agent": "default" + }, + "summary": "Deleted Argument 1 and saved the flowchart with the title “Zoo ban flow.” The screen confirms “Flowchart saved successfully.”", + "startPath": "/app/flowchart", + "endPath": "/app/flowchart", + "endAnchors": [ + { + "role": "button", + "name": "Close" + }, + { + "text": "Flowchart saved successfully" + }, + { + "role": "listitem", + "name": "Close Flowchart saved successfully" + }, + { + "text": "Argument: 2" + } + ], + "goneAnchors": [ + { + "text": "Argument 1" + }, + { + "text": "Your first argument" + }, + { + "text": "strong" + }, + { + "text": "Argument: 3" + }, + { + "role": "button", + "name": "Argument Argument 1 Your first argument strong", + "testId": "rf__node-c1" + } + ], + "endWaitMs": 12707 + } +} diff --git a/agentic/replay/1059abc82a45e5a709509256ec74f4290153c469cdcd41e0b0b33c7ee216a78a.json b/agentic/replay/1059abc82a45e5a709509256ec74f4290153c469cdcd41e0b0b33c7ee216a78a.json new file mode 100644 index 000000000..de89b0583 --- /dev/null +++ b/agentic/replay/1059abc82a45e5a709509256ec74f4290153c469cdcd41e0b0b33c7ee216a78a.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:40:11.925Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap button \"Start\"", + "target": { + "role": "button", + "name": "Start" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/workspace.e2e.ts::a%20student%20times%20a%20speech%20and%20pauses%20the%20timer", + "targetId": "desktop", + "instructionDigest": "232cce2805bf5871080e66d24477abaf8893967abffe918c09c1d02069d6c710", + "paramsDigest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "callIndex": 0, + "agent": "default" + }, + "summary": "Started the speech timer. The display now shows SPEAKING and the control changed to Pause.", + "startPath": "/app/timer", + "endPath": "/app/timer", + "endAnchors": [ + { + "text": "SPEAKING" + }, + { + "role": "button", + "name": "Pause" + } + ], + "goneAnchors": [ + { + "text": "PAUSED" + }, + { + "role": "button", + "name": "Start" + } + ], + "endWaitMs": 12024 + } +} diff --git a/agentic/replay/126b78cf76461974e31c525473531fea764059779def23ffec087cecc00a0950.json b/agentic/replay/126b78cf76461974e31c525473531fea764059779def23ffec087cecc00a0950.json new file mode 100644 index 000000000..ec8071a47 --- /dev/null +++ b/agentic/replay/126b78cf76461974e31c525473531fea764059779def23ffec087cecc00a0950.json @@ -0,0 +1,90 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:38:04.778Z", + "payload": { + "actions": [ + { + "name": "type", + "summary": "type \"Free public transport cuts congestion b…\" into textbox \"Your Argument *\"", + "target": { + "role": "textbox", + "name": "Your Argument *", + "placeholder": "Paste your debate case or argument here..." + }, + "value": "Free public transport cuts congestion because more people ride the bus." + }, + { + "name": "tap", + "summary": "tap button \"Find Weaknesses\"", + "target": { + "role": "button", + "name": "Find Weaknesses" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/ai-tools.e2e.ts::a%20student%20finds%20the%20weaknesses%20in%20an%20argument", + "targetId": "desktop", + "instructionDigest": "77dcac3ba53531b17b7f0c9763c004cc1fa40536d862036945fe3bfcaf86173a", + "paramsDigest": "5f470ab551069592515b10b474a86ed672c01e704ef36be168c427c4d995f17d", + "callIndex": 0, + "agent": "default" + }, + "summary": "Entered the provided argument and ran Find Weaknesses. The completed analysis is visible, identifying a major evidential weakness and offering an opponent attack, an improved version, and further critique.", + "startPath": "/app/attack", + "endPath": "/app/attack", + "endAnchors": [ + { + "role": "button", + "name": "Close" + }, + { + "text": "Attack analysis complete" + }, + { + "text": "The critique is ready." + }, + { + "role": "textbox", + "name": "Your Argument *", + "placeholder": "Paste your debate case or argument here...", + "value": "Free public transport cuts congestion because more people ride the bus." + }, + { + "role": "heading", + "name": "Overall Assessment" + }, + { + "text": "FAKE-PROVIDER attack summary: the central warrant is underdefended." + }, + { + "text": "Case Strength" + }, + { + "text": "62/100" + } + ], + "goneAnchors": [ + { + "role": "textbox", + "name": "Your Argument *", + "placeholder": "Paste your debate case or argument here..." + }, + { + "text": "⚔️" + }, + { + "role": "heading", + "name": "No Mercy Mode" + }, + { + "text": "Paste your argument to receive ruthless criticism, identify logical fallacies, and get actionable improvements." + } + ], + "endWaitMs": 14918 + } +} diff --git a/agentic/replay/2296281fbea5e96beb19e409eaaf03e9257953f151152980f9f33713c14bf2c3.json b/agentic/replay/2296281fbea5e96beb19e409eaaf03e9257953f151152980f9f33713c14bf2c3.json new file mode 100644 index 000000000..87d78c7bd --- /dev/null +++ b/agentic/replay/2296281fbea5e96beb19e409eaaf03e9257953f151152980f9f33713c14bf2c3.json @@ -0,0 +1,103 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:38:38.089Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap link \"History\" (1 of 2)", + "target": { + "role": "link", + "name": "History", + "position": { + "index": 0, + "of": 2 + } + } + }, + { + "name": "tap", + "summary": "tap button \"Load\" in \"Saved Cases\"", + "target": { + "role": "button", + "name": "Load", + "within": "Saved Cases" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/case-generation.e2e.ts::a%20student%20starts%20a%20debate%20and%20generates%20a%20case%20for%20it", + "targetId": "desktop", + "instructionDigest": "fb09b9b54eb268c0e9520a17583c722af19bc8aff71ce3b60d25fe2a1e11db55", + "paramsDigest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "callIndex": 0, + "agent": "default" + }, + "summary": "Opened History, found the saved PRO case for “This House would replace school exams with continuous assessment,” and selected Load. The app returned to Generate with the saved case displayed, including its thesis, arguments, predicted opponent case, and prepared counters.", + "startPath": "/app", + "endPath": "/app/generate", + "endAnchors": [ + { + "role": "heading", + "name": "Generate" + }, + { + "text": "Working in:" + }, + { + "text": "In favor" + }, + { + "role": "button", + "name": "Change" + }, + { + "role": "button", + "name": "Use once" + }, + { + "role": "heading", + "name": "Generate Debate Case" + }, + { + "text": "Create structured arguments for any motion" + }, + { + "text": "*" + } + ], + "goneAnchors": [ + { + "text": "Last activity less than a minute ago · 0 sources · 1 outputs" + }, + { + "role": "link", + "name": "Open debate" + }, + { + "role": "heading", + "name": "Prepare" + }, + { + "text": "Generate" + }, + { + "text": "Build a complete debate case." + }, + { + "text": "Brainstorm" + }, + { + "text": "Explore angles for a motion." + }, + { + "text": "Factoids" + } + ], + "endWaitMs": 15541 + } +} diff --git a/agentic/replay/c7adcf13daafc4d7b7fdb639f07aee1d7a7d8028597c925bd41fdeff99dc3848.json b/agentic/replay/23755b48030e2ba88ae7593c0e65506ecd8b0b7d98a4006cfda221eda5edb068.json similarity index 68% rename from agentic/replay/c7adcf13daafc4d7b7fdb639f07aee1d7a7d8028597c925bd41fdeff99dc3848.json rename to agentic/replay/23755b48030e2ba88ae7593c0e65506ecd8b0b7d98a4006cfda221eda5edb068.json index d42c1ab99..793a6bb93 100644 --- a/agentic/replay/c7adcf13daafc4d7b7fdb639f07aee1d7a7d8028597c925bd41fdeff99dc3848.json +++ b/agentic/replay/23755b48030e2ba88ae7593c0e65506ecd8b0b7d98a4006cfda221eda5edb068.json @@ -1,6 +1,6 @@ { "schemaVersion": "trace-1", - "createdAt": "2026-10-02T16:08:35.794Z", + "createdAt": "2026-10-06T08:38:38.078Z", "payload": { "actions": [ { @@ -59,9 +59,12 @@ "recordedFor": { "testId": "agentic/case-generation.e2e.ts::a%20student%20starts%20a%20debate%20and%20generates%20a%20case%20for%20it", "targetId": "desktop", - "instructionDigest": "86311d0b0c95064361ba72d803d642b0c0266b78be93afeba829dde446989804" + "instructionDigest": "86311d0b0c95064361ba72d803d642b0c0266b78be93afeba829dde446989804", + "paramsDigest": "d63cb21535f10ff8c68842c61788532a41b77a00b8533372ddf5f3bc2c00b258", + "callIndex": 0, + "agent": "default" }, - "summary": "Started a new debate with motion \"This House would replace school exams with continuous assessment\" (now shown as Current Debate in the header), then opened Generate and generated a case: thesis, framework, 3 structured arguments, predicted opponent case, and prepared counters are displayed (FAKE-P…", + "summary": "Started a debate for “This House would replace school exams with continuous assessment” and opened Generate. The generated debate case is visible, with the confirmation “Your debate case is ready” and a saved Case (In favor) result.", "startPath": "/app", "endPath": "/app/generate", "endAnchors": [ @@ -94,6 +97,35 @@ "name": "Generate Debate Case" } ], - "endWaitMs": 18577 + "goneAnchors": [ + { + "role": "button", + "name": "Start debate", + "testId": "current-debate-desktop" + }, + { + "text": "No current debate" + }, + { + "text": "Start a debate to keep one motion, its sources, and your results together." + }, + { + "role": "heading", + "name": "Prepare" + }, + { + "text": "Generate" + }, + { + "text": "Build a complete debate case." + }, + { + "text": "Brainstorm" + }, + { + "text": "Explore angles for a motion." + } + ], + "endWaitMs": 15301 } } diff --git a/agentic/replay/32ddc883c4362b19704486dd14d5fe76f672e02d9bd0633ed5a4d9f36794f5b2.json b/agentic/replay/32ddc883c4362b19704486dd14d5fe76f672e02d9bd0633ed5a4d9f36794f5b2.json new file mode 100644 index 000000000..890b7856a --- /dev/null +++ b/agentic/replay/32ddc883c4362b19704486dd14d5fe76f672e02d9bd0633ed5a4d9f36794f5b2.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:52:18.270Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap combobox \"Idioma de Salida IA\"", + "target": { + "role": "combobox", + "name": "Idioma de Salida IA" + } + }, + { + "name": "tap", + "summary": "tap option \"Magyar\"", + "target": { + "role": "option", + "name": "Magyar" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/workspace.e2e.ts::a%20student%20switches%20the%20interface%20to%20Spanish%20and%20the%20AI%20output%20to%20Hungarian", + "targetId": "mobile", + "instructionDigest": "0493cb3ea82426b6cb2ffdb5a79c9003647eb4eafbbb124ce74b65954f9a3ef4", + "paramsDigest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "callIndex": 0, + "agent": "default" + }, + "summary": "Changed “Idioma de Salida IA” to Magyar. The settings page now visibly shows Magyar as the selected output language.", + "startPath": "/app/settings", + "endPath": "/app/settings", + "endAnchors": [ + { + "text": "Magyar" + } + ], + "goneAnchors": [ + { + "text": "English" + } + ], + "endWaitMs": 17571 + } +} diff --git a/agentic/replay/495abc7af387dc9131381dcd246cd42ee762f88ed3d578f31d3211625788e805.json b/agentic/replay/495abc7af387dc9131381dcd246cd42ee762f88ed3d578f31d3211625788e805.json new file mode 100644 index 000000000..d24618551 --- /dev/null +++ b/agentic/replay/495abc7af387dc9131381dcd246cd42ee762f88ed3d578f31d3211625788e805.json @@ -0,0 +1,115 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:39:54.809Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap button \"Argument Argument 1 Your first argument…\"", + "target": { + "role": "button", + "name": "Argument Argument 1 Your first argument strong", + "testId": "rf__node-c1" + } + }, + { + "name": "tap", + "summary": "tap button \"Delete\" (1 of 2)", + "target": { + "role": "button", + "name": "Delete", + "position": { + "index": 0, + "of": 2 + } + } + }, + { + "name": "tap", + "summary": "tap button \"Delete node\"", + "target": { + "role": "button", + "name": "Delete node" + } + }, + { + "name": "tap", + "summary": "tap button \"Save\"", + "target": { + "role": "button", + "name": "Save" + } + }, + { + "name": "type", + "summary": "type \"Zoo ban flow\" into textbox \"Node title...\" in \"Save Flowchart\"", + "target": { + "role": "textbox", + "name": "Node title...", + "placeholder": "Node title...", + "within": "Save Flowchart" + }, + "value": "Zoo ban flow" + }, + { + "name": "tap", + "summary": "tap button \"Save Flowchart\"", + "target": { + "role": "button", + "name": "Save Flowchart" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/workspace.e2e.ts::a%20student%20edits%20and%20saves%20a%20flowchart", + "targetId": "desktop", + "instructionDigest": "f9b2512fe4780b8389cd48765f698834cc973af6a8e64ba553502c62ef3fe327", + "paramsDigest": "d8eb2b16cb9f438cdf20460c83ea28bd5d56bbe5cac29e3300059124989825cb", + "callIndex": 0, + "agent": "default" + }, + "summary": "Deleted Argument 1; the flowchart now shows Argument: 2. Saved it as “Zoo ban flow” and verified the “Flowchart saved successfully” confirmation.", + "startPath": "/app/flowchart", + "endPath": "/app/flowchart", + "endAnchors": [ + { + "role": "button", + "name": "Close" + }, + { + "text": "Flowchart saved successfully" + }, + { + "role": "listitem", + "name": "Close Flowchart saved successfully" + }, + { + "text": "Argument: 2" + } + ], + "goneAnchors": [ + { + "text": "Argument 1" + }, + { + "text": "Your first argument" + }, + { + "text": "strong" + }, + { + "text": "Argument: 3" + }, + { + "role": "button", + "name": "Argument Argument 1 Your first argument strong", + "testId": "rf__node-c1" + } + ], + "endWaitMs": 13598 + } +} diff --git a/agentic/replay/4b19b5a96c8088da6cfdc7035b87036fd9cc0ac6dbdd9fc8d128176a57f60dfd.json b/agentic/replay/4b19b5a96c8088da6cfdc7035b87036fd9cc0ac6dbdd9fc8d128176a57f60dfd.json new file mode 100644 index 000000000..06642db28 --- /dev/null +++ b/agentic/replay/4b19b5a96c8088da6cfdc7035b87036fd9cc0ac6dbdd9fc8d128176a57f60dfd.json @@ -0,0 +1,76 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:39:34.786Z", + "payload": { + "actions": [ + { + "name": "type", + "summary": "type \"How do I weigh access against cost in a…\" into textbox \"Ask your debate coach...\"", + "target": { + "role": "textbox", + "name": "Ask your debate coach...", + "placeholder": "Ask your debate coach..." + }, + "value": "How do I weigh access against cost in a rebuttal?" + }, + { + "name": "tap", + "summary": "tap button \"Send message\"", + "target": { + "role": "button", + "name": "Send message" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/ai-tools.e2e.ts::a%20student%20asks%20the%20coach%20a%20question%20and%20gets%20a%20streamed%20answer", + "targetId": "desktop", + "instructionDigest": "335f7825088365022905d2b2e0551e97f33b1036bd587821def3bf4f33836358", + "paramsDigest": "1208085b29a5f6f45f0f6c438ee587681c3055ffeaa92cfc652a127dab3c39d9", + "callIndex": 0, + "agent": "default" + }, + "summary": "Asked the debate coach “How do I weigh access against cost in a rebuttal?” and verified a visible answer: “FAKE-PROVIDER streamed answer: weigh access against cost.”", + "startPath": "/app/ask", + "endPath": "/app/ask", + "endAnchors": [ + { + "role": "heading", + "name": "Ask" + }, + { + "text": "No current debate. Results stay one-off until you start one." + }, + { + "role": "heading", + "name": "Ask Your Coach" + }, + { + "text": "Get answers to debate questions" + }, + { + "text": "How do I weigh access against cost in a rebuttal?" + }, + { + "text": "FAKE-PROVIDER streamed answer: weigh access against cost." + }, + { + "text": "Citations" + }, + { + "text": "Web search" + } + ], + "goneAnchors": [ + { + "role": "status", + "name": "Opening Ask" + } + ], + "endWaitMs": 15156 + } +} diff --git a/agentic/replay/4cfb98f5cbf1e6675d625a3ff77b3838cb6588a49a20fef44fcd8b7bc6ebb795.json b/agentic/replay/4cfb98f5cbf1e6675d625a3ff77b3838cb6588a49a20fef44fcd8b7bc6ebb795.json new file mode 100644 index 000000000..4ebc952cb --- /dev/null +++ b/agentic/replay/4cfb98f5cbf1e6675d625a3ff77b3838cb6588a49a20fef44fcd8b7bc6ebb795.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:42:52.183Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap button \"Start\"", + "target": { + "role": "button", + "name": "Start" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/workspace.e2e.ts::a%20student%20times%20a%20speech%20and%20pauses%20the%20timer", + "targetId": "mobile", + "instructionDigest": "232cce2805bf5871080e66d24477abaf8893967abffe918c09c1d02069d6c710", + "paramsDigest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "callIndex": 0, + "agent": "default" + }, + "summary": "Started the speech timer. The display changed from PAUSED to SPEAKING and the control now reads Pause.", + "startPath": "/app/timer", + "endPath": "/app/timer", + "endAnchors": [ + { + "text": "SPEAKING" + }, + { + "role": "button", + "name": "Pause" + } + ], + "goneAnchors": [ + { + "text": "PAUSED" + }, + { + "role": "button", + "name": "Start" + } + ], + "endWaitMs": 12221 + } +} diff --git a/agentic/replay/678251266bef215e1d2533056045f77d79a388ee8d3d2aee0a6d364c01cea935.json b/agentic/replay/678251266bef215e1d2533056045f77d79a388ee8d3d2aee0a6d364c01cea935.json new file mode 100644 index 000000000..8c6e408f1 --- /dev/null +++ b/agentic/replay/678251266bef215e1d2533056045f77d79a388ee8d3d2aee0a6d364c01cea935.json @@ -0,0 +1,162 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:41:51.795Z", + "payload": { + "actions": [ + { + "name": "type", + "summary": "type \"This House would lower the voting age t…\" into textbox \"Motion\"", + "target": { + "role": "textbox", + "name": "Motion", + "placeholder": "This house would require social media companies to verify the age of every user." + }, + "value": "This House would lower the voting age to 16" + }, + { + "name": "tap", + "summary": "tap combobox \"Round format\"", + "target": { + "role": "combobox", + "name": "Round format" + } + }, + { + "name": "tap", + "summary": "tap option \"Lincoln-Douglas · 5 speeches\"", + "target": { + "role": "option", + "name": "Lincoln-Douglas · 5 speeches" + } + }, + { + "name": "tap", + "summary": "tap button \"Start round\"", + "target": { + "role": "button", + "name": "Start round" + } + }, + { + "name": "tap", + "summary": "tap button \"Start debate\" in \"Start a debate with this motion?\"", + "target": { + "role": "button", + "name": "Start debate", + "within": "Start a debate with this motion?" + } + }, + { + "name": "type", + "summary": "type \"Sixteen-year-olds pay taxes and should …\" into textbox \"Draft for Affirmative constructive\"", + "target": { + "role": "textbox", + "name": "Draft for Affirmative constructive", + "placeholder": "Build your claim, warrant, and impact..." + }, + "value": "Sixteen-year-olds pay taxes and should have a say in how that money is spent." + }, + { + "name": "tap", + "summary": "tap button \"Submit speech\"", + "target": { + "role": "button", + "name": "Submit speech" + } + }, + { + "name": "type", + "summary": "type \"Sixteen-year-olds pay taxes and should …\" into textbox \"Draft for First affirmative rebuttal\"", + "target": { + "role": "textbox", + "name": "Draft for First affirmative rebuttal", + "placeholder": "Build your claim, warrant, and impact..." + }, + "value": "Sixteen-year-olds pay taxes and should have a say in how that money is spent." + }, + { + "name": "tap", + "summary": "tap button \"Submit speech\"", + "target": { + "role": "button", + "name": "Submit speech" + } + }, + { + "name": "type", + "summary": "type \"Sixteen-year-olds pay taxes and should …\" into textbox \"Draft for Second affirmative rebuttal\"", + "target": { + "role": "textbox", + "name": "Draft for Second affirmative rebuttal", + "placeholder": "Build your claim, warrant, and impact..." + }, + "value": "Sixteen-year-olds pay taxes and should have a say in how that money is spent." + }, + { + "name": "tap", + "summary": "tap button \"Submit speech\"", + "target": { + "role": "button", + "name": "Submit speech" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/practice.e2e.ts::a%20student%20practises%20a%20full%20sparring%20round%20and%20gets%20a%20verdict", + "targetId": "mobile", + "instructionDigest": "a24c487d613a9127f852667430286bc158068e0277fba5886fddf022b363a331", + "paramsDigest": "4e8b3a690950bfa3fe355c4e6cc27a7f47f902d50bbdfd4629ce085b25e8184c", + "callIndex": 0, + "agent": "default" + }, + "summary": "Started the motion as the Current Debate, selected Lincoln-Douglas, and submitted the provided speech for all three of your turns. The round ended and the verdict is shown: You won the ballot, 80–72; the judge’s verdict reads “FAKE-PROVIDER verdict: the user had the edge.”.", + "startPath": "/app/sparring", + "endPath": "/app/sparring", + "endAnchors": [ + { + "text": "Current debate" + }, + { + "text": "This House would lower the voting age to 16" + }, + { + "text": "Round review" + }, + { + "role": "heading", + "name": "Here is what happened in the room." + }, + { + "role": "button", + "name": "Export DOCX" + }, + { + "role": "button", + "name": "PDF" + }, + { + "role": "button", + "name": "Start another round" + }, + { + "text": "Final result" + } + ], + "goneAnchors": [ + { + "role": "status", + "name": "Opening Sparring" + }, + { + "role": "button", + "name": "Start debate", + "testId": "current-debate-mobile" + } + ], + "endWaitMs": 13987 + } +} diff --git a/agentic/replay/67db0f98d129349c3fc659ab0da340448eafbee4174149a9cd7c2643da67f437.json b/agentic/replay/67db0f98d129349c3fc659ab0da340448eafbee4174149a9cd7c2643da67f437.json new file mode 100644 index 000000000..456f5be49 --- /dev/null +++ b/agentic/replay/67db0f98d129349c3fc659ab0da340448eafbee4174149a9cd7c2643da67f437.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:52:21.873Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap combobox \"Idioma de Salida IA\"", + "target": { + "role": "combobox", + "name": "Idioma de Salida IA" + } + }, + { + "name": "tap", + "summary": "tap option \"Magyar\"", + "target": { + "role": "option", + "name": "Magyar" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/workspace.e2e.ts::a%20student%20switches%20the%20interface%20to%20Spanish%20and%20the%20AI%20output%20to%20Hungarian", + "targetId": "desktop", + "instructionDigest": "0493cb3ea82426b6cb2ffdb5a79c9003647eb4eafbbb124ce74b65954f9a3ef4", + "paramsDigest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "callIndex": 0, + "agent": "default" + }, + "summary": "Set the AI Output Language to Magyar. The language list shows Magyar as selected.", + "startPath": "/app/settings", + "endPath": "/app/settings", + "endAnchors": [ + { + "text": "Magyar" + } + ], + "goneAnchors": [ + { + "text": "English" + } + ], + "endWaitMs": 14920 + } +} diff --git a/agentic/replay/6944dfabce5ae31228a20c2c28d91e6b10f370683c1bdd64349ee6ebd73846e3.json b/agentic/replay/6944dfabce5ae31228a20c2c28d91e6b10f370683c1bdd64349ee6ebd73846e3.json new file mode 100644 index 000000000..04876ad16 --- /dev/null +++ b/agentic/replay/6944dfabce5ae31228a20c2c28d91e6b10f370683c1bdd64349ee6ebd73846e3.json @@ -0,0 +1,100 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:40:56.435Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap link \"History\" in \"Utilities\"", + "target": { + "role": "link", + "name": "History", + "within": "Utilities" + } + }, + { + "name": "tap", + "summary": "tap button \"Load\" in \"Saved Cases\"", + "target": { + "role": "button", + "name": "Load", + "within": "Saved Cases" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/case-generation.e2e.ts::a%20student%20starts%20a%20debate%20and%20generates%20a%20case%20for%20it", + "targetId": "mobile", + "instructionDigest": "fb09b9b54eb268c0e9520a17583c722af19bc8aff71ce3b60d25fe2a1e11db55", + "paramsDigest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "callIndex": 0, + "agent": "default" + }, + "summary": "Opened History, found the saved case for “This House would replace school exams with continuous assessment,” and selected Load. The app returned to Generate with the case loaded, showing the case structure, arguments, predicted opponent case, and prepared counters.", + "startPath": "/app", + "endPath": "/app/generate", + "endAnchors": [ + { + "role": "heading", + "name": "Generate" + }, + { + "text": "Working in:" + }, + { + "text": "In favor" + }, + { + "role": "button", + "name": "Change" + }, + { + "role": "button", + "name": "Use once" + }, + { + "role": "heading", + "name": "Generate Debate Case" + }, + { + "text": "Create structured arguments for any motion" + }, + { + "text": "*" + } + ], + "goneAnchors": [ + { + "text": "Last activity less than a minute ago · 0 sources · 1 outputs" + }, + { + "role": "link", + "name": "Open debate" + }, + { + "role": "heading", + "name": "Prepare" + }, + { + "text": "Generate" + }, + { + "text": "Build a complete debate case." + }, + { + "text": "Brainstorm" + }, + { + "text": "Explore angles for a motion." + }, + { + "text": "Factoids" + } + ], + "endWaitMs": 14008 + } +} diff --git a/agentic/replay/b4819cfeff66096f89bd15a116b3d35d58a8b3bbea01d479ba86a4b32575157f.json b/agentic/replay/729490fb7619dcddf35c0c60cedc58d2bc05aff8f07aad051c9ef2cd457f9799.json similarity index 68% rename from agentic/replay/b4819cfeff66096f89bd15a116b3d35d58a8b3bbea01d479ba86a4b32575157f.json rename to agentic/replay/729490fb7619dcddf35c0c60cedc58d2bc05aff8f07aad051c9ef2cd457f9799.json index 07f4dbccf..66b311fb6 100644 --- a/agentic/replay/b4819cfeff66096f89bd15a116b3d35d58a8b3bbea01d479ba86a4b32575157f.json +++ b/agentic/replay/729490fb7619dcddf35c0c60cedc58d2bc05aff8f07aad051c9ef2cd457f9799.json @@ -1,6 +1,6 @@ { "schemaVersion": "trace-1", - "createdAt": "2026-10-02T16:08:26.536Z", + "createdAt": "2026-10-06T08:40:56.425Z", "payload": { "actions": [ { @@ -59,9 +59,12 @@ "recordedFor": { "testId": "agentic/case-generation.e2e.ts::a%20student%20starts%20a%20debate%20and%20generates%20a%20case%20for%20it", "targetId": "mobile", - "instructionDigest": "86311d0b0c95064361ba72d803d642b0c0266b78be93afeba829dde446989804" + "instructionDigest": "86311d0b0c95064361ba72d803d642b0c0266b78be93afeba829dde446989804", + "paramsDigest": "d63cb21535f10ff8c68842c61788532a41b77a00b8533372ddf5f3bc2c00b258", + "callIndex": 0, + "agent": "default" }, - "summary": "Started a new debate on the motion \"This House would replace school exams with continuous assessment\" (now shown as Current Debate in header and home), opened Generate (working-in strip shows the motion) and generated the case. Result confirmed: \"Generate complete\" with Case Structure, Thesis/3 con…", + "summary": "Started a new debate on “This House would replace school exams with continuous assessment,” opened Generate, and generated a case. A visible “Generate complete — Your debate case is ready” confirmation appeared, with the saved case shown under the same motion.", "startPath": "/app", "endPath": "/app/generate", "endAnchors": [ @@ -94,6 +97,35 @@ "name": "Generate Debate Case" } ], - "endWaitMs": 16389 + "goneAnchors": [ + { + "role": "button", + "name": "Start debate", + "testId": "current-debate-mobile" + }, + { + "text": "No current debate" + }, + { + "text": "Start a debate to keep one motion, its sources, and your results together." + }, + { + "role": "heading", + "name": "Prepare" + }, + { + "text": "Generate" + }, + { + "text": "Build a complete debate case." + }, + { + "text": "Brainstorm" + }, + { + "text": "Explore angles for a motion." + } + ], + "endWaitMs": 15409 } } diff --git a/agentic/replay/79a9ee2259d8d16afd564cc66fa33f4b3906e33910768c64dc5c2d643baa1ea1.json b/agentic/replay/79a9ee2259d8d16afd564cc66fa33f4b3906e33910768c64dc5c2d643baa1ea1.json new file mode 100644 index 000000000..6c57d4360 --- /dev/null +++ b/agentic/replay/79a9ee2259d8d16afd564cc66fa33f4b3906e33910768c64dc5c2d643baa1ea1.json @@ -0,0 +1,77 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:46:53.105Z", + "payload": { + "actions": [ + { + "name": "type", + "summary": "type \"Cities with free buses have fewer cars …\" into textbox \"Statement *\"", + "target": { + "role": "textbox", + "name": "Statement *", + "placeholder": "e.g., Renewable energy is more cost-effective than fossil fuels" + }, + "value": "Cities with free buses have fewer cars downtown." + }, + { + "name": "tap", + "summary": "tap button \"Find Factoids\"", + "target": { + "role": "button", + "name": "Find Factoids" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/ai-tools.e2e.ts::a%20student%20looks%20up%20factoids%20for%20a%20statement", + "targetId": "desktop", + "instructionDigest": "c5db87feed7ab17397c5b18d8103e63359c9fd3267161d228707120f188c6734", + "paramsDigest": "d8a1d67bcd2eea2a8af37d3fa6dcb19b9fc23cda24c8503ef5ba290f52b5840d", + "callIndex": 0, + "agent": "default" + }, + "summary": "Submitted the factoid search for “Cities with free buses have fewer cars downtown.” Research completed and the results screen shows “Found 0 Factoids” with a no-matching-sources message.", + "startPath": "/app/factoids", + "endPath": "/app/factoids", + "endAnchors": [ + { + "role": "button", + "name": "Close" + }, + { + "text": "Research complete" + }, + { + "text": "Verified evidence is ready." + }, + { + "role": "heading", + "name": "Factoids" + }, + { + "text": "No current debate. Results stay one-off until you start one." + }, + { + "role": "heading", + "name": "Find Evidence" + }, + { + "text": "Get debate-ready factoids with verified citations" + }, + { + "text": "*" + } + ], + "goneAnchors": [ + { + "role": "status", + "name": "Opening Factoids" + } + ], + "endWaitMs": 14699 + } +} diff --git a/agentic/replay/a789bd72d0b74614dceff992bc569481bd13e73771983299bdea0f35a4ad1af1.json b/agentic/replay/a789bd72d0b74614dceff992bc569481bd13e73771983299bdea0f35a4ad1af1.json new file mode 100644 index 000000000..cf0f8619d --- /dev/null +++ b/agentic/replay/a789bd72d0b74614dceff992bc569481bd13e73771983299bdea0f35a4ad1af1.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:40:11.928Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap button \"Pause\"", + "target": { + "role": "button", + "name": "Pause" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/workspace.e2e.ts::a%20student%20times%20a%20speech%20and%20pauses%20the%20timer", + "targetId": "desktop", + "instructionDigest": "34bcb6621139704efb3a246d0ea57364c0729592733f3d58f9d4bd6876a8b4b9", + "paramsDigest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "callIndex": 0, + "agent": "default" + }, + "summary": "Paused the speech timer. The display changed to PAUSED and the control changed from Pause to Start.", + "startPath": "/app/timer", + "endPath": "/app/timer", + "endAnchors": [ + { + "text": "PAUSED" + }, + { + "role": "button", + "name": "Start" + } + ], + "goneAnchors": [ + { + "text": "SPEAKING" + }, + { + "role": "button", + "name": "Pause" + } + ], + "endWaitMs": 12180 + } +} diff --git a/agentic/replay/c194c806fb663c3248e8b760c4857321ddf1b33facc8effb258d09fea6fcf918.json b/agentic/replay/a7ab22475ff220f06530d1b208935a31518cb6002dad033bbe5549c162f9df6e.json similarity index 62% rename from agentic/replay/c194c806fb663c3248e8b760c4857321ddf1b33facc8effb258d09fea6fcf918.json rename to agentic/replay/a7ab22475ff220f06530d1b208935a31518cb6002dad033bbe5549c162f9df6e.json index c50666e29..efb7b374b 100644 --- a/agentic/replay/c194c806fb663c3248e8b760c4857321ddf1b33facc8effb258d09fea6fcf918.json +++ b/agentic/replay/a7ab22475ff220f06530d1b208935a31518cb6002dad033bbe5549c162f9df6e.json @@ -1,6 +1,6 @@ { "schemaVersion": "trace-1", - "createdAt": "2026-10-02T16:10:12.968Z", + "createdAt": "2026-10-06T08:41:36.597Z", "payload": { "actions": [ { @@ -67,9 +67,12 @@ "recordedFor": { "testId": "agentic/first-debate.e2e.ts::a%20new%20student%20starts%20their%20first%20debate%20from%20the%20onboarding%20tour", "targetId": "mobile", - "instructionDigest": "027693f6c76e21fe239c9a427a38bfdbab2bd0577d771749dc8a73dd02fb46f8" + "instructionDigest": "027693f6c76e21fe239c9a427a38bfdbab2bd0577d771749dc8a73dd02fb46f8", + "paramsDigest": "7b7f2a77eeb79b4f239bf2535592910008b2af7aeab8eea04264408c8ad7e948", + "callIndex": 0, + "agent": "default" }, - "summary": "Followed the onboarding tour: started the debate on motion \"This House would make debate compulsory in secondary schools\" at step 1 (dialog \"Start new debate\"), then clicked Next through steps 2 (Library), 3 (Generate), and 4 (\"You're Ready!\" → \"Get Started!\"). Tour dialog closed; header now shows …", + "summary": "Started a debate on “This House would make debate compulsory in secondary schools,” then advanced through the Library and Generate tour steps and selected “Get Started!” on the final step. The tour dialog closed, confirming completion.", "startPath": "/app", "endPath": "/app", "endAnchors": [ @@ -92,6 +95,38 @@ "testId": "current-debate-mobile" } ], - "endWaitMs": 13620 + "goneAnchors": [ + { + "role": "alertdialog", + "name": "Welcome to DebateLab! 🎉Everything in DebateLab belongs to one debate: the motion you are preparing, its sources, and your results. Start your first debate here.Start debate" + }, + { + "role": "button", + "name": "Start debate", + "testId": "current-debate-mobile" + }, + { + "text": "No current debate" + }, + { + "text": "Start a debate to keep one motion, its sources, and your results together." + }, + { + "role": "heading", + "name": "Welcome to DebateLab! 🎉" + }, + { + "text": "Everything in DebateLab belongs to one debate: the motion you are preparing, its sources, and your results. Start your first debate here." + }, + { + "role": "button", + "name": "Skip Tour" + }, + { + "role": "button", + "name": "Next (Step 1 of 4)" + } + ], + "endWaitMs": 12684 } } diff --git a/agentic/replay/b8ec74fc06b59f927c0c925c8e8172658e44af4d98c438f7b776fc1fe1b45112.json b/agentic/replay/b8ec74fc06b59f927c0c925c8e8172658e44af4d98c438f7b776fc1fe1b45112.json new file mode 100644 index 000000000..03f657930 --- /dev/null +++ b/agentic/replay/b8ec74fc06b59f927c0c925c8e8172658e44af4d98c438f7b776fc1fe1b45112.json @@ -0,0 +1,156 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:42:24.391Z", + "payload": { + "actions": [ + { + "name": "type", + "summary": "type \"This House would make public transport …\" into textbox \"Resolution\"", + "target": { + "role": "textbox", + "name": "Resolution", + "testId": "crossfire-resolution", + "placeholder": "Enter the motion or resolution you are debating..." + }, + "value": "This House would make public transport free" + }, + { + "name": "type", + "summary": "type \"Fares keep the poorest riders off the b…\" into textbox \"Your argument or case\"", + "target": { + "role": "textbox", + "name": "Your argument or case", + "testId": "crossfire-argument", + "placeholder": "Paste the argument you want to defend..." + }, + "value": "Fares keep the poorest riders off the bus." + }, + { + "name": "tap", + "summary": "tap button \"Start crossfire\"", + "target": { + "role": "button", + "name": "Start crossfire", + "testId": "start-crossfire" + } + }, + { + "name": "tap", + "summary": "tap button \"Start debate\" in \"Start a debate with this motion?\"", + "target": { + "role": "button", + "name": "Start debate", + "within": "Start a debate with this motion?" + } + }, + { + "name": "type", + "summary": "type \"What evidence links free fares to fewer…\" into textbox \"Your question\"", + "target": { + "role": "textbox", + "name": "Your question", + "testId": "crossfire-input", + "placeholder": "Ask a focused question..." + }, + "value": "What evidence links free fares to fewer cars?" + }, + { + "name": "tap", + "summary": "tap button \"Send question\"", + "target": { + "role": "button", + "name": "Send question", + "testId": "submit-turn" + } + }, + { + "name": "tap", + "summary": "tap button \"End session\"", + "target": { + "role": "button", + "name": "End session", + "testId": "end-crossfire" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/practice.e2e.ts::a%20student%20cross-examines%20the%20AI%20and%20gets%20a%20debrief", + "targetId": "mobile", + "instructionDigest": "14be6438f67651b8800cbf10b9bcd6c6b2b25a701f80951824044a68a6cf23e8", + "paramsDigest": "a16f3b388b51b313b67c8f80b04a5837d9c6a50eb69afa11823721cec0c11dc5", + "callIndex": 0, + "agent": "default" + }, + "summary": "Started a debate with “This House would make public transport free,” completed Question the AI crossfire using the argument, asked “What evidence links free fares to fewer cars?”, received the AI answer, and ended the session. The Crossfire debrief and transcript are visible.", + "startPath": "/app/crossfire", + "endPath": "/app/crossfire", + "endAnchors": [ + { + "text": "Current debate" + }, + { + "text": "This House would make public transport free" + }, + { + "role": "heading", + "name": "Crossfire debrief" + }, + { + "text": "Review what happened in the exchange and plan your next move." + }, + { + "role": "heading", + "name": "Concessions secured" + }, + { + "text": "No grounded finding in this transcript." + }, + { + "role": "heading", + "name": "Questions that controlled the exchange" + }, + { + "role": "heading", + "name": "Weak or repetitive questions" + } + ], + "goneAnchors": [ + { + "role": "button", + "name": "Start debate", + "testId": "current-debate-mobile" + }, + { + "role": "heading", + "name": "Crossfire" + }, + { + "text": "No current debate. Results stay one-off until you start one." + }, + { + "role": "heading", + "name": "Set up your crossfire" + }, + { + "text": "Choose the resolution, your case, and the kind of pressure you want to practise." + }, + { + "text": "Resolution" + }, + { + "role": "textbox", + "name": "Resolution", + "testId": "crossfire-resolution", + "placeholder": "Enter the motion or resolution you are debating..." + }, + { + "text": "A precise resolution gives the exchange something concrete to test." + } + ], + "endWaitMs": 12887 + } +} diff --git a/agentic/replay/c6cf102fb2172d14552e85bf19abb2d80a6075794717cc7c0c2cd0610bb9fe37.json b/agentic/replay/c6cf102fb2172d14552e85bf19abb2d80a6075794717cc7c0c2cd0610bb9fe37.json new file mode 100644 index 000000000..88482039c --- /dev/null +++ b/agentic/replay/c6cf102fb2172d14552e85bf19abb2d80a6075794717cc7c0c2cd0610bb9fe37.json @@ -0,0 +1,159 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:42:12.424Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap button \"Start debate\" (2 of 2)", + "target": { + "role": "button", + "name": "Start debate", + "position": { + "index": 1, + "of": 2 + } + } + }, + { + "name": "type", + "summary": "type \"This House would ban single-use plastics\" into textbox \"Motion\" in \"Start new debate\"", + "target": { + "role": "textbox", + "name": "Motion", + "placeholder": "This House would…", + "within": "Start new debate" + }, + "value": "This House would ban single-use plastics" + }, + { + "name": "tap", + "summary": "tap button \"Start debate\" in \"Start new debate\"", + "target": { + "role": "button", + "name": "Start debate", + "within": "Start new debate" + } + }, + { + "name": "tap", + "summary": "tap link \"Library\" in \"Utilities\"", + "target": { + "role": "link", + "name": "Library", + "within": "Utilities" + } + }, + { + "name": "tap", + "summary": "tap button \"Add Source\"", + "target": { + "role": "button", + "name": "Add Source" + } + }, + { + "name": "type", + "summary": "type \"Plastic bag levy study\" into textbox \"Source title...\"", + "target": { + "role": "textbox", + "name": "Source title...", + "placeholder": "Source title..." + }, + "value": "Plastic bag levy study" + }, + { + "name": "type", + "summary": "type \"After a 5p levy, single-use bag sales a…\" into textbox \"Paste source content...\"", + "target": { + "role": "textbox", + "name": "Paste source content...", + "placeholder": "Paste source content..." + }, + "value": "After a 5p levy, single-use bag sales at major retailers fell by more than 95 percent." + }, + { + "name": "tap", + "summary": "tap button \"Save to Library\"", + "target": { + "role": "button", + "name": "Save to Library" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/workspace.e2e.ts::a%20student%20adds%20a%20source%20to%20the%20debate%27s%20Library%20and%20pauses%20it", + "targetId": "mobile", + "instructionDigest": "36bd6ba640c32c1dd9ab808075c9cda3de5aac1fee2191e69f36d42871d5a3dd", + "paramsDigest": "bf59bb736176d61326f66ebabe06a3c96d8aaba47c010122e432e9fdd7dca5ca", + "callIndex": 0, + "agent": "default" + }, + "summary": "Started the debate “This House would ban single-use plastics” and added “Plastic bag levy study” to the Library with the provided content. The Library shows the saved source and a “Source Added” confirmation.", + "startPath": "/app", + "endPath": "/app/library", + "endAnchors": [ + { + "role": "button", + "name": "Close" + }, + { + "text": "Source Added" + }, + { + "text": "Your source is now available for grounding." + }, + { + "text": "Current debate" + }, + { + "text": "This House would ban single-use plastics" + }, + { + "role": "heading", + "name": "Source Library" + }, + { + "role": "button", + "name": "Change" + }, + { + "text": "All folders" + } + ], + "goneAnchors": [ + { + "role": "button", + "name": "Start debate", + "testId": "current-debate-mobile" + }, + { + "text": "No current debate" + }, + { + "text": "Start a debate to keep one motion, its sources, and your results together." + }, + { + "role": "heading", + "name": "Prepare" + }, + { + "text": "Generate" + }, + { + "text": "Build a complete debate case." + }, + { + "text": "Brainstorm" + }, + { + "text": "Explore angles for a motion." + } + ], + "endWaitMs": 12722 + } +} diff --git a/agentic/replay/cc6903d8aafb702ef5a496a05b9c45338886c52fc41ea6a7c0f6fd1435dfbfec.json b/agentic/replay/cc6903d8aafb702ef5a496a05b9c45338886c52fc41ea6a7c0f6fd1435dfbfec.json new file mode 100644 index 000000000..f1d904802 --- /dev/null +++ b/agentic/replay/cc6903d8aafb702ef5a496a05b9c45338886c52fc41ea6a7c0f6fd1435dfbfec.json @@ -0,0 +1,102 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:52:21.869Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap combobox \"Interface Language\"", + "target": { + "role": "combobox", + "name": "Interface Language" + } + }, + { + "name": "tap", + "summary": "tap option \"Español\"", + "target": { + "role": "option", + "name": "Español" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/workspace.e2e.ts::a%20student%20switches%20the%20interface%20to%20Spanish%20and%20the%20AI%20output%20to%20Hungarian", + "targetId": "desktop", + "instructionDigest": "2527feb760c51cb423dc8dc5c71d964f6370bcc973d7b06eb9f87ff5a1f1ba3f", + "paramsDigest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "callIndex": 0, + "agent": "default" + }, + "summary": "Changed the Interface Language setting to Español. The settings page and navigation are now displayed in Spanish, including “Ajustes” and “Idioma de Interfaz.”.", + "startPath": "/app/settings", + "endPath": "/app/settings", + "endAnchors": [ + { + "role": "image", + "name": "Logotipo de DebateLab" + }, + { + "text": "Inicio" + }, + { + "role": "button", + "name": "Iniciar debate", + "testId": "current-debate-desktop" + }, + { + "text": "Biblioteca" + }, + { + "text": "Historial" + }, + { + "text": "Cambiar tema" + }, + { + "role": "link", + "name": "Ajustes" + }, + { + "role": "heading", + "name": "Ajustes" + } + ], + "goneAnchors": [ + { + "role": "image", + "name": "DebateLab logo" + }, + { + "text": "Home" + }, + { + "role": "button", + "name": "Start debate", + "testId": "current-debate-desktop" + }, + { + "text": "Library" + }, + { + "text": "History" + }, + { + "text": "Toggle theme" + }, + { + "role": "link", + "name": "Settings" + }, + { + "role": "heading", + "name": "Settings" + } + ], + "endWaitMs": 17439 + } +} diff --git a/agentic/replay/185f3f3b58f5231404f8aece6743cb319e69c2bd78482e64b82cab4207b3f413.json b/agentic/replay/cec2ee4bacb9750f2fa98e3124caf500077ddc18df231c58d9c29724467af6ac.json similarity index 62% rename from agentic/replay/185f3f3b58f5231404f8aece6743cb319e69c2bd78482e64b82cab4207b3f413.json rename to agentic/replay/cec2ee4bacb9750f2fa98e3124caf500077ddc18df231c58d9c29724467af6ac.json index a015a2a00..8c206c45f 100644 --- a/agentic/replay/185f3f3b58f5231404f8aece6743cb319e69c2bd78482e64b82cab4207b3f413.json +++ b/agentic/replay/cec2ee4bacb9750f2fa98e3124caf500077ddc18df231c58d9c29724467af6ac.json @@ -1,6 +1,6 @@ { "schemaVersion": "trace-1", - "createdAt": "2026-10-02T16:03:12.575Z", + "createdAt": "2026-10-06T08:38:24.943Z", "payload": { "actions": [ { @@ -67,9 +67,12 @@ "recordedFor": { "testId": "agentic/first-debate.e2e.ts::a%20new%20student%20starts%20their%20first%20debate%20from%20the%20onboarding%20tour", "targetId": "desktop", - "instructionDigest": "027693f6c76e21fe239c9a427a38bfdbab2bd0577d771749dc8a73dd02fb46f8" + "instructionDigest": "027693f6c76e21fe239c9a427a38bfdbab2bd0577d771749dc8a73dd02fb46f8", + "paramsDigest": "7b7f2a77eeb79b4f239bf2535592910008b2af7aeab8eea04264408c8ad7e948", + "callIndex": 0, + "agent": "default" }, - "summary": "Followed the onboarding tour: at step 1 of 4 started a debate with motion \"This House would make debate compulsory in secondary schools\", then advanced through the Library, Generate steps and the final \"You're Ready!\" step via Get Started!. The tour is finished; home shows the Current Debate with t…", + "summary": "Started a debate on “This House would make debate compulsory in secondary schools” at the tour’s first step, advanced through Library and Generate, and selected Get Started! on the final “You’re Ready!” step. The tour dialog closed.", "startPath": "/app", "endPath": "/app", "endAnchors": [ @@ -92,6 +95,38 @@ "testId": "current-debate-desktop" } ], - "endWaitMs": 14375 + "goneAnchors": [ + { + "role": "alertdialog", + "name": "Welcome to DebateLab! 🎉Everything in DebateLab belongs to one debate: the motion you are preparing, its sources, and your results. Start your first debate here.Start debate" + }, + { + "role": "button", + "name": "Start debate", + "testId": "current-debate-desktop" + }, + { + "text": "No current debate" + }, + { + "text": "Start a debate to keep one motion, its sources, and your results together." + }, + { + "role": "heading", + "name": "Welcome to DebateLab! 🎉" + }, + { + "text": "Everything in DebateLab belongs to one debate: the motion you are preparing, its sources, and your results. Start your first debate here." + }, + { + "role": "button", + "name": "Skip Tour" + }, + { + "role": "button", + "name": "Next (Step 1 of 4)" + } + ], + "endWaitMs": 12998 } } diff --git a/agentic/replay/d15dccac4ac18fdabdb7d4c086d281513bd8832a317f7d3d65ac987fec417dc4.json b/agentic/replay/d15dccac4ac18fdabdb7d4c086d281513bd8832a317f7d3d65ac987fec417dc4.json new file mode 100644 index 000000000..13f86910a --- /dev/null +++ b/agentic/replay/d15dccac4ac18fdabdb7d4c086d281513bd8832a317f7d3d65ac987fec417dc4.json @@ -0,0 +1,106 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:52:18.268Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap combobox \"Interface Language\"", + "target": { + "role": "combobox", + "name": "Interface Language" + } + }, + { + "name": "tap", + "summary": "tap option \"Español\"", + "target": { + "role": "option", + "name": "Español" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/workspace.e2e.ts::a%20student%20switches%20the%20interface%20to%20Spanish%20and%20the%20AI%20output%20to%20Hungarian", + "targetId": "mobile", + "instructionDigest": "2527feb760c51cb423dc8dc5c71d964f6370bcc973d7b06eb9f87ff5a1f1ba3f", + "paramsDigest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "callIndex": 0, + "agent": "default" + }, + "summary": "Changed the Interface Language setting to Español. The settings page now displays Spanish labels, including “Idioma de Interfaz” with “Español” selected.", + "startPath": "/app/settings", + "endPath": "/app/settings", + "endAnchors": [ + { + "role": "image", + "name": "Logotipo de DebateLab" + }, + { + "text": "Cambiar tema" + }, + { + "role": "button", + "name": "Iniciar debate", + "testId": "current-debate-mobile" + }, + { + "role": "heading", + "name": "Ajustes" + }, + { + "role": "heading", + "name": "Suscripción" + }, + { + "text": "Pro" + }, + { + "text": "El acceso a la IA está activo. Pueden aplicarse límites de uso." + }, + { + "role": "heading", + "name": "Ajustes globales" + } + ], + "goneAnchors": [ + { + "role": "status", + "name": "Opening Settings" + }, + { + "role": "image", + "name": "DebateLab logo" + }, + { + "text": "Toggle theme" + }, + { + "role": "button", + "name": "Start debate", + "testId": "current-debate-mobile" + }, + { + "role": "link", + "name": "Terms" + }, + { + "role": "link", + "name": "Privacy" + }, + { + "role": "link", + "name": "DebateLab logo DebateLab" + }, + { + "role": "button", + "name": "Language" + } + ], + "endWaitMs": 12650 + } +} diff --git a/agentic/replay/d2b866f651427eaefa7414a35ebbd27735ac713ce264dcac4e6a90447fcc16d1.json b/agentic/replay/d2b866f651427eaefa7414a35ebbd27735ac713ce264dcac4e6a90447fcc16d1.json new file mode 100644 index 000000000..320b402dd --- /dev/null +++ b/agentic/replay/d2b866f651427eaefa7414a35ebbd27735ac713ce264dcac4e6a90447fcc16d1.json @@ -0,0 +1,135 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:40:02.345Z", + "payload": { + "actions": [ + { + "name": "type", + "summary": "type \"This House would make public transport …\" into textbox \"Resolution\"", + "target": { + "role": "textbox", + "name": "Resolution", + "testId": "crossfire-resolution", + "placeholder": "Enter the motion or resolution you are debating..." + }, + "value": "This House would make public transport free" + }, + { + "name": "type", + "summary": "type \"Fares keep the poorest riders off the b…\" into textbox \"Your argument or case\"", + "target": { + "role": "textbox", + "name": "Your argument or case", + "testId": "crossfire-argument", + "placeholder": "Paste the argument you want to defend..." + }, + "value": "Fares keep the poorest riders off the bus." + }, + { + "name": "tap", + "summary": "tap button \"Start crossfire\"", + "target": { + "role": "button", + "name": "Start crossfire", + "testId": "start-crossfire" + } + }, + { + "name": "tap", + "summary": "tap button \"Start debate\" in \"Start a debate with this motion?\"", + "target": { + "role": "button", + "name": "Start debate", + "within": "Start a debate with this motion?" + } + }, + { + "name": "type", + "summary": "type \"What evidence links free fares to fewer…\" into textbox \"Your question\"", + "target": { + "role": "textbox", + "name": "Your question", + "testId": "crossfire-input", + "placeholder": "Ask a focused question..." + }, + "value": "What evidence links free fares to fewer cars?" + }, + { + "name": "tap", + "summary": "tap button \"Send question\"", + "target": { + "role": "button", + "name": "Send question", + "testId": "submit-turn" + } + }, + { + "name": "tap", + "summary": "tap button \"End session\"", + "target": { + "role": "button", + "name": "End session", + "testId": "end-crossfire" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/practice.e2e.ts::a%20student%20cross-examines%20the%20AI%20and%20gets%20a%20debrief", + "targetId": "desktop", + "instructionDigest": "14be6438f67651b8800cbf10b9bcd6c6b2b25a701f80951824044a68a6cf23e8", + "paramsDigest": "a16f3b388b51b313b67c8f80b04a5837d9c6a50eb69afa11823721cec0c11dc5", + "callIndex": 0, + "agent": "default" + }, + "summary": "Started a Current Debate for “This House would make public transport free,” ran Crossfire in Question the AI mode with the provided argument, submitted “What evidence links free fares to fewer cars?”, received the AI answer, and ended the session. The transcript is visible in the Crossfire debrief.", + "startPath": "/app/crossfire", + "endPath": "/app/crossfire", + "endAnchors": [ + { + "text": "Current debate" + }, + { + "role": "heading", + "name": "Crossfire debrief" + }, + { + "text": "Review what happened in the exchange and plan your next move." + }, + { + "role": "heading", + "name": "Concessions secured" + }, + { + "text": "No grounded finding in this transcript." + }, + { + "role": "heading", + "name": "Questions that controlled the exchange" + }, + { + "role": "heading", + "name": "Weak or repetitive questions" + }, + { + "role": "heading", + "name": "Evasive or incomplete answers" + } + ], + "goneAnchors": [ + { + "role": "status", + "name": "Opening Crossfire" + }, + { + "role": "button", + "name": "Start debate", + "testId": "current-debate-desktop" + } + ], + "endWaitMs": 13614 + } +} diff --git a/agentic/replay/d636e78bd57e1346e9ad878bdd394cbf70ccb746bdcfcc35af4c0a7d2cf6795e.json b/agentic/replay/d636e78bd57e1346e9ad878bdd394cbf70ccb746bdcfcc35af4c0a7d2cf6795e.json new file mode 100644 index 000000000..f0c6b8875 --- /dev/null +++ b/agentic/replay/d636e78bd57e1346e9ad878bdd394cbf70ccb746bdcfcc35af4c0a7d2cf6795e.json @@ -0,0 +1,76 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:41:41.696Z", + "payload": { + "actions": [ + { + "name": "type", + "summary": "type \"How do I weigh access against cost in a…\" into textbox \"Ask your debate coach...\"", + "target": { + "role": "textbox", + "name": "Ask your debate coach...", + "placeholder": "Ask your debate coach..." + }, + "value": "How do I weigh access against cost in a rebuttal?" + }, + { + "name": "tap", + "summary": "tap button \"Send message\"", + "target": { + "role": "button", + "name": "Send message" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/ai-tools.e2e.ts::a%20student%20asks%20the%20coach%20a%20question%20and%20gets%20a%20streamed%20answer", + "targetId": "mobile", + "instructionDigest": "335f7825088365022905d2b2e0551e97f33b1036bd587821def3bf4f33836358", + "paramsDigest": "1208085b29a5f6f45f0f6c438ee587681c3055ffeaa92cfc652a127dab3c39d9", + "callIndex": 0, + "agent": "default" + }, + "summary": "Asked the debate coach, “How do I weigh access against cost in a rebuttal?” The conversation displays the question and a response: “FAKE-PROVIDER streamed answer: weigh access against cost.”", + "startPath": "/app/ask", + "endPath": "/app/ask", + "endAnchors": [ + { + "role": "heading", + "name": "Ask" + }, + { + "text": "No current debate. Results stay one-off until you start one." + }, + { + "text": "0 saved" + }, + { + "role": "heading", + "name": "Ask Your Coach" + }, + { + "text": "Get answers to debate questions" + }, + { + "text": "How do I weigh access against cost in a rebuttal?" + }, + { + "text": "FAKE-PROVIDER streamed answer: weigh access against cost." + }, + { + "text": "Citations" + } + ], + "goneAnchors": [ + { + "role": "status", + "name": "Opening Ask" + } + ], + "endWaitMs": 12792 + } +} diff --git a/agentic/replay/d73284119fd8669a81b6ad8f2680b21f11103285e5b28beed208c736f63fd98c.json b/agentic/replay/d73284119fd8669a81b6ad8f2680b21f11103285e5b28beed208c736f63fd98c.json new file mode 100644 index 000000000..942a2d496 --- /dev/null +++ b/agentic/replay/d73284119fd8669a81b6ad8f2680b21f11103285e5b28beed208c736f63fd98c.json @@ -0,0 +1,77 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:44:38.227Z", + "payload": { + "actions": [ + { + "name": "type", + "summary": "type \"Cities with free buses have fewer cars …\" into textbox \"Statement *\"", + "target": { + "role": "textbox", + "name": "Statement *", + "placeholder": "e.g., Renewable energy is more cost-effective than fossil fuels" + }, + "value": "Cities with free buses have fewer cars downtown." + }, + { + "name": "tap", + "summary": "tap button \"Find Factoids\"", + "target": { + "role": "button", + "name": "Find Factoids" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/ai-tools.e2e.ts::a%20student%20looks%20up%20factoids%20for%20a%20statement", + "targetId": "mobile", + "instructionDigest": "c5db87feed7ab17397c5b18d8103e63359c9fd3267161d228707120f188c6734", + "paramsDigest": "d8a1d67bcd2eea2a8af37d3fa6dcb19b9fc23cda24c8503ef5ba290f52b5840d", + "callIndex": 0, + "agent": "default" + }, + "summary": "Searched factoids for “Cities with free buses have fewer cars downtown.” Research completed; the results screen confirms “Found 0 Factoids” and reports no verified matching sources.", + "startPath": "/app/factoids", + "endPath": "/app/factoids", + "endAnchors": [ + { + "role": "button", + "name": "Close" + }, + { + "text": "Research complete" + }, + { + "text": "Verified evidence is ready." + }, + { + "role": "heading", + "name": "Factoids" + }, + { + "text": "No current debate. Results stay one-off until you start one." + }, + { + "role": "heading", + "name": "Find Evidence" + }, + { + "text": "Get debate-ready factoids with verified citations" + }, + { + "text": "*" + } + ], + "goneAnchors": [ + { + "role": "status", + "name": "Opening Factoids" + } + ], + "endWaitMs": 14622 + } +} diff --git a/agentic/replay/d837eeb2b0665a23180f65fb205e2ea67f8b44f1a1a6a400067d474f1b21cb5f.json b/agentic/replay/d837eeb2b0665a23180f65fb205e2ea67f8b44f1a1a6a400067d474f1b21cb5f.json new file mode 100644 index 000000000..f13c2f6ca --- /dev/null +++ b/agentic/replay/d837eeb2b0665a23180f65fb205e2ea67f8b44f1a1a6a400067d474f1b21cb5f.json @@ -0,0 +1,92 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:38:37.416Z", + "payload": { + "actions": [ + { + "name": "type", + "summary": "type \"This House would ban zoos\" into textbox \"Motion / Topic *\"", + "target": { + "role": "textbox", + "name": "Motion / Topic *", + "placeholder": "e.g., Universal Basic Income should be implemented globally" + }, + "value": "This House would ban zoos" + }, + { + "name": "tap", + "summary": "tap button \"Start Brainstorming\"", + "target": { + "role": "button", + "name": "Start Brainstorming" + } + }, + { + "name": "tap", + "summary": "tap button \"Start debate\" in \"Start a debate with this motion?\"", + "target": { + "role": "button", + "name": "Start debate", + "within": "Start a debate with this motion?" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/ai-tools.e2e.ts::a%20student%20brainstorms%20a%20motion%20and%20it%20is%20filed%20in%20a%20new%20debate", + "targetId": "desktop", + "instructionDigest": "fefbe6831f05cbe5226d052540ae0cd061ed3bc58e2263c57786ba7fa9ef4590", + "paramsDigest": "4799d5c28b5c2c4b6bcd1f82a33d5e0da6d430c1b76bb4436e6761692f30059f", + "callIndex": 0, + "agent": "default" + }, + "summary": "Started a Current Debate for “This House would ban zoos” and brainstormed angles. The screen confirms “Brainstorm complete” and shows the saved Top 5 Argument Angles result.", + "startPath": "/app/brainstorm", + "endPath": "/app/brainstorm", + "endAnchors": [ + { + "text": "Current debate" + }, + { + "role": "heading", + "name": "Brainstorm" + }, + { + "text": "Working in:" + }, + { + "text": "This House would ban zoos" + }, + { + "text": "Both sides" + }, + { + "role": "button", + "name": "Change" + }, + { + "role": "button", + "name": "Use once" + }, + { + "role": "heading", + "name": "Brainstorm Ideas" + } + ], + "goneAnchors": [ + { + "role": "status", + "name": "Opening Brainstorm" + }, + { + "role": "button", + "name": "Start debate", + "testId": "current-debate-desktop" + } + ], + "endWaitMs": 17658 + } +} diff --git a/agentic/replay/da8918880a1bd2016fba3916020eaf8543bff300298b021cd7b406ac08305ded.json b/agentic/replay/da8918880a1bd2016fba3916020eaf8543bff300298b021cd7b406ac08305ded.json new file mode 100644 index 000000000..cba2b58ac --- /dev/null +++ b/agentic/replay/da8918880a1bd2016fba3916020eaf8543bff300298b021cd7b406ac08305ded.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:42:52.186Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap button \"Pause\"", + "target": { + "role": "button", + "name": "Pause" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/workspace.e2e.ts::a%20student%20times%20a%20speech%20and%20pauses%20the%20timer", + "targetId": "mobile", + "instructionDigest": "34bcb6621139704efb3a246d0ea57364c0729592733f3d58f9d4bd6876a8b4b9", + "paramsDigest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "callIndex": 0, + "agent": "default" + }, + "summary": "Paused the speech timer. The display now reads PAUSED and the control changed from Pause to Start.", + "startPath": "/app/timer", + "endPath": "/app/timer", + "endAnchors": [ + { + "text": "PAUSED" + }, + { + "role": "button", + "name": "Start" + } + ], + "goneAnchors": [ + { + "text": "SPEAKING" + }, + { + "role": "button", + "name": "Pause" + } + ], + "endWaitMs": 12420 + } +} diff --git a/agentic/replay/e0c912b52a6de1dadb5287bb54346d1fac813a9d7ac70a1742898dc3abdf4a99.json b/agentic/replay/e0c912b52a6de1dadb5287bb54346d1fac813a9d7ac70a1742898dc3abdf4a99.json new file mode 100644 index 000000000..569dada6e --- /dev/null +++ b/agentic/replay/e0c912b52a6de1dadb5287bb54346d1fac813a9d7ac70a1742898dc3abdf4a99.json @@ -0,0 +1,92 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:40:42.795Z", + "payload": { + "actions": [ + { + "name": "type", + "summary": "type \"This House would ban zoos\" into textbox \"Motion / Topic *\"", + "target": { + "role": "textbox", + "name": "Motion / Topic *", + "placeholder": "e.g., Universal Basic Income should be implemented globally" + }, + "value": "This House would ban zoos" + }, + { + "name": "tap", + "summary": "tap button \"Start Brainstorming\"", + "target": { + "role": "button", + "name": "Start Brainstorming" + } + }, + { + "name": "tap", + "summary": "tap button \"Start debate\" in \"Start a debate with this motion?\"", + "target": { + "role": "button", + "name": "Start debate", + "within": "Start a debate with this motion?" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/ai-tools.e2e.ts::a%20student%20brainstorms%20a%20motion%20and%20it%20is%20filed%20in%20a%20new%20debate", + "targetId": "mobile", + "instructionDigest": "fefbe6831f05cbe5226d052540ae0cd061ed3bc58e2263c57786ba7fa9ef4590", + "paramsDigest": "4799d5c28b5c2c4b6bcd1f82a33d5e0da6d430c1b76bb4436e6761692f30059f", + "callIndex": 0, + "agent": "default" + }, + "summary": "Started a debate for “This House would ban zoos” and brainstormed angles. The page shows the current debate set to that motion and a saved Brainstorm result with five argument angles.", + "startPath": "/app/brainstorm", + "endPath": "/app/brainstorm", + "endAnchors": [ + { + "text": "Current debate" + }, + { + "text": "This House would ban zoos" + }, + { + "role": "heading", + "name": "Brainstorm" + }, + { + "text": "Working in:" + }, + { + "text": "Both sides" + }, + { + "role": "button", + "name": "Change" + }, + { + "role": "button", + "name": "Use once" + }, + { + "role": "heading", + "name": "Brainstorm Ideas" + } + ], + "goneAnchors": [ + { + "role": "status", + "name": "Opening Brainstorm" + }, + { + "role": "button", + "name": "Start debate", + "testId": "current-debate-mobile" + } + ], + "endWaitMs": 24713 + } +} diff --git a/agentic/replay/e753f1e71c4d4ed90310f668e44ebdc847f876ca5ec6d2ade3d672c29aedcaf9.json b/agentic/replay/e753f1e71c4d4ed90310f668e44ebdc847f876ca5ec6d2ade3d672c29aedcaf9.json new file mode 100644 index 000000000..6abd19266 --- /dev/null +++ b/agentic/replay/e753f1e71c4d4ed90310f668e44ebdc847f876ca5ec6d2ade3d672c29aedcaf9.json @@ -0,0 +1,77 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:40:03.869Z", + "payload": { + "actions": [ + { + "name": "type", + "summary": "type \"Free public transport cuts congestion b…\" into textbox \"Your Argument *\"", + "target": { + "role": "textbox", + "name": "Your Argument *", + "placeholder": "Paste your debate case or argument here..." + }, + "value": "Free public transport cuts congestion because more people ride the bus." + }, + { + "name": "tap", + "summary": "tap button \"Find Weaknesses\"", + "target": { + "role": "button", + "name": "Find Weaknesses" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/ai-tools.e2e.ts::a%20student%20finds%20the%20weaknesses%20in%20an%20argument", + "targetId": "mobile", + "instructionDigest": "77dcac3ba53531b17b7f0c9763c004cc1fa40536d862036945fe3bfcaf86173a", + "paramsDigest": "5f470ab551069592515b10b474a86ed672c01e704ef36be168c427c4d995f17d", + "callIndex": 0, + "agent": "default" + }, + "summary": "Entered the argument and selected Find Weaknesses. The attack analysis completed and shows an identified major evidential weakness: the congestion claim lacks comparative evidence, with criticism and improvement suggestions displayed.", + "startPath": "/app/attack", + "endPath": "/app/attack", + "endAnchors": [ + { + "role": "button", + "name": "Close" + }, + { + "text": "Attack analysis complete" + }, + { + "text": "The critique is ready." + }, + { + "role": "heading", + "name": "Attack" + }, + { + "text": "No current debate. Results stay one-off until you start one." + }, + { + "role": "heading", + "name": "Attack My Case" + }, + { + "text": "Find weaknesses and get improvement suggestions" + }, + { + "text": "*" + } + ], + "goneAnchors": [ + { + "role": "status", + "name": "Opening Attack" + } + ], + "endWaitMs": 13006 + } +} diff --git a/agentic/replay/f6d6a30f85648dfe2446aa2888ca7fba9df68b2746461d556a193ac4ed89d357.json b/agentic/replay/f6d6a30f85648dfe2446aa2888ca7fba9df68b2746461d556a193ac4ed89d357.json new file mode 100644 index 000000000..11923333c --- /dev/null +++ b/agentic/replay/f6d6a30f85648dfe2446aa2888ca7fba9df68b2746461d556a193ac4ed89d357.json @@ -0,0 +1,162 @@ +{ + "schemaVersion": "trace-1", + "createdAt": "2026-10-06T08:51:36.896Z", + "payload": { + "actions": [ + { + "name": "tap", + "summary": "tap button \"Start debate\" (2 of 2)", + "target": { + "role": "button", + "name": "Start debate", + "position": { + "index": 1, + "of": 2 + } + } + }, + { + "name": "type", + "summary": "type \"This House would ban single-use plastics\" into textbox \"Motion\" in \"Start new debate\"", + "target": { + "role": "textbox", + "name": "Motion", + "placeholder": "This House would…", + "within": "Start new debate" + }, + "value": "This House would ban single-use plastics" + }, + { + "name": "tap", + "summary": "tap button \"Start debate\" in \"Start new debate\"", + "target": { + "role": "button", + "name": "Start debate", + "within": "Start new debate" + } + }, + { + "name": "tap", + "summary": "tap link \"Library\" (1 of 2)", + "target": { + "role": "link", + "name": "Library", + "position": { + "index": 0, + "of": 2 + } + } + }, + { + "name": "tap", + "summary": "tap button \"Add Source\"", + "target": { + "role": "button", + "name": "Add Source" + } + }, + { + "name": "type", + "summary": "type \"Plastic bag levy study\" into textbox \"Source title...\"", + "target": { + "role": "textbox", + "name": "Source title...", + "placeholder": "Source title..." + }, + "value": "Plastic bag levy study" + }, + { + "name": "type", + "summary": "type \"After a 5p levy, single-use bag sales a…\" into textbox \"Paste source content...\"", + "target": { + "role": "textbox", + "name": "Paste source content...", + "placeholder": "Paste source content..." + }, + "value": "After a 5p levy, single-use bag sales at major retailers fell by more than 95 percent." + }, + { + "name": "tap", + "summary": "tap button \"Save to Library\"", + "target": { + "role": "button", + "name": "Save to Library" + } + } + ], + "executor": { + "name": "e2e-default-agent", + "version": "2" + }, + "recordedFor": { + "testId": "agentic/workspace.e2e.ts::a%20student%20adds%20a%20source%20to%20the%20debate%27s%20Library%20and%20pauses%20it", + "targetId": "desktop", + "instructionDigest": "36bd6ba640c32c1dd9ab808075c9cda3de5aac1fee2191e69f36d42871d5a3dd", + "paramsDigest": "bf59bb736176d61326f66ebabe06a3c96d8aaba47c010122e432e9fdd7dca5ca", + "callIndex": 0, + "agent": "default" + }, + "summary": "Started the debate “This House would ban single-use plastics” and added “Plastic bag levy study” to the Library with the requested content. The Library shows the source and a “Source Added” confirmation.", + "startPath": "/app", + "endPath": "/app/library", + "endAnchors": [ + { + "role": "button", + "name": "Close" + }, + { + "text": "Source Added" + }, + { + "text": "Your source is now available for grounding." + }, + { + "text": "Current debate" + }, + { + "role": "heading", + "name": "Source Library" + }, + { + "text": "This House would ban single-use plastics" + }, + { + "role": "button", + "name": "Change" + }, + { + "text": "All folders" + } + ], + "goneAnchors": [ + { + "role": "button", + "name": "Start debate", + "testId": "current-debate-desktop" + }, + { + "text": "No current debate" + }, + { + "text": "Start a debate to keep one motion, its sources, and your results together." + }, + { + "role": "heading", + "name": "Prepare" + }, + { + "text": "Generate" + }, + { + "text": "Build a complete debate case." + }, + { + "text": "Brainstorm" + }, + { + "text": "Explore angles for a motion." + } + ], + "endWaitMs": 14336 + } +} diff --git a/agentic/workspace.e2e.ts b/agentic/workspace.e2e.ts index c7ff77c59..20ac0f539 100644 --- a/agentic/workspace.e2e.ts +++ b/agentic/workspace.e2e.ts @@ -38,10 +38,13 @@ test("a student times a speech and pauses the timer", options, async ({ app, age const display = browser.locator('[role="timer"][aria-label="Timer display"]'); await expect(display).toBeVisible(); const initial = (await display.textContent()) ?? ""; - await agent.act("Start the speech timer, then pause it."); + // Two goals: a replay presses buttons faster than the clock ticks. + await agent.act("Start the speech timer."); + await expect(screen.getByText("SPEAKING").first()).toBeVisible(); + await expect(display).not.toHaveText(initial); + await agent.act("Pause the speech timer."); await expect(screen.getByText("PAUSED").first()).toBeVisible(); - await expect(display).not.toHaveText(initial); }); test("a student switches the interface to Spanish and the AI output to Hungarian", options, async ({ app, agent, browser, student }) => { diff --git a/docs/testing.md b/docs/testing.md index 837a079d3..a988edadb 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -255,15 +255,15 @@ How it runs: - Always through `verify.sh e2e `. The wrapper runs `doctor`, loads the stack environment, turns e2e telemetry off, and takes the testing agent's key from `OPENROUTER_API_KEY`, `LIVE_OPENROUTER_API_KEY`, or `~/.config/debatelab/live-verify.env`. It writes the report (`summary.md`, `report.json`), screenshots, traces, and failure videos to `.verify/artifacts/-e2e--/`. The config refuses a non-loopback URL and never starts a dev server, because a plain `npm run dev` reaches production. - Each signed-in test signs in a fresh demo-code user through the Login page and deletes it afterwards. Public-page tests use the `visitorTest` fixture with the same checks. - There are two targets, `desktop` (1280 by 800) and `mobile` (393 by 851). e2e does not emulate touch. The app lays out by viewport width and never reads the user agent, so the viewport is what matters; the Playwright Pixel 5 project still owns touch behavior. -- The testing agent has two models on OpenRouter, separate from the app's provider mode, and a fake-mode stack still needs both. The actor (`agent.act` and explorations) is `qwen/qwen3.8-27b:free`. It is free, and recorded flows replay without it, but it is slow and its upstream sometimes answers `temporarily rate-limited upstream`. Treat that as infrastructure and rerun with `--last-failed`. The judge (`agent.assert`, `waitFor`, `extract`) is `meta/muse-spark-1.3-contributor` ($0.10 and $0.20 per million input and output tokens), the same tier as the app's live pin. It answers every judgment on every run. COST_LINE The judge cannot act, because Meta's endpoint accepts only `tool_choice: "auto"`. `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` override the two. Both default models may keep prompts, so use made-up debate content only. The agent's context explains that fake AI answers are canned free-transport text. -- Replay. When a later check passes, an `agent.act` flow is recorded in [`agentic/replay/`](../agentic/replay/). Later runs replay it with no model call and verify that its effect happened, until the app changes and the agent takes over and records again. The model is not part of the cache key, so record new flows with a fast actor, `E2E_AGENT_MODEL=openai/gpt-6-luna $H e2e run ...` (cents per run), and CI replays them with the free one. Commit new and changed entries with the test that made them. Local runs read and write the cache; CI only reads it. Changing an instruction, its params, the test or target name, the agent's `context`, or e2e's minor version re-records. +- The testing agent has two models on OpenRouter, separate from the app's provider mode, and a fake-mode stack still needs both. The actor (`agent.act` and explorations) is `openai/gpt-6-luna`. Recorded flows replay without it, so it runs only on a cache miss and in explorations, at a fraction of a cent per step. OpenRouter withdrew the free Qwen tier the suite used before, which now fails with `This model is unavailable for free`. The judge (`agent.assert`, `waitFor`, `extract`) is `meta/muse-spark-1.3-contributor` ($0.10 and $0.20 per million input and output tokens), the same tier as the app's live pin. It answers every judgment on every run. A full run of both viewports makes about 250 judgments and costs about ten cents. It takes about 45 minutes on one machine with three workers, which is why CI splits it into four jobs. The judge cannot act, because Meta's endpoint accepts only `tool_choice: "auto"`. `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` override the two. The judge's tier may keep prompts, so use made-up debate content only. A transient OpenRouter error such as `[Meta] The requested model was not found` is infrastructure: rerun with `--last-failed`. The agent's context explains that fake AI answers are canned free-transport text. +- Replay. When a later check passes, an `agent.act` flow is recorded in [`agentic/replay/`](../agentic/replay/). Later runs replay it with no model call and verify that its effect happened, until the app changes and the agent takes over and records again. The model is not part of the cache key, so a flow recorded with one actor replays under another. A step that ends on a passing toast or other transient state can hand off to the agent on replay (`end-mismatch` in the report); that costs a model call, not a failure. Commit new and changed entries with the test that made them. Local runs read and write the cache; CI only reads it. Changing an instruction, its params, the test or target name, the agent's `context`, or e2e's minor version re-records. - Without a model key or the stack environment, agent tests skip, so a bare `npx e2e run` proves nothing. `failOnSkippedFailure` fails a run in which a test skipped after a failure. To write a test, import `test`, `expect`, and the fixtures from [`agentic/support/fixtures.ts`](../agentic/support/fixtures.ts) and pass the `agentic` options with a `journey` or `pages` tag. Follow every `agent.act` with a locator assertion, because only verified flows are recorded, and pin side effects with a database read through `admin`. Prefer `agent.assert` for what a selector cannot express, and an exact `expect` for everything else. A new user-facing tool gets a journey, and a new route joins `APP_PAGES` or `PUBLIC_PAGES` in `pages.e2e.ts`. When the judge flags vocabulary the app uses on purpose, add it to `context` in `e2e.config.ts` rather than weakening the assertion. The [`e2e` skill](../.claude/skills/e2e/SKILL.md), `npx e2e guide `, and `node_modules/e2e/docs` cover the API for the installed version. A bug bash runs several explorations in parallel, one charter per area and persona, then proves each finding with a repro test. Ask an agent to "bug bash "; the `e2e` skill's `bug-bash` topic has the procedure. A finding counts only when its repro test fails for the reported reason. Tag repro tests `{ tags: ["repro"] }` so the scheduled suite skips them until the fix lands, then remove the tag so they become regression tests. The client runs on a Vite dev server, so a route that is still compiling can look like a dead link to an explorer. Reproduce before you believe it. -[`.github/workflows/agentic-e2e.yml`](../.github/workflows/agentic-e2e.yml) runs the suite nightly at 03:47 UTC and on pull requests that change the app (`src/`, `public/`, Edge Functions, migrations, dependencies) or the suite, one job per viewport. Every Sunday at 05:13 UTC it runs a bug bash instead: one exploration per persona, each with its own charter. On demand with a goal, it runs one exploration with the chosen persona and viewport. It uses the `LIVE_OPENROUTER_API_KEY` secret (the `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` repository variables override the models), fails when the key is missing, and writes the report to the job summary. Explorations never fail the workflow: their findings are leads to reproduce. The suite is not a required check: model judgments can flake, and a red nightly is a finding to triage. +[`.github/workflows/agentic-e2e.yml`](../.github/workflows/agentic-e2e.yml) runs the suite nightly at 03:47 UTC and on pull requests that change the app (`src/`, `public/`, Edge Functions, migrations, dependencies) or the suite, in four jobs: the page sweep and everything else, on each viewport. Every Sunday at 05:13 UTC it runs a bug bash instead: one exploration per persona, each with its own charter. On demand with a goal, it runs one exploration with the chosen persona and viewport. It uses the `LIVE_OPENROUTER_API_KEY` secret (the `E2E_AGENT_MODEL` and `E2E_JUDGE_MODEL` repository variables override the models), fails when the key is missing, and writes the report to the job summary. Explorations never fail the workflow: their findings are leads to reproduce. The suite is not a required check: model judgments can flake, and a red nightly is a finding to triage. e2e is pre-1.0 and its minor releases have broken APIs, so it is pinned exactly. Read its changelog before upgrading, refresh the vendored skill as its header says, and expect the replay cache to re-record after a minor bump. diff --git a/e2e.config.ts b/e2e.config.ts index 8a821391c..6d8ac4cc9 100644 --- a/e2e.config.ts +++ b/e2e.config.ts @@ -17,12 +17,14 @@ const live = process.env.VERIFY_MODE === "live"; // The testing agent's models, separate from the app's own. Both need tool // calling and image input. The actor drives agent.act and explorations; its -// recorded flows replay with no model call, so a free, slow model is enough. +// recorded flows replay with no model call, so it runs only on a cache miss +// and in explorations, for a fraction of a cent per step. GPT-6 Luna is fast +// and reliable at it (OpenRouter withdrew the free Qwen tier used before). // The judge answers every agent.assert, waitFor and extract on every run, so it -// must be reliable: the Muse Spark contributor tier is the cheapest that is, -// and like the app's live pin it may keep prompts (made-up content only). +// must be cheap and reliable: the Muse Spark contributor tier, which like the +// app's live pin may keep prompts (made-up content only). // `||`, not `??`: an unset Actions variable arrives as an empty string. -const model = openrouter(process.env.E2E_AGENT_MODEL || "qwen/qwen3.8-27b:free"); +const model = openrouter(process.env.E2E_AGENT_MODEL || "openai/gpt-6-luna"); const judge = openrouter(process.env.E2E_JUDGE_MODEL || "meta/muse-spark-1.3-contributor"); // `context` reaches every model call, judges included: what the app calls things. @@ -31,7 +33,7 @@ const context = [ "Each student works in one Current Debate at a time. The header control reads \"Start debate\" until one exists, then names the motion. Tool pages show a strip \"Working in: \".", "Tools: Generate (a full debate case: thesis, contentions, predicted opponent case, prepared counters), Attack, Brainstorm, Factoids, Ask, Sparring (a practice round against an AI opponent, judged at the end), Crossfire (cross-examination practice), Flowchart, Timer, Library (sources for the Current Debate), and History (saved results).", "A tool given a motion with no Current Debate asks \"Start a debate with this motion?\". A motion that differs from the Current Debate asks \"This is a different motion\".", - "The interface language can be English, Spanish or Hungarian. Source titles, citations, user content and model names may stay in their own language. Hungarian abbreviates AI as MI (mesterséges intelligencia), so \"MI-modell\" is Hungarian, and an English loanword with a Hungarian ending, such as \"coachingot\", is Hungarian. The Spanish interface deliberately keeps the loanwords coach, coaching and sparring (\"Pregunta a tu coach\", \"Para Coaches\", \"Rondas de sparring\"); they are Spanish there.", + "The interface language can be English, Spanish or Hungarian. Source titles, citations, user content and model names may stay in their own language. Hungarian abbreviates AI as MI (mesterséges intelligencia), so \"MI-modell\" is Hungarian, and English loanwords such as coaching are Hungarian there too, inflected (\"coachingot\") or as a modifier (\"coaching funkciók\"). The Spanish interface deliberately keeps the loanwords coach, coaching and sparring (\"Pregunta a tu coach\", \"Para Coaches\", \"Rondas de sparring\"); they are Spanish there.", "Long values in selects, chips and narrow fields are shortened with an ellipsis (…) on purpose.", "This is a local test stack. Google sign-in, Stripe checkout, ElevenLabs voices and Google Docs export are not configured here, so their failures are expected.", "The client runs on a Vite dev server, so the first visit to a page can take a few seconds to render.", @@ -79,11 +81,10 @@ export default { app: { url: appUrl, identity: "debatelab-verify-stack", environment: "test" }, }, ], - // AI steps wait on Edge Functions, and a free model can take minutes per judgment. + // AI steps wait on Edge Functions, and a judgment can take a minute. timeout: 600_000, - // Judgments go to the paid judge, and replayed steps call no model, so three - // workers keep the page sweep short. Recording many new flows at once can - // trip the free actor's rate limit; record with a paid actor (docs/testing.md). + // Judgments dominate the run time, and replayed steps call no model, so + // three workers keep the page sweep short. workers: 3, // A test that skips after a failed soft assertion or attempt is a failure. failOnSkippedFailure: true, diff --git a/e2e/full-stack/account-data.spec.ts b/e2e/full-stack/account-data.spec.ts index f55851b1e..1d14955fe 100644 --- a/e2e/full-stack/account-data.spec.ts +++ b/e2e/full-stack/account-data.spec.ts @@ -79,7 +79,7 @@ test("Delete All My Data erases the account's saved work and nobody else's", asy test("the output language setting follows the account and reaches the model", async ({ signedIn: page, user, browser }) => { await page.goto("/app/settings"); - await page.locator("div.space-y-2", { has: page.getByText("AI Output Language", { exact: true }) }).getByRole("combobox").click(); + await page.getByRole("combobox", { name: "AI Output Language" }).click(); await page.getByRole("option", { name: "Magyar" }).click(); await expect.poll(async () => { const { data } = await admin.from("user_settings").select("output_language").eq("user_id", await user.userId()).single(); diff --git a/package-lock.json b/package-lock.json index 7bccc8eef..3dc00cf0e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,7 @@ "@capacitor/status-bar": "^8.0.2", "@hocuspocus/provider": "4.7.0", "@hocuspocus/server": "4.7.0", - "@hookform/resolvers": "^3.10.0", + "@hookform/resolvers": "^5.4.0", "@radix-ui/react-accordion": "^1.2.11", "@radix-ui/react-alert-dialog": "^1.1.14", "@radix-ui/react-checkbox": "^1.3.2", @@ -1314,12 +1314,15 @@ } }, "node_modules/@hookform/resolvers": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/@hookform/resolvers/-/resolvers-3.10.0.tgz", - "integrity": "sha512-79Dv+3mDF7i+2ajj7SkypSKHhl1cbln1OGavqrsF7p6mbUv11xpqpacPsGDCTRvCSjEEIez2ef1NveSVL3b0Ag==", + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/@hookform/resolvers/-/resolvers-5.4.0.tgz", + "integrity": "sha512-EIsqr/t/qbinPIhGjMdtvutIN1Kk4uwbROE9/UQ93CAVGR7GkA7Y92+fX80OzXi/OB67jVFYwKGO1WzkxmkFZw==", "license": "MIT", + "dependencies": { + "@standard-schema/utils": "^0.3.0" + }, "peerDependencies": { - "react-hook-form": "^7.0.0" + "react-hook-form": "^7.55.0" } }, "node_modules/@humanfs/core": { @@ -3972,6 +3975,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@standard-schema/utils": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@standard-schema/utils/-/utils-0.3.0.tgz", + "integrity": "sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g==", + "license": "MIT" + }, "node_modules/@supabase/auth-js": { "version": "2.114.0", "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.114.0.tgz", diff --git a/package.json b/package.json index 6ec536adf..4c258e757 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ "@capacitor/status-bar": "^8.0.2", "@hocuspocus/provider": "4.7.0", "@hocuspocus/server": "4.7.0", - "@hookform/resolvers": "^3.10.0", + "@hookform/resolvers": "^5.4.0", "@radix-ui/react-accordion": "^1.2.11", "@radix-ui/react-alert-dialog": "^1.1.14", "@radix-ui/react-checkbox": "^1.3.2", diff --git a/src/components/features/factoids/FactoidsForm.tsx b/src/components/features/factoids/FactoidsForm.tsx index 94226eabc..31286d4ae 100644 --- a/src/components/features/factoids/FactoidsForm.tsx +++ b/src/components/features/factoids/FactoidsForm.tsx @@ -1,6 +1,7 @@ // src/components/features/factoids/FactoidsForm.tsx import { useTranslation } from 'react-i18next'; import { useForm } from 'react-hook-form'; +import type { z } from 'zod'; import { zodResolver } from '@hookform/resolvers/zod'; import { BookOpen, Search } from 'lucide-react'; import { Button } from '@/components/ui/button'; @@ -35,7 +36,8 @@ export function FactoidsForm({ onSubmit, isLoading, debateContext = true }: Fact const { t } = useTranslation(); const { factoidsForm, setFactoidsForm } = useSessionStore(); - const { register, handleSubmit, setValue, watch, formState: { errors } } = useForm({ + // The schema coerces and defaults fields, so the form's input type differs from what submit receives. + const { register, handleSubmit, setValue, watch, formState: { errors } } = useForm, unknown, FactoidsInput>({ resolver: zodResolver(FactoidsInputSchema), defaultValues: { claim: '', @@ -57,6 +59,7 @@ export function FactoidsForm({ onSubmit, isLoading, debateContext = true }: Fact }, [watch, setFactoidsForm]); const currentValues = watch(); + const sourceCount = Number(currentValues.sourceCount) || 5; return ( @@ -130,12 +133,12 @@ export function FactoidsForm({ onSubmit, isLoading, debateContext = true }: Fact
- {currentValues.sourceCount || 5} + {sourceCount}
- +
- +
- +