Repository navigation
test: agentic browser tests with TesterArmy e2e #4
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Agentic E2E | |
| # The e2e suite in agentic/ against the local stack in fake mode: an AI agent | |
| # drives goal-level flows and judges screens. Recorded flows replay from | |
| # agentic/replay without model calls; judgments always call the model. | |
| # On demand it runs one exploration (a persona looking for bugs) instead. | |
| # Not a required check. See docs/testing.md#agentic-browser-tests. | |
| on: | |
| pull_request: | |
| paths: | |
| - "agentic/**" | |
| - "e2e.config.ts" | |
| - "e2e/full-stack/stack-core.ts" | |
| - ".claude/skills/verify-debatelab/bin/verify.sh" | |
| - ".github/workflows/agentic-e2e.yml" | |
| schedule: | |
| - cron: "47 3 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| explore: | |
| description: "Exploration goal. Empty runs the test suite instead." | |
| required: false | |
| default: "" | |
| persona: | |
| description: "Agent from e2e.config.ts that explores" | |
| type: choice | |
| default: first-timer | |
| options: [default, first-timer, coach, edge-input, hungarian] | |
| target: | |
| description: "Viewport to explore on" | |
| type: choice | |
| default: desktop | |
| options: [desktop, mobile] | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: agentic-e2e-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| agentic-e2e: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Node | |
| uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 | |
| with: | |
| node-version: 22 | |
| - name: Set up Supabase CLI | |
| uses: supabase/setup-cli@46f7f98c7f948ad727d22c1e67fab04c223a0520 # v3.0.0 | |
| with: | |
| version: 2.116.0 | |
| - name: Install dependencies | |
| run: npm ci | |
| - name: Install Chromium | |
| run: npx playwright install --with-deps chromium | |
| # Without the key every agent test skips, which would read as a pass. | |
| # Same accepted risk as live-e2e.yml: pull-request code runs with it. | |
| - name: Require the testing agent's model key | |
| env: | |
| LIVE_OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }} | |
| run: test -n "$LIVE_OPENROUTER_API_KEY" || { echo "LIVE_OPENROUTER_API_KEY is not set" >&2; exit 1; } | |
| - name: Start the local stack (fake model provider) | |
| env: | |
| SUPABASE_INTERNAL_IMAGE_REGISTRY: public.ecr.aws | |
| run: .claude/skills/verify-debatelab/bin/verify.sh up | |
| - name: Agentic suite | |
| if: inputs.explore == '' | |
| env: | |
| LIVE_OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }} | |
| E2E_AGENT_MODEL: ${{ vars.E2E_AGENT_MODEL }} | |
| E2E_JUDGE_MODEL: ${{ vars.E2E_JUDGE_MODEL }} | |
| # Bug-bash repro tests fail until their bug is fixed; they run on demand. | |
| run: .claude/skills/verify-debatelab/bin/verify.sh e2e run --exclude-tag repro --reporter list,markdown,junit | |
| - name: Exploration | |
| if: inputs.explore != '' | |
| env: | |
| LIVE_OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }} | |
| E2E_AGENT_MODEL: ${{ vars.E2E_AGENT_MODEL }} | |
| E2E_JUDGE_MODEL: ${{ vars.E2E_JUDGE_MODEL }} | |
| GOAL: ${{ inputs.explore }} | |
| PERSONA: ${{ inputs.persona }} | |
| TARGET: ${{ inputs.target }} | |
| run: >- | |
| .claude/skills/verify-debatelab/bin/verify.sh e2e explore "$GOAL" | |
| --session student --agent "$PERSONA" --target "$TARGET" | |
| --trace=on --video=on --reporter list,markdown | |
| - name: Publish the report | |
| if: always() | |
| run: | | |
| for summary in .verify/artifacts/*-e2e-*/summary.md; do | |
| [ -f "$summary" ] && cat "$summary" >>"$GITHUB_STEP_SUMMARY" | |
| done | |
| true | |
| - name: Stop the local stack | |
| if: always() | |
| run: .claude/skills/verify-debatelab/bin/verify.sh down | |
| # Local test data only: fake model output, throwaway demo users. | |
| - name: Upload evidence | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: agentic-e2e-${{ github.run_id }} | |
| path: | | |
| .verify/artifacts/ | |
| .verify/logs/ | |
| if-no-files-found: ignore | |
| retention-days: 7 |