DebateOwl/.github/workflows/live-e2e.yml at main · Programcode/DebateOwl · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
name: AI evals

# The AI evals: the local stack against real OpenRouter, Jev, and Exa. Every
# run spends provider credit, so it runs weekly and on demand, never on pull
# requests. The production canary's daily run makes two real AI requests, so
# provider drift still shows up within a day.
# - live: the live provider journeys, about 8 minutes.
# - model-matrix: one case from every catalog model, unpinned, so a model that
#   times out or returns invalid JSON fails here before users hit it. About $1.
# A branch that changes AI, research, prompts or model routing runs them with
# `gh workflow run live-e2e.yml --ref <branch>` (add `-f suite=model-matrix`
# or `-f suite=both`).
on:
  schedule:
    - cron: "17 3 * * 1"
  workflow_dispatch:
    inputs:
      suite:
        description: "Which eval to run"
        type: choice
        default: live
        options: [live, model-matrix, both]

permissions:
  contents: read

concurrency:
  group: live-e2e-${{ github.ref }}-${{ inputs.suite || 'schedule' }}
  cancel-in-progress: true

jobs:
  eval:
    # Forks have none of the secrets this job needs.
    if: github.repository == 'Programcode/DebateOwl'
    strategy:
      fail-fast: false
      matrix:
        # The schedule runs both; a dispatch runs the suite it names.
        suite: ${{ fromJSON((github.event_name == 'schedule' || inputs.suite == 'both') && '["live","model-matrix"]' || format('["{0}"]', inputs.suite)) }}
    name: ${{ matrix.suite }}
    runs-on: ubuntu-latest
    timeout-minutes: 45
    steps:
      - name: Checkout code
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false

      - name: Set up Node
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: 22

      - name: Set up Supabase CLI
        uses: supabase/setup-cli@46f7f98c7f948ad727d22c1e67fab04c223a0520 # v3.0.0
        with:
          version: 2.121.0

      - name: Install dependencies
        run: npm ci

      - name: Install Chromium
        run: npx playwright install --with-deps chromium

      # db-gate.sh fails before starting anything when either key is missing.
      # Accepted risk (2026-09-24): a dispatch can run branch code with these
      # keys, so anyone who can push a branch here could read them. Keep them
      # low-limit.
      - name: Live provider journeys
        env:
          LIVE_PROVIDERS: 1
          # live: a repository variable override, empty falls back to
          # db-gate.sh's default. model-matrix: every catalog model.
          LIVE_MODEL: ${{ matrix.suite == 'model-matrix' && 'as-requested' || vars.LIVE_MODEL }}
          LIVE_OPENROUTER_API_KEY: ${{ secrets.LIVE_OPENROUTER_API_KEY }}
          LIVE_EXA_API_KEY: ${{ secrets.LIVE_EXA_API_KEY }}
          SUPABASE_INTERNAL_IMAGE_REGISTRY: public.ecr.aws
          WORKSPACE_GATE_ARTIFACT_ROOT: ${{ runner.temp }}/workspace-gate
        run: ./.superset/db-gate.sh

      - name: Upload live failure artifacts
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: ${{ matrix.suite }}-failure-${{ github.run_id }}
          path: |
            ${{ runner.temp }}/workspace-gate/
            test-results/
            playwright-report/
          if-no-files-found: ignore
          retention-days: 7