chained_e2e.yml

name: 'Testing: E2E (Chained)'

on:
  push:
    branches:
      - 'main'
  merge_group:
  workflow_run:
    workflows: ['Trigger E2E']
    types: ['completed']
  workflow_dispatch:
    inputs:
      head_sha:
        description: 'SHA of the commit to test'
        required: true
      repo_name:
        description: 'Repository name (e.g., owner/repo)'
        required: true

concurrency:
  group: '${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.ref }}'
  cancel-in-progress: |-
    ${{ github.event_name != 'push' && github.event_name != 'merge_group' }}

permissions:
  contents: 'read'
  statuses: 'write'

jobs:
  merge_queue_skipper:
    name: 'Merge Queue Skipper'
    permissions: 'read-all'
    runs-on: 'gemini-cli-ubuntu-16-core'
    if: "github.repository == 'google-gemini/gemini-cli'"
    outputs:
      skip: '${{ steps.merge-queue-e2e-skipper.outputs.skip-check }}'
    steps:
      - id: 'merge-queue-e2e-skipper'
        uses: 'cariad-tech/merge-queue-ci-skipper@1032489e59437862c90a08a2c92809c903883772' # ratchet:cariad-tech/merge-queue-ci-skipper@main
        with:
          secret: '${{ secrets.GEMINI_CLI_ROBOT_GITHUB_PAT }}'
    continue-on-error: true

  download_repo_name:
    runs-on: 'gemini-cli-ubuntu-16-core'
    if: "github.repository == 'google-gemini/gemini-cli' && (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run')"
    outputs:
      repo_name: '${{ steps.output-repo-name.outputs.repo_name }}'
      head_sha: '${{ steps.output-repo-name.outputs.head_sha }}'
    steps:
      - name: 'Mock Repo Artifact'
        if: "${{ github.event_name == 'workflow_dispatch' }}"
        env:
          REPO_NAME: '${{ github.event.inputs.repo_name }}'
        run: |
          mkdir -p ./pr
          echo "${REPO_NAME}" > ./pr/repo_name
      - uses: 'actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' # ratchet:actions/upload-artifact@v4
        with:
          name: 'repo_name'
          path: 'pr/'
      - name: 'Download the repo_name artifact'
        uses: 'actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0' # ratchet:actions/download-artifact@v5
        env:
          RUN_ID: "${{ github.event_name == 'workflow_run' && github.event.workflow_run.id || github.run_id  }}"
        with:
          github-token: '${{ secrets.GITHUB_TOKEN }}'
          name: 'repo_name'
          run-id: '${{ env.RUN_ID }}'
          path: '${{ runner.temp }}/artifacts'
      - name: 'Output Repo Name and SHA'
        id: 'output-repo-name'
        uses: 'actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd' # ratchet:actions/github-script@v8
        with:
          github-token: '${{ secrets.GITHUB_TOKEN }}'
          script: |
            const fs = require('fs');
            const path = require('path');
            const temp = '${{ runner.temp }}/artifacts';
            const repoPath = path.join(temp, 'repo_name');
            if (fs.existsSync(repoPath)) {
              const repo_name = String(fs.readFileSync(repoPath)).trim();
              core.setOutput('repo_name', repo_name);
            }
            const shaPath = path.join(temp, 'head_sha');
            if (fs.existsSync(shaPath)) {
              const head_sha = String(fs.readFileSync(shaPath)).trim();
              core.setOutput('head_sha', head_sha);
            }

  parse_run_context:
    name: 'Parse run context'
    runs-on: 'gemini-cli-ubuntu-16-core'
    needs: 'download_repo_name'
    if: "github.repository == 'google-gemini/gemini-cli' && always()"
    outputs:
      repository: '${{ steps.set_context.outputs.REPO }}'
      sha: '${{ steps.set_context.outputs.SHA }}'
    steps:
      - id: 'set_context'
        name: 'Set dynamic repository and SHA'
        env:
          REPO: '${{ needs.download_repo_name.outputs.repo_name || github.repository }}'
          SHA: '${{ needs.download_repo_name.outputs.head_sha || github.event.inputs.head_sha || github.event.workflow_run.head_sha || github.sha }}'
        shell: 'bash'
        run: |
          echo "REPO=$REPO" >> "$GITHUB_OUTPUT"
          echo "SHA=$SHA" >> "$GITHUB_OUTPUT"

  set_pending_status:
    runs-on: 'gemini-cli-ubuntu-16-core'
    permissions: 'write-all'
    needs:
      - 'parse_run_context'
    if: "github.repository == 'google-gemini/gemini-cli' && always()"
    steps:
      - name: 'Set pending status'
        uses: 'myrotvorets/set-commit-status-action@16037e056d73b2d3c88e37e393ff369047f70886' # ratchet:myrotvorets/set-commit-status-action@master
        if: "github.repository == 'google-gemini/gemini-cli' && always()"
        with:
          allowForks: 'true'
          repo: '${{ github.repository }}'
          sha: '${{ needs.parse_run_context.outputs.sha }}'
          token: '${{ secrets.GEMINI_CLI_ROBOT_GITHUB_PAT }}'
          status: 'pending'
          context: 'E2E (Chained)'

  e2e_linux:
    name: 'E2E Test (Linux) - ${{ matrix.sandbox }}'
    needs:
      - 'merge_queue_skipper'
      - 'parse_run_context'
    runs-on: 'gemini-cli-ubuntu-16-core'
    if: |
      github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true')
    strategy:
      fail-fast: false
      matrix:
        sandbox:
          - 'sandbox:none'
          - 'sandbox:docker'
        node-version:
          - '20.x'

    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5
        with:
          ref: '${{ needs.parse_run_context.outputs.sha }}'
          repository: '${{ needs.parse_run_context.outputs.repository }}'
          persist-credentials: false

      - name: 'Set up Node.js ${{ matrix.node-version }}'
        uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4
        with:
          node-version: '${{ matrix.node-version }}'

      - name: 'Install dependencies'
        run: 'npm ci'

      - name: 'Build project'
        run: 'npm run build'

      - name: 'Set up Docker'
        if: "${{matrix.sandbox == 'sandbox:docker'}}"
        uses: 'docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435' # ratchet:docker/setup-buildx-action@v3

      - name: 'Run E2E tests'
        env:
          GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
          GEMINI_CLI_TRUST_WORKSPACE: true
          KEEP_OUTPUT: 'true'
          VERBOSE: 'true'
          BUILD_SANDBOX_FLAGS: '--cache-from type=gha --cache-to type=gha,mode=max'
        shell: 'bash'
        run: |
          if [[ "${{ matrix.sandbox }}" == "sandbox:docker" ]]; then
            npm run test:integration:sandbox:docker
          else
            npm run test:integration:sandbox:none
          fi

  e2e_mac:
    name: 'E2E Test (macOS)'
    needs:
      - 'merge_queue_skipper'
      - 'parse_run_context'
    runs-on: 'macos-latest-large'
    if: |
      github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true')
    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5
        with:
          ref: '${{ needs.parse_run_context.outputs.sha }}'
          repository: '${{ needs.parse_run_context.outputs.repository }}'
          persist-credentials: false

      - name: 'Set up Node.js 20.x'
        uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4
        with:
          node-version: '20.x'

      - name: 'Install dependencies'
        run: 'npm ci'

      - name: 'Build project'
        run: 'npm run build'

      - name: 'Fix rollup optional dependencies on macOS'
        if: "${{runner.os == 'macOS'}}"
        run: |
          npm cache clean --force
      - name: 'Run E2E tests (non-Windows)'
        if: "${{runner.os != 'Windows'}}"
        env:
          GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
          GEMINI_CLI_TRUST_WORKSPACE: true
          KEEP_OUTPUT: 'true'
          SANDBOX: 'sandbox:none'
          VERBOSE: 'true'
        run: 'npm run test:integration:sandbox:none'

  e2e_windows:
    name: 'Slow E2E - Win'
    needs:
      - 'merge_queue_skipper'
      - 'parse_run_context'
    if: |
      github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true')
    runs-on: 'gemini-cli-windows-16-core'
    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5
        with:
          ref: '${{ needs.parse_run_context.outputs.sha }}'
          repository: '${{ needs.parse_run_context.outputs.repository }}'
          persist-credentials: false

      - name: 'Set up Node.js 20.x'
        uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4
        with:
          node-version: '20.x'
          cache: 'npm'

      - name: 'Configure Windows Defender exclusions'
        run: |
          Add-MpPreference -ExclusionPath $env:GITHUB_WORKSPACE -Force
          Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE\node_modules" -Force
          Add-MpPreference -ExclusionPath "$env:GITHUB_WORKSPACE\packages" -Force
          Add-MpPreference -ExclusionPath "$env:TEMP" -Force
        shell: 'pwsh'

      - name: 'Configure npm for Windows performance'
        run: |
          npm config set progress false
          npm config set audit false
          npm config set fund false
          npm config set loglevel error
          npm config set maxsockets 32
          npm config set registry https://registry.npmjs.org/
        shell: 'pwsh'

      - name: 'Install dependencies'
        run: 'npm ci'
        shell: 'pwsh'

      - name: 'Build project'
        run: 'npm run build'
        shell: 'pwsh'

      - name: 'Ensure Chrome is available'
        shell: 'pwsh'
        run: |
          $chromePaths = @(
            "${env:ProgramFiles}\Google\Chrome\Application\chrome.exe",
            "${env:ProgramFiles(x86)}\Google\Chrome\Application\chrome.exe"
          )
          $chromeExists = $chromePaths | Where-Object { Test-Path $_ } | Select-Object -First 1
          if (-not $chromeExists) {
            Write-Host 'Chrome not found, installing via Chocolatey...'
            choco install googlechrome -y --no-progress --ignore-checksums
          }
          $installed = $chromePaths | Where-Object { Test-Path $_ } | Select-Object -First 1
          if ($installed) {
            Write-Host "Chrome found at: $installed"
            & $installed --version
          } else {
            Write-Error 'Chrome installation failed'
            exit 1
          }

      - name: 'Run E2E tests'
        env:
          GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
          GEMINI_CLI_TRUST_WORKSPACE: true
          KEEP_OUTPUT: 'true'
          SANDBOX: 'sandbox:none'
          VERBOSE: 'true'
          NODE_OPTIONS: '--max-old-space-size=32768 --max-semi-space-size=256'
          UV_THREADPOOL_SIZE: '32'
          NODE_ENV: 'test'
        shell: 'pwsh'
        run: 'npm run test:integration:sandbox:none'

  evals:
    name: 'Evals (ALWAYS_PASSING)'
    needs:
      - 'merge_queue_skipper'
      - 'parse_run_context'
    runs-on: 'gemini-cli-ubuntu-16-core'
    if: |
      github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true')
    steps:
      - name: 'Checkout'
        uses: 'actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955' # ratchet:actions/checkout@v5
        with:
          ref: '${{ needs.parse_run_context.outputs.sha }}'
          repository: '${{ needs.parse_run_context.outputs.repository }}'
          persist-credentials: false
          fetch-depth: 0

      - name: 'Set up Node.js 20.x'
        uses: 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020' # ratchet:actions-node@v4
        with:
          node-version: '20.x'

      - name: 'Install dependencies'
        run: 'npm ci'

      - name: 'Build project'
        run: 'npm run build'

      - name: 'Check if evals should run'
        id: 'check_evals'
        run: |
          SHOULD_RUN=$(node scripts/changed_prompt.js)
          echo "should_run=$SHOULD_RUN" >> "$GITHUB_OUTPUT"

      - name: 'Run Evals (Required to pass)'
        if: "${{ steps.check_evals.outputs.should_run == 'true' }}"
        env:
          GEMINI_API_KEY: '${{ secrets.GEMINI_API_KEY }}'
          GEMINI_CLI_TRUST_WORKSPACE: true
          GEMINI_MODEL: 'gemini-3-pro-preview'
          # Only run always passes behavioral tests.
          EVAL_SUITE_TYPE: 'behavioral'
          # Disable Vitest internal retries to avoid double-retrying;
          # custom retry logic is handled in evals/test-helper.ts
          VITEST_RETRY: 0
        run: 'npm run test:always_passing_evals'

      - name: 'Upload Reliability Logs'
        if: "always() && steps.check_evals.outputs.should_run == 'true'"
        uses: 'actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' # ratchet:actions/upload-artifact@v4
        with:
          name: 'eval-logs-${{ github.run_id }}-${{ github.run_attempt }}'
          path: 'evals/logs/api-reliability.jsonl'
          retention-days: 7

  e2e:
    name: 'E2E'
    if: |
      github.repository == 'google-gemini/gemini-cli' && always() && (needs.merge_queue_skipper.result !='success' || needs.merge_queue_skipper.outputs.skip != 'true')
    needs:
      - 'e2e_linux'
      - 'e2e_mac'
      - 'e2e_windows'
      - 'evals'
      - 'merge_queue_skipper'
    runs-on: 'gemini-cli-ubuntu-16-core'
    steps:
      - name: 'Check E2E test results'
        run: |
          if [[ ${NEEDS_E2E_LINUX_RESULT} != 'success' || \
               ${NEEDS_E2E_MAC_RESULT} != 'success' || \
               ${NEEDS_E2E_WINDOWS_RESULT} != 'success' || \
               ${NEEDS_EVALS_RESULT} != 'success' ]]; then
            echo "One or more E2E jobs failed."
            exit 1
          fi
          echo "All required E2E jobs passed!"
        env:
          NEEDS_E2E_LINUX_RESULT: '${{ needs.e2e_linux.result }}'
          NEEDS_E2E_MAC_RESULT: '${{ needs.e2e_mac.result }}'
          NEEDS_E2E_WINDOWS_RESULT: '${{ needs.e2e_windows.result }}'
          NEEDS_EVALS_RESULT: '${{ needs.evals.result }}'

  set_workflow_status:
    runs-on: 'gemini-cli-ubuntu-16-core'
    permissions: 'write-all'
    if: "github.repository == 'google-gemini/gemini-cli' && always()"
    needs:
      - 'parse_run_context'
      - 'e2e'
    steps:
      - name: 'Set workflow status'
        uses: 'myrotvorets/set-commit-status-action@16037e056d73b2d3c88e37e393ff369047f70886' # ratchet:myrotvorets/set-commit-status-action@master
        if: "github.repository == 'google-gemini/gemini-cli' && always()"
        with:
          allowForks: 'true'
          repo: '${{ github.repository }}'
          sha: '${{ needs.parse_run_context.outputs.sha }}'
          token: '${{ secrets.GITHUB_TOKEN }}'
          status: '${{ needs.e2e.result }}'
          context: 'E2E (Chained)'