conformance-weekly.yaml

name: conformance-weekly

# Tracks the newest published @modelcontextprotocol/conformance; the
# on:pull_request pipeline pins an exact version instead. Also publishes
# score badges to the orphan `badges` branch, consumed by the README.
on:
  schedule:
    - cron: '0 6 * * 1' # Mondays 06:00 UTC
  workflow_dispatch:

permissions:
  contents: write
  issues: write

jobs:
  server:
    name: conformance / server (${{ matrix.spec-version }}, ${{ matrix.dist-tag }})
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - spec-version: '2025-11-25'
            dist-tag: latest
            path: '/'
            baseline: conformance-baseline-2025-11-25.yml
          - spec-version: '2026-07-28'
            dist-tag: alpha
            path: '/'
            baseline: conformance-baseline-2026-07-28.yml
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-node@v7
        with:
          node-version: '22'
      - run: composer install --prefer-dist --no-progress --no-interaction
      - name: Start conformance server
        run: |
          mkdir -p tests/Conformance/sessions tests/Conformance/logs
          chmod -R 777 tests/Conformance/sessions tests/Conformance/logs
          docker compose -f tests/Conformance/Fixtures/docker-compose.yml up -d
          sleep 5
      - name: Run conformance tests
        working-directory: ./tests/Conformance
        run: >-
          npx --yes @modelcontextprotocol/conformance@${{ matrix.dist-tag }} server
          --url http://localhost:8000${{ matrix.path }}
          --suite all
          --spec-version ${{ matrix.spec-version }}
          --expected-failures ${{ matrix.baseline }}
          --output-dir results
      - name: Generate score badge
        if: always()
        run: php tests/Conformance/score.php server ${{ matrix.spec-version }}
      - name: Show docker logs on failure
        if: failure()
        run: docker compose -f tests/Conformance/Fixtures/docker-compose.yml logs
      - name: Upload conformance results
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: conformance-server-results-${{ matrix.spec-version }}
          path: |
            tests/Conformance/logs
            tests/Conformance/results
      - name: Upload score badge
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: badge-server-${{ matrix.spec-version }}
          path: tests/Conformance/server-conformance-${{ matrix.spec-version }}.json

  client:
    name: conformance / client (${{ matrix.spec-version }}, ${{ matrix.dist-tag }})
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - spec-version: '2025-11-25'
            dist-tag: latest
            baseline: conformance-baseline-2025-11-25.yml
          - spec-version: '2026-07-28'
            dist-tag: alpha
            baseline: conformance-baseline-2026-07-28.yml
    steps:
      - uses: actions/checkout@v7
      - uses: shivammathur/setup-php@v2
        with:
          php-version: '8.4'
          coverage: none
      - uses: actions/setup-node@v7
        with:
          node-version: '22'
      - run: composer install --prefer-dist --no-progress --no-interaction
      - run: mkdir -p tests/Conformance/logs
      - name: Run conformance tests
        working-directory: ./tests/Conformance
        run: >-
          npx --yes @modelcontextprotocol/conformance@${{ matrix.dist-tag }} client
          --command "php ${{ github.workspace }}/tests/Conformance/client.php"
          --suite all
          --spec-version ${{ matrix.spec-version }}
          --expected-failures ${{ matrix.baseline }}
          --output-dir results
      - name: Generate score badge
        if: always()
        run: php tests/Conformance/score.php client ${{ matrix.spec-version }}
      - name: Upload conformance results
        if: failure()
        uses: actions/upload-artifact@v7
        with:
          name: conformance-client-results-${{ matrix.spec-version }}
          path: |
            tests/Conformance/logs
            tests/Conformance/results
      - name: Upload score badge
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: badge-client-${{ matrix.spec-version }}
          path: tests/Conformance/client-conformance-${{ matrix.spec-version }}.json

  notify:
    name: Open issue on failure
    runs-on: ubuntu-latest
    needs: [server, client]
    if: failure()
    env:
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      GH_REPO: ${{ github.repository }}
      EVENT_NAME: ${{ github.event_name }}
      RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
    steps:
      - name: File or comment tracking issue
        run: |
          existing=$(gh issue list --label conformance-weekly --state open --json number --jq '.[0].number // empty')
          if [ -n "$existing" ]; then
            gh issue comment "$existing" --body "New failure on $(date -u +%FT%TZ): $RUN_URL"
          else
            gh issue create \
              --title '[conformance] Weekly conformance run failed' \
              --label conformance-weekly \
              --body "Conformance against the newest published \`@modelcontextprotocol/conformance\` failed.

          - Run: $RUN_URL
          - Trigger: $EVENT_NAME
          - Date: $(date -u +%FT%TZ)

          Upstream likely published a release whose scenarios the SDK does not satisfy. Either fix the SDK, update the conformance fixtures, or add the new failure to the matching \`tests/Conformance/conformance-baseline-<spec-version>.yml\`."
          fi

  publish:
    name: Publish conformance badges
    runs-on: ubuntu-latest
    needs: [server, client]
    if: ${{ !cancelled() && github.repository == 'modelcontextprotocol/php-sdk' }}
    steps:
      - uses: actions/checkout@v7
      - uses: actions/download-artifact@v8
        with:
          pattern: badge-*
          path: badges-in
          merge-multiple: true
      - name: Publish to badges branch
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"

          if git ls-remote --exit-code --heads origin badges >/dev/null 2>&1; then
            git fetch origin badges
            git worktree add badges-wt badges
          else
            git worktree add --detach badges-wt
            git -C badges-wt checkout --orphan badges
            git -C badges-wt rm -rf --quiet . >/dev/null 2>&1 || true
          fi

          rm -f badges-wt/server-conformance.json badges-wt/client-conformance.json
          cp badges-in/*.json badges-wt/

          cd badges-wt
          git add -A
          if git diff --cached --quiet; then
            echo "Conformance scores unchanged."
          else
            git commit -m "Update conformance score badges"
            git push origin badges
          fi