fuzz.yaml

name: Fuzz

# runs the fuzzers against SQLAlchemy main and rel_2_0.  differences are
# listed in the run summary, each with a command that replays it locally;
# a run fails if anything is found.  the weekly run then files the findings
# as issues in this repo, at most one open issue per fuzzer and check, and
# closes those whose check passes and whose SQLAlchemy issues are closed
# (see src/sqlalchemy_fuzz/issues.py).  see README.rst for why this runs every
# week.
#
# every run tests a fixed range of seeds, whose result only changes when
# SQLAlchemy or the fuzzer changes, plus as many seeds again sampled at
# random from outside it, to keep looking for new bugs.  the sampling is
# seeded with the run id, so main and rel_2_0 get the same seeds and a
# rerun of a job repeats them.
#
# a push to main that changes the fuzzers also runs them, without filing
# issues, so a new or changed fuzzer gets a run to link to right away.
#
# run from the Actions tab or with:
#
#   gh workflow run fuzz.yaml -R sqlalchemy/sqlalchemy-fuzz \
#       -f seeds=0:20000 -f sample=20000 -f refs='["main"]'
on:
  schedule:
    # mondays at 06:00 UTC
    - cron: "0 6 * * 1"
  push:
    branches:
      - main
    paths:
      - "src/**"
      - "pyproject.toml"
      - ".github/workflows/fuzz.yaml"
  workflow_dispatch:
    inputs:
      seeds:
        description: "fixed range of seeds, e.g. 0:20000"
        required: true
        default: "0:20000"
      sample:
        description: "number of seeds to sample outside the fixed range"
        required: true
        default: "20000"
      sample_seed:
        description: "seed for the sampling, to repeat a run; default: run id"
        required: false
        default: ""
      refs:
        description: "SQLAlchemy refs to test, as a JSON list"
        required: true
        default: '["main", "rel_2_0"]'
      file_issues:
        description: "file findings as issues, as the weekly run does"
        type: boolean
        default: false

permissions:
  contents: read

# a newer push supersedes a run started by an older one
concurrency:
  group: >-
    ${{ github.event_name == 'push' && 'fuzz-push' || github.run_id }}
  cancel-in-progress: true

jobs:
  fuzz:
    name: ${{ matrix.fuzzer }}-${{ matrix.ref }}
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        ref: ${{ fromJSON(inputs.refs || '["main", "rel_2_0"]') }}
        fuzzer:
          - orm-loading
          - orm-evaluate
          - core-cache-key
          - orm-unit-of-work
          - orm-strategies
    env:
      # some SQLAlchemy behavior depends on the iteration order of sets of
      # strings (orm-evaluate: the order SET values are evaluated in), so
      # fix the hash seed for runs of the same seeds to give the same result
      PYTHONHASHSEED: "0"

    steps:
      - name: Checkout sqlalchemy-fuzz
        uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Checkout SQLAlchemy ${{ matrix.ref }}
        uses: actions/checkout@v7
        with:
          repository: sqlalchemy/sqlalchemy
          ref: ${{ matrix.ref }}
          path: sqlalchemy
          persist-credentials: false

      - name: Set up python
        uses: actions/setup-python@v7
        with:
          python-version: "3.13"

      - name: Install SQLAlchemy and sqlalchemy-fuzz into a virtualenv
        env:
          # fail rather than silently falling back to pure python
          REQUIRE_SQLALCHEMY_CEXT: "1"
          REF: ${{ matrix.ref }}
        run: |
          python -m venv .venv
          .venv/bin/python -m pip install --upgrade pip
          .venv/bin/python -m pip install ./sqlalchemy
          .venv/bin/python -m pip install .
          .venv/bin/python -m pip list
          echo "SQLALCHEMY_SHA=$(git -C sqlalchemy rev-parse HEAD)" \
            >> "$GITHUB_ENV"
          echo "SQLALCHEMY_FUZZ_SHA=$GITHUB_SHA" >> "$GITHUB_ENV"
          echo "SQLALCHEMY_REF=$REF" >> "$GITHUB_ENV"
          mkdir results

      - name: Run on fixed and sampled seeds
        env:
          SEEDS: ${{ inputs.seeds || '0:20000' }}
          SAMPLE: ${{ inputs.sample || '20000' }}
          SAMPLE_SEED: ${{ inputs.sample_seed || github.run_id }}
          FUZZER: ${{ matrix.fuzzer }}
        run: |
          .venv/bin/sqla-fuzz "$FUZZER" \
            --seeds "$SEEDS" \
            --sample "$SAMPLE" --sample-seed "$SAMPLE_SEED" \
            --markdown "$GITHUB_STEP_SUMMARY" \
            --json "results/$FUZZER-$SQLALCHEMY_REF.json"

      - name: Upload results
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v7
        with:
          name: results-${{ matrix.fuzzer }}-${{ matrix.ref }}
          path: results/

  file-issues:
    name: file-issues
    needs: fuzz
    if: >-
      ${{ !cancelled()
          && (github.event_name == 'schedule' || inputs.file_issues) }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
      issues: write

    steps:
      - name: Checkout sqlalchemy-fuzz
        uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Set up python
        uses: actions/setup-python@v7
        with:
          python-version: "3.13"

      - name: Download results
        uses: actions/download-artifact@v8
        with:
          pattern: results-*
          path: results
          merge-multiple: true

      - name: File issues
        env:
          GH_TOKEN: ${{ github.token }}
          REFS: ${{ inputs.refs || '["main", "rel_2_0"]' }}
          RUN_URL: >-
            ${{ github.server_url }}/${{ github.repository
            }}/actions/runs/${{ github.run_id }}
        run: |
          python -m pip install .
          sqla-fuzz-issues results/*.json \
            --repo "$GITHUB_REPOSITORY" --run-url "$RUN_URL" --refs "$REFS"