test.yml

# .github/workflows/test.yml
name: Python Test

on:
  push:
    branches:
      - main
      - 'release/v**'
      - 'maint/**'
  pull_request:
    branches:
      - main
      - 'release/v**'
      - 'maint/**'
  workflow_dispatch:

permissions:
  contents: read

env:
  # rhosocial-activerecord and rhosocial-activerecord-testsuite dev releases are
  # not published to PyPI; install them from the feature/parameter-inline branch.
  CORE_PKG: "rhosocial-activerecord @ git+https://github.com/rhosocial/python-activerecord.git@feature/activerecord-derived-ddl"
  TESTSUITE_PKG: "rhosocial-activerecord-testsuite @ git+https://github.com/rhosocial/python-activerecord-testsuite.git@feature/activerecord-derived-ddl"

jobs:
  test:
    name: Python ${{ matrix.python-version }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']

    steps:
      - name: Check out the repository
        uses: actions/checkout@v6

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          pip install "$CORE_PKG" "$TESTSUITE_PKG"
          pip install -r requirements.txt
          pip install -r requirements-test.txt

      - name: Install package
        run: |
          pip install -e . --no-deps

      - name: Start BigQuery emulator
        run: |
          docker run -d --name bigquery-emulator -p 9050:9050 -p 9060:9060 \
            ghcr.io/goccy/bigquery-emulator:latest --project=test --dataset=test_dataset
          sleep 4
          docker ps

      - name: Run backend tests
        run: |
          pytest tests/ --tb=short -q -o asyncio_mode=auto
        env:
          PYTHONPATH: tests
          BIGQUERY_SCENARIOS_CONFIG_PATH: tests/config/bigquery_scenarios.yaml
          BIGQUERY_ACTIVE_SCENARIOS: bigquery_emulator
          TESTSUITE_PROVIDER_REGISTRY: providers.registry:provider_registry

      - name: Run testsuite feature tests (sharded across parallel emulators)
        run: |
          # goccy/bigquery-emulator serializes all requests (single-threaded),
          # so pytest-xdist gives no speedup. Instead we shard the testsuite
          # subdirectories across SHARDS emulator containers on distinct ports,
          # each shard being its own sequential pipeline with a fresh emulator
          # per subdirectory (the emulator degrades as jobs/tables accumulate).
          SHARDS=4
          BASE_PORT=9050
          # The "Start BigQuery emulator" step above binds host port 9050 via
          # its own container; drop it so the shard containers can own the
          # 9050-9053 ports.
          docker rm -f bigquery-emulator >/dev/null 2>&1 || true
          for i in $(seq 0 $((SHARDS-1))); do
            docker run -d --name bq-emulator-$i -p $((BASE_PORT+i)):9050 -p $((9060+i)):9060 \
              ghcr.io/goccy/bigquery-emulator:latest --project=test --dataset=test_dataset >/dev/null
          done
          TESTSUITE_DIR=$(python - <<'PY'
          import pathlib
          import rhosocial.activerecord.testsuite as testsuite

          print(pathlib.Path(testsuite.__file__).parent / "feature")
          PY
          )
          SUBDIRS=()
          # goccy/bigquery-emulator is unreliable for these query
          # subdirectories: individual queries stall past the per-test timeout
          # or error out, so they are excluded from CI coverage.
          SKIP_SUBDIRS=" query/error_handling query/eager_loading query/optimization query/range_queries "
          for suite in basic relation query; do
            for d in "$TESTSUITE_DIR/$suite"/*/; do
              [ -d "$d" ] || continue
              name="$(basename "$d")"
              case "$name" in _*|.*|fixtures) continue ;; esac
              sub="$suite/$name"
              case "$SKIP_SUBDIRS" in *" $sub "*) continue ;; esac
              SUBDIRS+=("$sub")
            done
          done
          # Round-robin shard assignment. The emulator is CPU-bound, so
          # parallel emulators compete for cores and give no net speedup on a
          # 4-core runner; sharding is kept for resilience (a stuck subdir
          # only stalls its own shard, not the whole job).
          # (SHARD_GROUPS: ``GROUPS`` is a bash special array, avoid it.)
          declare -a SHARD_GROUPS=()
          for i in $(seq 0 $((SHARDS-1))); do SHARD_GROUPS[$i]=""; done
          idx=0
          for sub in "${SUBDIRS[@]}"; do
            SHARD_GROUPS[$((idx % SHARDS))]="${SHARD_GROUPS[$((idx % SHARDS))]} $sub"
            idx=$((idx + 1))
          done
          pids=()
          for i in $(seq 0 $((SHARDS-1))); do
            (
              port=$((BASE_PORT+i))
              export BIGQUERY_EMULATOR_PORT=$port
              restart_emulator() {
                docker rm -f bq-emulator-$i >/dev/null 2>&1 || true
                docker run -d --name bq-emulator-$i -p $((BASE_PORT+i)):9050 -p $((9060+i)):9060 \
                  ghcr.io/goccy/bigquery-emulator:latest --project=test --dataset=test_dataset >/dev/null
                sleep 4
              }
              for sub in ${SHARD_GROUPS[$i]}; do
                suite="${sub%%/*}"
                name="${sub#*/}"
                d="$TESTSUITE_DIR/$suite/$name"
                echo "=== [shard $i] $sub (fresh emulator) ==="
                restart_emulator
                set +e
                extra_args=()
                # The goccy/bigquery-emulator does not support ROLLBACK; these
                # cases require real BigQuery multi-statement transactions.
                if [ "$sub" = "basic/crud" ]; then
                  extra_args=(-k "not test_transaction_crud")
                fi
                # BigQuery does not enforce NOT NULL constraints at storage level.
                if [ "$sub" = "basic/type_adapter" ]; then
                  extra_args=(-k "not test_db_null_with_non_optional_field_raises_error")
                fi
                # goccy/bigquery-emulator returns 0 for SUM() OVER on NUMERIC
                # columns (works for INT64/FLOAT64); the dialect itself is fine.
                if [ "$sub" = "query/window_functions" ]; then
                  extra_args=(-k "not test_aggregate_window_functions and not test_unbounded_window_frames")
                fi
                # Each subdirectory gets a hard timeout so it can never hang
                # the whole job.
                timeout 900 pytest "$d" --tb=short -q -o asyncio_mode=auto \
                  --confcutdir="$TESTSUITE_DIR/$suite" "${extra_args[@]}"
                rc=$?
                set -e
                if [ $rc -eq 5 ]; then
                  echo "no tests collected in $sub, skipping"
                elif [ $rc -ne 0 ]; then
                  echo "FAILED shard $i: $sub (rc=$rc)"
                  exit $rc
                fi
              done
              echo "=== [shard $i] group done ==="
            ) > /tmp/shard_$i.log 2>&1 &
            pids+=($!)
          done
          failed=0
          for pid in "${pids[@]}"; do
            wait "$pid" || failed=$((failed+1))
          done
          if [ "$failed" -gt 0 ]; then
            for i in $(seq 0 $((SHARDS-1))); do
              echo "----- shard $i log tail -----"
              tail -30 /tmp/shard_$i.log
            done
            exit 1
          fi
        env:
          PYTHONPATH: tests
          BIGQUERY_SCENARIOS_CONFIG_PATH: tests/config/bigquery_scenarios.yaml
          BIGQUERY_ACTIVE_SCENARIOS: bigquery_emulator
          TESTSUITE_PROVIDER_REGISTRY: providers.registry:provider_registry
        timeout-minutes: 60