Add ignore_order_in for $lookup joined array in compatibility tests #1861
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PR Tests | |
| on: | |
| pull_request: | |
| branches: [main] | |
| jobs: | |
| # Derive the test matrix from dev/compose.yaml so targets are declared in one | |
| # place. Each entry has the target name, its compose profile, connection | |
| # string, and engine name. | |
| discover-targets: | |
| name: Discover test targets | |
| runs-on: ubuntu-latest | |
| outputs: | |
| targets: ${{ steps.matrix.outputs.targets }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: pip install -r requirements.txt | |
| - name: Build target matrix from compose | |
| id: matrix | |
| run: | | |
| TARGETS=$(python -m documentdb_tests.framework.ci_matrix) | |
| echo "targets=$TARGETS" >> "$GITHUB_OUTPUT" | |
| test: | |
| name: "Tests (${{ matrix.target.name }})" | |
| needs: discover-targets | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: ${{ fromJson(needs.discover-targets.outputs.targets) }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: pip install -r requirements.txt | |
| - name: Start ${{ matrix.target.name }} target | |
| run: dev/compose-up.sh "${{ matrix.target.profile }}" | |
| - name: Run compatibility tests against ${{ matrix.target.name }} | |
| run: | | |
| # Sample memory, load, and per-container usage to the live step log | |
| # while the tests run. Writing to the live log means the samples | |
| # survive even when the job is killed before it finishes, which is | |
| # when a resource problem is most likely to show up. Every line begins | |
| # with resource-monitor so the timeline can be read on its own by | |
| # filtering the log for that word. | |
| echo "resource-monitor samples follow. Each line shows the UTC time, host memory as used over total in megabytes, and swap in megabytes. It then shows the 1, 5, and 15 minute load average and the memory each container is using. Filter the log for resource-monitor to read them on their own." | |
| sample_resources() { | |
| set +e | |
| while true; do | |
| memory=$(free -m | awk '/^Mem:/{print $3"/"$2}') | |
| swap=$(free -m | awk '/^Swap:/{print $3}') | |
| load=$(cut -d' ' -f1-3 /proc/loadavg | tr ' ' ',') | |
| containers=$(docker stats --no-stream --format '{{.Name}} {{.MemUsage}}' 2>/dev/null \ | |
| | awk '{print $1"="$2}' | paste -sd' ' -) | |
| echo "resource-monitor $(date -u +%H:%M:%S) memory=${memory}MB swap=${swap}MB load=${load} ${containers}" | |
| sleep 10 | |
| done | |
| } | |
| sample_resources & | |
| # When the machine runs out of memory the kernel logs a line naming | |
| # the process it kills. Follow the kernel log and forward that line so | |
| # it lands in the step log, since it is the clearest evidence of an | |
| # out of memory kill. The search terms are the exact phrases the | |
| # kernel writes. | |
| ( sudo -n dmesg --follow 2>/dev/null \ | |
| | grep --line-buffered -iE 'out of memory|killed process|oom-kill' \ | |
| | sed -u 's/^/resource-monitor out-of-memory /' ) & | |
| trap 'kill %1 %2 2>/dev/null || true' EXIT | |
| pytest documentdb_tests/compatibility/tests \ | |
| --connection-string "${{ matrix.target.connection_string }}" \ | |
| --engine-name "${{ matrix.target.engine }}" \ | |
| -n auto \ | |
| -v \ | |
| --json-report --json-report-file=${{ github.workspace }}/.test-results/${{ matrix.target.name }}-report.json \ | |
| --junitxml=${{ github.workspace }}/.test-results/${{ matrix.target.name }}-results.xml | |
| - name: Dump container logs | |
| if: always() | |
| run: | | |
| # One file per service in the profile | |
| mkdir -p "${{ github.workspace }}/.test-results/container-logs" | |
| for svc in $(docker compose -f dev/compose.yaml --profile ${{ matrix.target.profile }} config --services); do | |
| docker compose -f dev/compose.yaml --profile ${{ matrix.target.profile }} \ | |
| logs --no-color --timestamps "$svc" \ | |
| > "${{ github.workspace }}/.test-results/container-logs/${svc}.log" 2>&1 || true | |
| done | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: test-results-${{ matrix.target.name }} | |
| include-hidden-files: true | |
| path: ${{ github.workspace }}/.test-results/ | |
| if-no-files-found: warn | |
| - name: Generate test summary | |
| if: always() | |
| run: | | |
| REPORT=${{ github.workspace }}/.test-results/${{ matrix.target.name }}-report.json | |
| ANALYSIS=${{ github.workspace }}/.test-results/${{ matrix.target.name }}-analysis.md | |
| if [ -f "$REPORT" ]; then | |
| # Emit GitHub-flavored markdown (tables + collapsible <details>) and | |
| # write it unfenced so the step summary renders it, rather than | |
| # dumping preformatted text inside a code block. | |
| python -m documentdb_tests.compatibility.result_analyzer -i "$REPORT" -o "$ANALYSIS" -f markdown || true | |
| echo "## ${{ matrix.target.name }} Test Results" >> $GITHUB_STEP_SUMMARY | |
| cat "$ANALYSIS" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| # Crash tests kill the server, so each runs in isolation in its own job (one | |
| # job per intended target/test pair, each starting its own server). A pair is | |
| # "intended" when the test is not deselected against that target by its | |
| # requires(...) markers, so a test that does not apply to a target's topology | |
| # produces no job at all. Discovery brings each target up and collects against | |
| # it, so the intended set (and the test ids) match what the run will see. | |
| discover-xcrash: | |
| name: Collect crash tests | |
| needs: discover-targets | |
| runs-on: ubuntu-latest | |
| outputs: | |
| pairs: ${{ steps.collect.outputs.pairs }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: pip install -r requirements.txt | |
| - name: Enumerate intended target/test pairs | |
| id: collect | |
| env: | |
| TARGETS: ${{ needs.discover-targets.outputs.targets }} | |
| run: | | |
| # For each target, bring its server up and collect the crash tests | |
| # that are not deselected by their requires(...) markers against that | |
| # target (zero-config discovery resolves the live topology, exactly as | |
| # the run will). Collection runs as its own command so a failure fails | |
| # the job. The result is one matrix entry per (target, test) pair, | |
| # carrying the full target. Each target is torn down before the next | |
| # so only one topology is live at a time. | |
| PAIRS='[]' | |
| for row in $(echo "$TARGETS" | jq -c '.[]'); do | |
| PROFILE=$(echo "$row" | jq -r .profile) | |
| dev/compose-up.sh "$PROFILE" | |
| # Bring a replica-set target up to a writable primary so topology | |
| # detection classifies it correctly (collection does not initiate). | |
| python -m documentdb_tests.framework.engine_registry | |
| # Collect the crash tests not deselected against this target. No | |
| # crash test applying to a target is a valid outcome: pytest then | |
| # exits 5 (no tests collected), which is not an error here. Any | |
| # other non-zero exit is a real collection failure and fails the job. | |
| set +e | |
| pytest documentdb_tests/compatibility/tests \ | |
| --collect-only -m engine_xcrash --run-crash-tests -q > collect.out | |
| STATUS=$? | |
| set -e | |
| docker compose -f dev/compose.yaml --profile "$PROFILE" down | |
| if [ "$STATUS" -ne 0 ] && [ "$STATUS" -ne 5 ]; then | |
| cat collect.out | |
| echo "::error::Collecting crash tests for $PROFILE failed (exit $STATUS)" | |
| exit 1 | |
| fi | |
| TESTS=$(grep '<Function' collect.out \ | |
| | sed 's/.*<Function \(.*\)>/\1/' \ | |
| | jq -R -s -c 'split("\n") | map(select(length > 0))') | |
| PAIRS=$(jq -c \ | |
| --argjson target "$row" \ | |
| --argjson tests "$TESTS" \ | |
| '. + [$tests[] | { target: $target, test: . }]' \ | |
| <<< "$PAIRS") | |
| done | |
| echo "pairs=$PAIRS" >> "$GITHUB_OUTPUT" | |
| test-xcrash: | |
| name: "Crash Test: ${{ matrix.pair.target.name }} / ${{ matrix.pair.test }}" | |
| needs: discover-xcrash | |
| if: needs.discover-xcrash.outputs.pairs != '[]' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| pair: ${{ fromJson(needs.discover-xcrash.outputs.pairs) }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: pip install -r requirements.txt | |
| - name: Start ${{ matrix.pair.target.name }} target | |
| run: dev/compose-up.sh "${{ matrix.pair.target.profile }}" | |
| - name: "Run: ${{ matrix.pair.test }} against ${{ matrix.pair.target.name }}" | |
| run: | | |
| # The pair survived discovery, so the test is intended to run against | |
| # this target. --run-crash-tests lets it execute (it is skipped by | |
| # default); zero-config discovery resolves the same live target as in | |
| # the discovery step, so the test id is stable and always selected. | |
| # | |
| # Outcomes: | |
| # - exit 5 (no tests selected): discovery and run disagree, a | |
| # harness bug, not a crash -> fail the job. | |
| # - exit 0 (the test ran and passed): the server no longer crashes, | |
| # the bug is fixed and the engine_xcrash marker should be removed | |
| # -> fail the job. | |
| # - any other non-zero exit: the server crashed (expected) | |
| # -> success. | |
| set +e | |
| pytest documentdb_tests/compatibility/tests \ | |
| -m engine_xcrash -k "${{ matrix.pair.test }}" \ | |
| --run-crash-tests \ | |
| --timeout=10 -v | |
| STATUS=$? | |
| set -e | |
| if [ "$STATUS" -eq 5 ]; then | |
| echo "::error::No tests selected for ${{ matrix.pair.test }} on ${{ matrix.pair.target.name }} - discovery and run disagree" | |
| exit 1 | |
| elif [ "$STATUS" -eq 0 ]; then | |
| echo "::error::Test passed unexpectedly on ${{ matrix.pair.target.name }} - server bug may be fixed, remove engine_xcrash marker" | |
| exit 1 | |
| fi |