diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml new file mode 100644 index 0000000000000..aa17cde7c6237 --- /dev/null +++ b/.github/workflows/plugin-compatibility.yml @@ -0,0 +1,312 @@ +## +# Confirms that the most popular plugins in the WordPress.org directory can be activated against a version of +# WordPress without fataling. +# +# Core's test suites cover core itself, but nothing checks that a new version of WordPress can still boot with +# popular plugins active. When a plugin's assumptions about core stop holding, the result is a fatal error on +# every request and a white screen for real sites. This workflow is a smoke test for that class of failure, so +# that it can be found while there is still time to fix core or notify the plugin author. +# +# The plugin list is fetched from the WordPress.org API at run time and is sharded across a matrix sized to the +# number of plugins being tested. Each plugin is installed and activated on its own, so one broken plugin cannot +# hide another. +# +# It runs weekly against nightly, and can be dispatched manually against any version with any number of plugins, +# which is the intended way to use it as part of the pre-release checklist: point it at the beta or RC and give +# it a count. +# +# A dispatched run can also name the plugins to test instead of taking them from the directory, which is how to +# check a single plugin against a release candidate, or to reproduce a failure from an earlier run without +# waiting for the other plugins in its shard. +# +# The checks each shard runs live in `tools/plugin-compatibility/test-plugins.sh`. Running that script from a +# terminal runs exactly what a shard here runs, which is the quickest way to reproduce a result from a run +# without waiting on Actions. +# +# This workflow tests packaged builds downloaded from WordPress.org: a release, a beta or RC, or the nightly +# build of trunk. It never tests the code in this checkout, so a change to core is only tested here once it +# has reached a nightly build. +## +name: Plugin Compatibility Tests + +on: + push: + branches: + - trunk + # Always test the workflow after it's updated. + paths: + - '.github/workflows/plugin-compatibility.yml' + - '.github/workflows/reusable-plugin-compatibility.yml' + - 'tools/plugin-compatibility/**' + pull_request: + # This workflow is only meant to run from trunk. Pull requests changing this file with different BASE branches should be ignored. + branches: + - trunk + # Always test the workflow when changes are suggested. + paths: + - '.github/workflows/plugin-compatibility.yml' + - '.github/workflows/reusable-plugin-compatibility.yml' + - 'tools/plugin-compatibility/**' + schedule: + - cron: '0 2 * * 1' + workflow_dispatch: + inputs: + wp-version: + description: 'The version of WordPress to test plugins against. Accepts "latest", "nightly", or a specific version number such as a beta or RC, for a pre-release check.' + type: string + default: 'nightly' + plugin-count: + description: 'Number of most popular plugins to test. Accepts any number from 1 to 1000, eg. 10, 50, or 250. Ignored when plugin-slugs is given.' + type: string + default: '100' + plugin-slugs: + description: 'Optional. Test these WordPress.org plugin slugs instead of the most popular ones. Accepts a comma or space separated list, eg. "woocommerce, classic-editor".' + type: string + default: '' + +# Cancels all previous workflow runs for pull requests that have not completed. +concurrency: + # The concurrency group contains the workflow name and the branch name for pull requests + # or the commit hash for any other events. + # + # Dispatched runs are grouped by run instead, so that testing one plugin list does not cancel a run someone + # else started with a different one against the same version of WordPress. + group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Builds the list of plugins to test and splits it into shards for the test matrix. + # + # The list is fetched at run time so that it never goes stale, and it is ordered by popularity so that a + # smaller count still tests the plugins with the widest reach. A dispatched run can name the plugins to test + # instead, in which case the directory is not queried at all. + # + # Performs the following steps: + # - Queries the WordPress.org plugin directory API for the most popular plugins, or reads the plugin-slugs input. + # - Splits the resulting slugs into shards and returns them as a job output. + build-plugin-matrix: + name: Build plugin matrix + permissions: + contents: read + runs-on: ubuntu-24.04 + # Dispatched runs are allowed anywhere so that this can be run from a fork, which is the only way for + # someone without write access to the repository to try it against a plugin of their choosing. + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'workflow_dispatch' }} + timeout-minutes: 5 + outputs: + shards: ${{ steps.plugin-shards.outputs.shards }} + + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} + persist-credentials: false + sparse-checkout: tools/plugin-compatibility + sparse-checkout-cone-mode: false + + - name: Build the list of plugins to test + id: plugin-shards + env: + # An explicit list of slugs to test in place of the most popular ones. Empty for everything but a + # dispatched run that filled the input in. + PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} + # Runs that carry no inputs fall back to the defaults. Pull request and push runs exist to test + # this workflow rather than the ecosystem, so they use a small count. See the note on the + # plugin-compatibility-tests job below. + PLUGIN_COUNT: ${{ inputs.plugin-count || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && '10' ) || '100' }} + # Anything higher would take longer than the 20 minute wall time this is meant to fit inside. The + # directory holds roughly 66,000 plugins, so this is a guard against a typo, not a real limit. + MAX_PLUGIN_COUNT: '1000' + # Shards are sized rather than counted, so that a run of 10 does not spin up 5 near empty jobs and a + # run of 250 is not squeezed into the same 5. + TARGET_PER_SHARD: '25' + MAX_SHARDS: '10' + run: | + set -euo pipefail + + RAW_SLUGS="${RUNNER_TEMP}/slugs-raw.txt" + DEDUPED_SLUGS="${RUNNER_TEMP}/slugs.txt" + : > "${RAW_SLUGS}" + + if [ -n "${PLUGIN_SLUGS}" ]; then + # A named list replaces the popularity query entirely. Commas and whitespace both separate slugs, + # so a list can be pasted in however it was written down. + printf '%s' "${PLUGIN_SLUGS}" | tr ',[:space:]' '\n' > "${RAW_SLUGS}" + awk '!seen[$0]++ && NF > 0' "${RAW_SLUGS}" > "${DEDUPED_SLUGS}" + + # A value outside the character set the directory uses is a typo worth stopping for now, rather + # than a plugin reported as impossible to download several minutes from now. + if INVALID_SLUGS="$( grep -Ev '^[a-z0-9-]+$' "${DEDUPED_SLUGS}" )"; then + printf 'The plugin-slugs input contains values that are not WordPress.org plugin slugs:\n%s\n' "${INVALID_SLUGS}" + exit 1 + fi + + PLUGIN_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" + + if [ "${PLUGIN_COUNT}" -lt 1 ]; then + printf 'The plugin-slugs input did not contain any plugin slugs.\n' + exit 1 + fi + + if [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then + printf 'The plugin-slugs input names %s plugins, more than the %s this workflow will test in one run.\n' "${PLUGIN_COUNT}" "${MAX_PLUGIN_COUNT}" + exit 1 + fi + + printf 'Testing the %s plugin(s) named in the plugin-slugs input.\n' "${PLUGIN_COUNT}" + else + # Guard against a non-numeric value being passed to the API. + if ! printf '%s' "${PLUGIN_COUNT}" | grep -Eq '^[0-9]+$'; then + printf 'The plugin-count input must be a positive integer, got "%s".\n' "${PLUGIN_COUNT}" + exit 1 + fi + + if [ "${PLUGIN_COUNT}" -lt 1 ] || [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then + printf 'The plugin-count input must be between 1 and %s, got %s.\n' "${MAX_PLUGIN_COUNT}" "${PLUGIN_COUNT}" + printf 'Testing more than %s plugins will not fit in the wall time this workflow targets.\n' "${MAX_PLUGIN_COUNT}" + exit 1 + fi + + # The same script a full run on a developer's machine uses, so both test the same plugins. + ./tools/plugin-compatibility/list-plugins.sh --count="${PLUGIN_COUNT}" > "${DEDUPED_SLUGS}" + fi + + SLUGS="$( head -n "${PLUGIN_COUNT}" "${DEDUPED_SLUGS}" | jq -R -s -c 'split( "\n" ) | map( select( . != "" ) )' )" + TOTAL="$( printf '%s' "${SLUGS}" | jq 'length' )" + + if [ "${TOTAL}" -lt 1 ]; then + printf 'The WordPress.org API did not return any plugins.\n' + exit 1 + fi + + if [ "${TOTAL}" -lt "${PLUGIN_COUNT}" ]; then + printf 'Only %s plugins were available, fewer than the %s requested.\n' "${TOTAL}" "${PLUGIN_COUNT}" + fi + + # Aim for TARGET_PER_SHARD plugins in each shard, up to MAX_SHARDS shards. Past that point the + # shards get longer instead of more numerous. + SHARD_COUNT="$( awk -v total="${TOTAL}" -v per="${TARGET_PER_SHARD}" -v max="${MAX_SHARDS}" 'BEGIN { + count = int( ( total + per - 1 ) / per ); + if ( count < 1 ) { count = 1 } + if ( count > max ) { count = max } + print count + }' )" + + # Split the slugs into evenly sized shards, dropping any shard that ends up empty because fewer + # plugins were requested than there are shards. Each shard's slugs are passed to the reusable + # workflow as a JSON string. + SHARDS="$( printf '%s' "${SLUGS}" | jq -c --argjson shard_count "${SHARD_COUNT}" ' + . as $slugs + | ( ( length + $shard_count - 1 ) / $shard_count | floor ) as $size + | [ + range( 0; $shard_count ) + | { index: ( . + 1 ), slugs: $slugs[ ( . * $size ) : ( ( . + 1 ) * $size ) ] } + ] + | map( select( .slugs | length > 0 ) ) + | map( { index: .index, slugs: ( .slugs | @json ) } ) + ' )" + + printf 'Testing %s plugins across %s shard(s).\n' "${TOTAL}" "$( printf '%s' "${SHARDS}" | jq 'length' )" + printf '%s\n' "${SHARDS}" | jq -r '.[] | "Shard \(.index): \(.slugs)"' + + printf 'shards=%s\n' "${SHARDS}" >> "${GITHUB_OUTPUT}" + + # Tests each shard of plugins against the version of WordPress being tested. + # + # Pull request and push runs are here to check that this workflow itself still works, not to report on the + # health of the ecosystem. They use the latest stable release and a small number of plugins, because a genuine + # fatal against nightly is a true result that should not sit as a red check on every future change to these + # two files. Scheduled and manually dispatched runs are the ones that carry the ecosystem signal, and they + # default to nightly and the full count. + plugin-compatibility-tests: + name: WP ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} / Shard ${{ matrix.shard.index }} + uses: ./.github/workflows/reusable-plugin-compatibility.yml + permissions: + contents: read + needs: [ build-plugin-matrix ] + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'workflow_dispatch' }} + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON( needs.build-plugin-matrix.outputs.shards ) }} + with: + wp-version: ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} + php-version: '8.5' + plugin-slugs: ${{ matrix.shard.slugs }} + db-type: 'mysql' + db-version: '9.7' + + # Tests the fixture plugins, each built to break in one specific way, and passes only when every one of them + # gets the result listed for it in tools/plugin-compatibility/fixtures/expected.txt. + # + # The real plugins above can only show that nothing broke. This is what shows that a broken plugin is still + # caught, so a change to the checks that stops catching one fails here instead of quietly passing everything. + # The fixtures are installed from the checkout, so nothing is downloaded but WordPress itself. + fixture-tests: + name: WP ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} / Fixture plugins + uses: ./.github/workflows/reusable-plugin-compatibility.yml + permissions: + contents: read + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'workflow_dispatch' }} + with: + wp-version: ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} + php-version: '8.5' + fixtures: true + db-type: 'mysql' + db-version: '9.7' + + slack-notifications: + name: Slack Notifications + uses: ./.github/workflows/slack-notifications.yml + permissions: + actions: read + contents: read + needs: [ build-plugin-matrix, plugin-compatibility-tests, fixture-tests ] + if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name != 'pull_request' && always() }} + with: + calling_status: ${{ contains( needs.*.result, 'cancelled' ) && 'cancelled' || contains( needs.*.result, 'failure' ) && 'failure' || 'success' }} + secrets: + SLACK_GHA_SUCCESS_WEBHOOK: ${{ secrets.SLACK_GHA_SUCCESS_WEBHOOK }} + SLACK_GHA_CANCELLED_WEBHOOK: ${{ secrets.SLACK_GHA_CANCELLED_WEBHOOK }} + SLACK_GHA_FIXED_WEBHOOK: ${{ secrets.SLACK_GHA_FIXED_WEBHOOK }} + SLACK_GHA_FAILURE_WEBHOOK: ${{ secrets.SLACK_GHA_FAILURE_WEBHOOK }} + + failed-workflow: + name: Failed workflow tasks + runs-on: ubuntu-24.04 + permissions: + actions: write + # `needs` only exposes the result of the jobs listed here, and Slack succeeds whether or not the runs it + # reports on did, so the test jobs are listed too. + needs: [ build-plugin-matrix, plugin-compatibility-tests, fixture-tests, slack-notifications ] + if: | + always() && + github.repository == 'WordPress/wordpress-develop' && + github.event_name != 'pull_request' && + github.run_attempt < 2 && + ( + contains( needs.*.result, 'cancelled' ) || + contains( needs.*.result, 'failure' ) + ) + + steps: + - name: Dispatch workflow run + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + retries: 2 + retry-exempt-status-codes: 418 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'failed-workflow.yml', + ref: 'trunk', + inputs: { + run_id: `${context.runId}`, + } + }); diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml new file mode 100644 index 0000000000000..3f7c56a7e187d --- /dev/null +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -0,0 +1,137 @@ +## +# A reusable workflow that installs a version of WordPress and then checks that a list of plugins can be +# activated against it without fataling. +# +# Each plugin in the `plugin-slugs` shard is tested on its own: it is installed, activated, exercised, and then +# the site is reset to the fresh install before the next one is installed. This keeps one broken plugin from +# masking (or breaking) the next. +# +# The exception is a plugin that declares a `Requires Plugins` dependency, which core will not activate until +# that dependency is installed and active. Those dependencies are installed and activated alongside it, and the +# site is checked with only the dependencies active first so that a dependency's own breakage is not reported +# against the plugin under test. +# +# The checks themselves live in `tools/plugin-compatibility/test-plugins.sh` rather than in this file, so that +# the same code can be run from a terminal to reproduce a result from a run here. See the comments at the top +# of that script. +## +name: Plugin Compatibility Tests + +on: + workflow_call: + inputs: + wp-version: + description: 'The version of WordPress to test against. Accepts a version number, "latest", or "nightly".' + required: false + type: 'string' + default: 'nightly' + php-version: + description: 'The version of PHP to use. Expected format: X.Y.' + required: false + type: 'string' + default: '8.5' + plugin-slugs: + description: 'A JSON array of WordPress.org plugin slugs to test in this shard. Ignored when fixtures is true.' + required: false + type: 'string' + default: '[]' + fixtures: + description: 'Test the fixture plugins in tools/plugin-compatibility/fixtures, which are built to break, and pass only when each gets the result expected for it.' + required: false + type: 'boolean' + default: false + db-type: + description: 'Database type. Valid types are mysql and mariadb.' + required: false + type: 'string' + default: 'mysql' + db-version: + description: 'Database version.' + required: false + type: 'string' + default: '9.7' + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Tests that each plugin in the shard can be activated against the given version of WordPress. + # + # Performs the following steps: + # - Checks out the repository, for the script that runs the checks. + # - Sets up PHP and WP-CLI. + # - Runs the checks against every plugin in the shard. + plugin-compatibility-tests: + name: PHP ${{ inputs.php-version }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }} + permissions: + contents: read + runs-on: ubuntu-24.04 + # Installing a plugin's dependencies before testing it adds a download and an activation for each + # one, and a dependency the size of WooCommerce is not quick to activate. + timeout-minutes: 45 + + services: + database: + # The database type and version are inputs so that this workflow can be pointed at any supported + # combination, which means the image cannot be pinned to a digest. This matches how the database + # service is declared in install-testing.yml and reusable-upgrade-testing.yml. + image: ${{ inputs.db-type }}:${{ inputs.db-version }} # zizmor: ignore[unpinned-images] + ports: + - 3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval="30s" + --health-timeout="10s" + --health-retries="5" + -e MYSQL_ROOT_PASSWORD="root" + -e MYSQL_DATABASE="test_db" + + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} + persist-credentials: false + sparse-checkout: tools/plugin-compatibility + sparse-checkout-cone-mode: false + + - name: Set up PHP ${{ inputs.php-version }} + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 + with: + php-version: '${{ inputs.php-version }}' + coverage: none + tools: wp-cli + + # The runner already provides PHP, WP-CLI and a database, so the checks run here rather than in the + # containers the script provisions when it is run from a terminal. + # + # WordPress is installed outside the workspace so that the checkout above is not part of the site under + # test. + - name: Test each plugin in isolation + env: + PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} + FIXTURES: ${{ inputs.fixtures }} + WP_VERSION: ${{ inputs.wp-version }} + DB_PORT: ${{ job.services.database.ports['3306'] }} + run: | + set -uo pipefail + + if [ "${FIXTURES}" = "true" ]; then + SELECTION=( + --fixtures=./tools/plugin-compatibility/fixtures + --expect=./tools/plugin-compatibility/fixtures/expected.txt + ) + else + SELECTION=( --plugins="$( printf '%s' "${PLUGIN_SLUGS}" | jq -r 'join( " " )' )" ) + fi + + ./tools/plugin-compatibility/test-plugins.sh \ + --no-docker \ + --wp-version="${WP_VERSION}" \ + --dir="${RUNNER_TEMP}/wordpress" \ + --db-host="127.0.0.1:${DB_PORT}" \ + --db-name=test_db \ + --db-user=root \ + --db-pass=root \ + "${SELECTION[@]}" diff --git a/tools/plugin-compatibility/README.md b/tools/plugin-compatibility/README.md new file mode 100644 index 0000000000000..a6b6994d77db3 --- /dev/null +++ b/tools/plugin-compatibility/README.md @@ -0,0 +1,60 @@ +# Plugin compatibility tests + +Checks that plugins from the WordPress.org directory can be activated against a version of WordPress without fataling. The Plugin Compatibility Tests workflow runs these scripts against the most popular plugins. The same scripts run locally, which is how to test far more plugins than CI should. + +| Script | What it does | +| --- | --- | +| `test-plugins.sh` | Tests a list of plugins against one version of WordPress. This is what each workflow shard runs. | +| `list-plugins.sh` | Prints plugin slugs from the directory: the most popular, all of them, or those updated since a date. | +| `compare-results.sh` | Compares a baseline run with a candidate run and reports the plugins that regressed. | +| `test-all-plugins.sh` | Runs all of the above for a large list, in parallel, resumable. | +| `fixtures/` | Plugins built to break in one specific way each, with the result each must get. | + +All of them need Docker. The list scripts also need curl and jq. + +## Comparing against a baseline + +Across the whole directory, thousands of plugins fail on a current PHP whatever version of WordPress they run on. A list of failures from one run is mostly noise. What matters before a release is the plugins that **pass on the current release and fail on the next one**. + +So a large run tests every plugin twice, and compares: + +``` +plugins.txt ──┬── test-plugins.sh --wp-version=latest ──► latest.tsv ──┐ + │ ├── compare-results.sh ──► report.md + └── test-plugins.sh --wp-version=7.2-RC1 ──► 7.2-RC1.tsv ──┘ +``` + +The report lists plugins that regressed (passed on the baseline, failed on the candidate) and plugins the candidate fixed. Plugins that fail on both are counted but not listed. A plugin whose version changed between the two runs is flagged, since the plugin changed too. + +## Testing the whole directory + +``` +# The 5,000 most popular plugins, 8 at a time, against 7.2-RC1 compared with the current release. +tools/plugin-compatibility/test-all-plugins.sh --count=5000 --wp-version=7.2-RC1 --jobs=8 + +# Every plugin updated in the last two years. +tools/plugin-compatibility/test-all-plugins.sh --updated-since=2024-11-01 --wp-version=7.2-RC1 --jobs=8 + +# Everything. +tools/plugin-compatibility/test-all-plugins.sh --all --wp-version=7.2-RC1 --jobs=8 +``` + +Everything is written to `plugin-compatibility-results/`: the plugin list, a results file and log per copy, the download cache, and `report.md`. If the run is interrupted, run the same command again to carry on where it stopped. Pass an exact version rather than `nightly` for a run that may be resumed on another day, because nightly moves. + +Rough sizes, at the roughly 7 seconds per plugin measured on the most popular plugins. Smaller plugins are likely quicker. + +| List | Plugins | One version, 8 jobs | Baseline and candidate, 8 jobs | +| --- | --- | --- | --- | +| `--count=1000` | 1,000 | 15 minutes | 30 minutes | +| `--updated-since` two years ago | about 42,000 | 10 hours | 20 hours | +| `--all` | about 78,000 | 19 hours | 38 hours | + +Each job runs its own database and PHP containers, so memory is usually what limits `--jobs`. + +## Testing a few plugins + +``` +tools/plugin-compatibility/test-plugins.sh --wp-version=nightly woocommerce classic-editor +``` + +Run `test-plugins.sh --help` for every option, including `--results` and `--cache-dir` for long runs and `--no-docker` to use a PHP, WP-CLI and database you already have. diff --git a/tools/plugin-compatibility/compare-results.sh b/tools/plugin-compatibility/compare-results.sh new file mode 100755 index 0000000000000..8a2a1d4fcbc96 --- /dev/null +++ b/tools/plugin-compatibility/compare-results.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +## +# Compares two plugin compatibility runs and prints the plugins whose result changed between them. +# +# Across the whole directory, thousands of plugins fail on a current PHP whatever version of WordPress they +# run on, so the list of failures from a single run is mostly noise. What matters before a release is the +# plugins that pass on the current release and fail on the next one. This finds those by comparing the +# results files `test-plugins.sh --results` writes for each version: +# +# tools/plugin-compatibility/compare-results.sh latest.tsv nightly.tsv +# +# Prints a markdown report, and exits 1 when at least one plugin regressed. +## + +set -uo pipefail + +usage() { + cat <<'USAGE_EOF' +Usage: compare-results.sh + +Compares two results files written by test-plugins.sh --results, such as one run against the current +release and one against nightly, and prints a markdown report of what changed. + + Regressed passed on the baseline, failed on the candidate. These are the ones to look at. + Fixed failed on the baseline, passed on the candidate. + Fail on both failed either way, so not caused by the candidate. Counted, not listed. + +Plugins that were not tested in both runs, or were skipped in either, are counted but not compared. A +regressed plugin whose version differs between the runs is flagged, since the plugin changed as well. + +Exits 1 when a plugin regressed and 0 otherwise. +USAGE_EOF +} + +case "${1:-}" in + -h | --help ) usage; exit 0 ;; +esac + +if [ "$#" -ne 2 ]; then + usage >&2 + exit 2 +fi + +BASELINE="${1}" +CANDIDATE="${2}" + +for FILE in "${BASELINE}" "${CANDIDATE}"; do + [ -s "${FILE}" ] || { printf 'Error: %s does not exist or is empty.\n' "${FILE}" >&2; exit 2; } +done + +# One row per plugin that is in both files: slug, baseline version, baseline result, candidate version, +# candidate result, candidate reason. A plugin listed twice in a file keeps its last row, which is the one a +# resumed run recorded. +JOINED="$( awk -F '\t' -v OFS='\t' ' + NR == FNR { version[$1] = $2; status[$1] = $3; next } + { cand_version[$1] = $2; cand_status[$1] = $3; cand_reason[$1] = $5 } + END { + for ( slug in cand_status ) { + if ( slug in status ) { + print slug, version[slug], status[slug], cand_version[slug], cand_status[slug], cand_reason[slug] + } + } + } +' "${BASELINE}" "${CANDIDATE}" | sort )" + +BASELINE_TOTAL="$( cut -f 1 "${BASELINE}" | sort -u | grep -c . )" +CANDIDATE_TOTAL="$( cut -f 1 "${CANDIDATE}" | sort -u | grep -c . )" +BOTH_TOTAL="$( printf '%s' "${JOINED}" | grep -c . )" + +REGRESSED="$( printf '%s\n' "${JOINED}" | awk -F '\t' '$3 == "PASS" && $5 == "FAIL"' )" +FIXED="$( printf '%s\n' "${JOINED}" | awk -F '\t' '$3 == "FAIL" && $5 == "PASS"' )" +REGRESSED_COUNT="$( printf '%s' "${REGRESSED}" | grep -c . )" +FIXED_COUNT="$( printf '%s' "${FIXED}" | grep -c . )" +BOTH_FAIL_COUNT="$( printf '%s\n' "${JOINED}" | awk -F '\t' '$3 == "FAIL" && $5 == "FAIL" { n++ } END { print n + 0 }' )" +BOTH_PASS_COUNT="$( printf '%s\n' "${JOINED}" | awk -F '\t' '$3 == "PASS" && $5 == "PASS" { n++ } END { print n + 0 }' )" +NOT_COMPARED_COUNT="$(( BOTH_TOTAL - REGRESSED_COUNT - FIXED_COUNT - BOTH_FAIL_COUNT - BOTH_PASS_COUNT ))" + +printf '## Plugin compatibility: %s compared with %s\n\n' "$( basename "${CANDIDATE}" )" "$( basename "${BASELINE}" )" +printf '%s plugins in the baseline, %s in the candidate, %s in both.\n\n' "${BASELINE_TOTAL}" "${CANDIDATE_TOTAL}" "${BOTH_TOTAL}" +printf '| Regressed | Fixed | Pass on both | Fail on both | Skipped in either |\n' +printf '| --- | --- | --- | --- | --- |\n' +printf '| %s | %s | %s | %s | %s |\n\n' "${REGRESSED_COUNT}" "${FIXED_COUNT}" "${BOTH_PASS_COUNT}" "${BOTH_FAIL_COUNT}" "${NOT_COMPARED_COUNT}" + +print_rows() { + printf '| Plugin | Version | Details |\n' + printf '| --- | --- | --- |\n' + + printf '%s\n' "${1}" | awk -F '\t' 'NF > 0 { + version = ( $2 == $4 ) ? $4 : $2 " to " $4 " (plugin changed too)" + details = ( $6 == "-" ) ? "" : $6 + gsub( /\|/, "\\|", details ) + printf "| [%s](https://wordpress.org/plugins/%s/) | %s | %s |\n", $1, $1, version, details + }' + + printf '\n' +} + +if [ "${REGRESSED_COUNT}" -gt 0 ]; then + printf '### Regressed\n\nPassed on the baseline and failed on the candidate.\n\n' + print_rows "${REGRESSED}" +else + printf 'No plugin that passed on the baseline failed on the candidate.\n\n' +fi + +if [ "${FIXED_COUNT}" -gt 0 ]; then + printf '### Fixed\n\nFailed on the baseline and passed on the candidate.\n\n' + print_rows "${FIXED}" +fi + +[ "${REGRESSED_COUNT}" -eq 0 ] diff --git a/tools/plugin-compatibility/fixtures/expected.txt b/tools/plugin-compatibility/fixtures/expected.txt new file mode 100644 index 0000000000000..6a3719674ba10 --- /dev/null +++ b/tools/plugin-compatibility/fixtures/expected.txt @@ -0,0 +1,31 @@ +# The result each fixture plugin is expected to get from test-plugins.sh, in the order they are tested. +# +# test-plugins.sh --fixtures=tools/plugin-compatibility/fixtures --expect=tools/plugin-compatibility/fixtures/expected.txt +# +# Each fixture is a plugin built to break in one specific way, so that a change to the checks which stops a +# broken plugin from being caught fails a run instead of quietly passing everything. +# +# Format: [text the "Also active" or "Details" column must contain] + +fixture-healthy PASS +fixture-fatal-on-load FAIL The plugin could not be activated +fixture-fatal-on-activation FAIL The plugin could not be activated +fixture-exit-on-activation SKIPPED the plugin was not active afterwards +fixture-breaks-site-on-activation FAIL The request to / returned HTTP 500 +fixture-requires-newer-php SKIPPED declared requirements are not met +fixture-requires-newer-wordpress SKIPPED declared requirements are not met +fixture-requires-newer-both SKIPPED declared requirements are not met +fixture-fatal-on-front-end FAIL The request to / returned HTTP 500 +fixture-fatal-on-login FAIL The request to /wp-login.php returned HTTP 500 +fixture-fatal-after-output FAIL PHP Fatal error +fixture-parse-error-after-output FAIL PHP Parse error +fixture-fatal-in-background-request FAIL PHP Fatal error +fixture-wp-cli-only-fatal PASS WP-CLI fatals when it loads WordPress +fixture-requires-akismet PASS akismet +fixture-requires-broken-dependency SKIPPED The required plugins are not healthy on their own +fixture-requires-wp-cli-only-fatal SKIPPED WP-CLI cannot load WordPress with the required plugins active + +# These two must stay together and in this order: the second fatals if anything the first wrote survived the +# reset between plugins. +fixture-leaves-files-behind PASS +fixture-checks-for-leftovers PASS diff --git a/tools/plugin-compatibility/fixtures/fixture-breaks-site-on-activation/fixture-breaks-site-on-activation.php b/tools/plugin-compatibility/fixtures/fixture-breaks-site-on-activation/fixture-breaks-site-on-activation.php new file mode 100644 index 0000000000000..21d7901478d9b --- /dev/null +++ b/tools/plugin-compatibility/fixtures/fixture-breaks-site-on-activation/fixture-breaks-site-on-activation.php @@ -0,0 +1,14 @@ +get_var( "SHOW TABLES LIKE '{$wpdb->prefix}fixture_leftover'" ) ) { + fixture_function_that_does_not_exist(); + } + } +); diff --git a/tools/plugin-compatibility/fixtures/fixture-exit-on-activation/fixture-exit-on-activation.php b/tools/plugin-compatibility/fixtures/fixture-exit-on-activation/fixture-exit-on-activation.php new file mode 100644 index 0000000000000..cd4cbbfdb6d8c --- /dev/null +++ b/tools/plugin-compatibility/fixtures/fixture-exit-on-activation/fixture-exit-on-activation.php @@ -0,0 +1,13 @@ + false, + 'timeout' => 0.01, + ) + ); + } +); + +add_action( + 'wp_ajax_nopriv_fixture_background_task', + static function () { + ignore_user_abort( true ); + sleep( 3 ); + fixture_function_that_does_not_exist(); + } +); diff --git a/tools/plugin-compatibility/fixtures/fixture-fatal-on-activation/fixture-fatal-on-activation.php b/tools/plugin-compatibility/fixtures/fixture-fatal-on-activation/fixture-fatal-on-activation.php new file mode 100644 index 0000000000000..276e67d09e62d --- /dev/null +++ b/tools/plugin-compatibility/fixtures/fixture-fatal-on-activation/fixture-fatal-on-activation.php @@ -0,0 +1,8 @@ +query( "CREATE TABLE IF NOT EXISTS {$wpdb->prefix}fixture_leftover ( id INT )" ); + update_option( 'fixture_leftover', 'left behind' ); + } +); diff --git a/tools/plugin-compatibility/fixtures/fixture-parse-error-after-output/fixture-parse-error-after-output.php b/tools/plugin-compatibility/fixtures/fixture-parse-error-after-output/fixture-parse-error-after-output.php new file mode 100644 index 0000000000000..d6a9db897cc96 --- /dev/null +++ b/tools/plugin-compatibility/fixtures/fixture-parse-error-after-output/fixture-parse-error-after-output.php @@ -0,0 +1,19 @@ + plugins.txt +# tools/plugin-compatibility/list-plugins.sh --all > plugins.txt +# tools/plugin-compatibility/list-plugins.sh --updated-since=2024-11-01 > plugins.txt +# +# Requires curl and jq. +## + +set -uo pipefail + +COUNT="" +UPDATED_SINCE="" +# The API caps `per_page` at 250 and returns 250 without complaint for anything larger. +PAGE_SIZE=250 + +usage() { + cat <<'USAGE_EOF' +Usage: list-plugins.sh --count= | --all | --updated-since= [--count=] + +Prints WordPress.org plugin slugs, one per line. + +Options: + --count= How many plugins to list. + --all List every plugin in the directory, roughly 78,000 of them, most recently + updated first. + --updated-since= List the plugins updated on or after , written YYYY-MM-DD, most recently + updated first. Combine with --count to cap the list. A plugin nobody has updated + in years is likely to fail on a current PHP whatever version of WordPress it + runs on, so this is the more useful list once the most popular are covered. + -h, --help Print this message. +USAGE_EOF +} + +error() { + printf 'Error: %s\n' "${1}" >&2 + exit 1 +} + +while [ "$#" -gt 0 ]; do + case "${1}" in + --count=* ) COUNT="${1#*=}" ;; + --all ) COUNT="all" ;; + --updated-since=* ) UPDATED_SINCE="${1#*=}" ;; + -h | --help ) usage; exit 0 ;; + * ) usage >&2; error "Unknown option ${1}." ;; + esac + shift +done + +if [ -n "${UPDATED_SINCE}" ]; then + printf '%s' "${UPDATED_SINCE}" | grep -Eq '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' \ + || error "--updated-since must be a date written YYYY-MM-DD, got \"${UPDATED_SINCE}\"." + COUNT="${COUNT:-all}" + BROWSE="updated" +elif [ "${COUNT}" = "all" ]; then + # Popularity is a poor order to page through all of: tens of thousands of plugins share the same rounded + # install count, their order shifts from one page to the next, and paging to the end returns only about + # 57,000 different plugins. Sorted by update, paging returns all of them, about 78,000. + BROWSE="updated" +else + BROWSE="popular" +fi + +[ -n "${COUNT}" ] || { usage >&2; error "Pass --count, --all or --updated-since."; } + +if [ "${COUNT}" != "all" ] && ! printf '%s' "${COUNT}" | grep -Eq '^[1-9][0-9]*$'; then + error "--count must be a positive integer, got \"${COUNT}\"." +fi + +for REQUIRED_COMMAND in curl jq; do + command -v "${REQUIRED_COMMAND}" > /dev/null 2>&1 || error "${REQUIRED_COMMAND} is required." +done + +PAGE_FILE="$( mktemp )" +RAW_FILE="$( mktemp )" +trap 'rm -f "${PAGE_FILE}" "${RAW_FILE}"' EXIT + +PAGE=1 +UNIQUE_COUNT=0 + +while : ; do + # The unneeded response fields are turned off to keep the payload small. + curl -sS --fail --retry 3 --retry-delay 5 \ + --get 'https://api.wordpress.org/plugins/info/1.2/' \ + --data-urlencode 'action=query_plugins' \ + --data-urlencode "request[browse]=${BROWSE}" \ + --data-urlencode "request[per_page]=${PAGE_SIZE}" \ + --data-urlencode "request[page]=${PAGE}" \ + --data-urlencode 'request[fields][short_description]=0' \ + --data-urlencode 'request[fields][sections]=0' \ + --data-urlencode 'request[fields][icons]=0' \ + --data-urlencode 'request[fields][banners]=0' \ + --data-urlencode 'request[fields][ratings]=0' \ + --data-urlencode 'request[fields][tags]=0' \ + --data-urlencode 'request[fields][compatibility]=0' \ + --data-urlencode 'request[fields][screenshots]=0' \ + -o "${PAGE_FILE}" || error "Page ${PAGE} could not be fetched from the WordPress.org API." + + PAGE_COUNT="$( jq '.plugins | length' "${PAGE_FILE}" )" || error "Page ${PAGE} was not valid JSON." + + # `last_updated` reads like "2024-11-25 3:02pm GMT", so its first ten characters compare as a date. + jq -r --arg since "${UPDATED_SINCE}" \ + '.plugins[] | select( $since == "" or ( .last_updated // "" )[0:10] >= $since ) | .slug // empty' \ + "${PAGE_FILE}" >> "${RAW_FILE}" + + # Ordering can shift between two requests, so the same slug can turn up on more than one page. Only its + # first appearance counts. + UNIQUE_COUNT="$( awk 'NF > 0 && !seen[$0]++' "${RAW_FILE}" | wc -l | tr -d ' ' )" + + printf 'Page %s returned %s plugins, %s listed so far.\n' "${PAGE}" "${PAGE_COUNT}" "${UNIQUE_COUNT}" >&2 + + if [ "${COUNT}" != "all" ] && [ "${UNIQUE_COUNT}" -ge "${COUNT}" ]; then + break + fi + + # A short page means the directory has nothing left to give. + if [ "${PAGE_COUNT}" -lt "${PAGE_SIZE}" ]; then + break + fi + + # Sorted by update, a page ending before the cutoff means every page after it does too. + if [ -n "${UPDATED_SINCE}" ] \ + && [ "$( jq -r '.plugins[-1].last_updated // ""' "${PAGE_FILE}" | cut -c 1-10 )" \< "${UPDATED_SINCE}" ]; then + break + fi + + PAGE=$(( PAGE + 1 )) +done + +[ "${UNIQUE_COUNT}" -gt 0 ] || error "The WordPress.org API did not return any plugins." + +if [ "${COUNT}" = "all" ]; then + awk 'NF > 0 && !seen[$0]++' "${RAW_FILE}" + exit 0 +fi + +awk 'NF > 0 && !seen[$0]++' "${RAW_FILE}" | head -n "${COUNT}" + +if [ "${UNIQUE_COUNT}" -lt "${COUNT}" ]; then + printf 'Only %s plugins were available, fewer than the %s requested.\n' "${UNIQUE_COUNT}" "${COUNT}" >&2 +fi diff --git a/tools/plugin-compatibility/test-all-plugins.sh b/tools/plugin-compatibility/test-all-plugins.sh new file mode 100755 index 0000000000000..3f33ae120d069 --- /dev/null +++ b/tools/plugin-compatibility/test-all-plugins.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +## +# Tests a large list of plugins - up to the whole WordPress.org directory - against a baseline and a candidate +# version of WordPress, and reports the plugins that pass on the baseline and fail on the candidate. +# +# This is the run to make ahead of a major or security release, on a developer's machine or a spare server +# rather than in Actions: the whole directory is roughly 78,000 plugins, which at about 7 seconds each is far +# more runner time than a shared CI pool should give to one job. +# +# tools/plugin-compatibility/test-all-plugins.sh --count=5000 --wp-version=7.2-RC1 --jobs=8 +# tools/plugin-compatibility/test-all-plugins.sh --updated-since=2024-11-01 --wp-version=7.2-RC1 +# tools/plugin-compatibility/test-all-plugins.sh --all --wp-version=7.2-RC1 --jobs=8 +# +# It splits the list into --jobs parts and runs that many copies of test-plugins.sh side by side, each with +# its own containers, first against the baseline and then against the candidate. Every result is written to +# the output directory as it is recorded, so running the same command again after an interruption carries on +# where it stopped. Downloads are cached there too, so the candidate run does not download every plugin again. +# +# Needs Docker, curl and jq. +## + +set -uo pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null 2>&1 && pwd )" + +LIST_ARGS=() +PLUGINS_FILE="" +WP_VERSION="nightly" +BASELINE_VERSION="latest" +JOBS=4 +OUT_DIR="plugin-compatibility-results" +PASSTHROUGH_ARGS=() +# How many times a part is started again when it stops before testing everything, as long as each attempt +# gets further than the one before. +MAX_ATTEMPTS=3 + +usage() { + cat <<'USAGE_EOF' +Usage: test-all-plugins.sh (--count= | --all | --updated-since= | --plugins-file=) [options] + +Tests many plugins against a baseline and a candidate version of WordPress, and reports the plugins that +pass on the baseline and fail on the candidate. + +Which plugins: + --count= The n most popular plugins. + --all Every plugin in the directory. + --updated-since= Plugins updated on or after , written YYYY-MM-DD. Combine with --count to cap. + --plugins-file= Slugs from a file, one per line. + +Options: + --wp-version= The candidate: the version of WordPress being checked. Pass an exact version such as + 7.2-RC1 for a run that may be resumed on another day, because nightly moves. + Default: nightly. + --baseline= The version to compare with, or "none" to only test the candidate. Default: latest. + --jobs= How many copies to run side by side. Each runs its own database and PHP containers. + Default: 4. + --out= Where the plugin list, results, logs, download cache and report are written. + Default: plugin-compatibility-results. + --php-image= Passed on to test-plugins.sh. + --db-image= Passed on to test-plugins.sh. + --timeout= Passed on to test-plugins.sh. + -h, --help Print this message. + +The report is written to /report.md. Exits 1 when a plugin regressed, and 0 otherwise. +USAGE_EOF +} + +error() { + printf 'Error: %s\n' "${1}" >&2 + exit 1 +} + +while [ "$#" -gt 0 ]; do + case "${1}" in + --count=* | --updated-since=* ) LIST_ARGS=( ${LIST_ARGS[@]+"${LIST_ARGS[@]}"} "${1}" ) ;; + --all ) LIST_ARGS=( ${LIST_ARGS[@]+"${LIST_ARGS[@]}"} --all ) ;; + --plugins-file=* ) PLUGINS_FILE="${1#*=}" ;; + --wp-version=* ) WP_VERSION="${1#*=}" ;; + --baseline=* ) BASELINE_VERSION="${1#*=}" ;; + --jobs=* ) JOBS="${1#*=}" ;; + --out=* ) OUT_DIR="${1#*=}" ;; + --php-image=* | --db-image=* | --timeout=* ) PASSTHROUGH_ARGS=( ${PASSTHROUGH_ARGS[@]+"${PASSTHROUGH_ARGS[@]}"} "${1}" ) ;; + -h | --help ) usage; exit 0 ;; + * ) usage >&2; error "Unknown option ${1}." ;; + esac + shift +done + +printf '%s' "${JOBS}" | grep -Eq '^[1-9][0-9]*$' || error "--jobs must be a positive integer, got \"${JOBS}\"." + +if [ "${BASELINE_VERSION}" = "${WP_VERSION}" ]; then + error "The baseline and the candidate are both ${WP_VERSION}." +fi + +mkdir -p "${OUT_DIR}" || error "The output directory could not be created." +OUT_DIR="$( cd "${OUT_DIR}" > /dev/null 2>&1 && pwd )" +LIST_FILE="${OUT_DIR}/plugins.txt" + +# The list is built once and kept, so that a resumed run, and the candidate run after the baseline, test +# exactly the same plugins even if the directory has changed in the meantime. +if [ -s "${LIST_FILE}" ]; then + printf 'Using the %s plugins already listed in %s.\n' "$( grep -c . "${LIST_FILE}" )" "${LIST_FILE}" +elif [ -n "${PLUGINS_FILE}" ]; then + grep -v '^[[:space:]]*#' "${PLUGINS_FILE}" | grep . > "${LIST_FILE}" || error "No plugins were listed in ${PLUGINS_FILE}." +elif [ "${#LIST_ARGS[@]}" -gt 0 ]; then + "${SCRIPT_DIR}/list-plugins.sh" "${LIST_ARGS[@]}" > "${LIST_FILE}.partial" || error "The plugin list could not be built." + mv "${LIST_FILE}.partial" "${LIST_FILE}" +else + usage >&2 + error "Pass --count, --all, --updated-since or --plugins-file." +fi + +printf 'Testing %s plugins with %s jobs, writing to %s.\n' "$( grep -c . "${LIST_FILE}" )" "${JOBS}" "${OUT_DIR}" + +# Each copy's own interrupt trap cannot be relied on here: a background job in a script ignores SIGINT. So an +# interrupt stops the copies explicitly, and each one cleans up its containers on the way out. +RUNNING_PIDS=() + +# shellcheck disable=SC2329 # Called from the traps below. +stop_jobs() { + if [ "${#RUNNING_PIDS[@]}" -gt 0 ]; then + kill -TERM "${RUNNING_PIDS[@]}" > /dev/null 2>&1 + wait "${RUNNING_PIDS[@]}" > /dev/null 2>&1 + fi +} + +trap 'printf "\nStopping. Run the same command again to carry on.\n" >&2; stop_jobs; exit 130' INT +trap 'stop_jobs; exit 143' TERM + +# remaining_plugins +# +# Prints the listed plugins that have no result in any of the results files in . +remaining_plugins() { + # The leading line keeps the first input from ever being empty, which would make `NR == FNR` true for the + # list as well and leave nothing to test. + { printf '#\n'; cat "${1}"/part-*.tsv 2>/dev/null; } | awk -F '\t' 'NR == FNR { done[$1] = 1; next } NF > 0 && !( $0 in done )' - "${LIST_FILE}" +} + +# run_version +# +# Tests every listed plugin against one version of WordPress, in parallel, and merges the results into +# /.tsv. +run_version() { + VERSION="${1}" + LABEL="$( printf '%s' "${VERSION}" | tr -c 'A-Za-z0-9.-' '_' )" + RESULTS_DIR="${OUT_DIR}/${LABEL}" + mkdir -p "${RESULTS_DIR}" || error "${RESULTS_DIR} could not be created." + + ATTEMPT=1 + REMAINING_COUNT="$( remaining_plugins "${RESULTS_DIR}" | grep -c . )" + + while [ "${REMAINING_COUNT}" -gt 0 ] && [ "${ATTEMPT}" -le "${MAX_ATTEMPTS}" ]; do + printf '\nWordPress %s: %s plugins to test, attempt %s.\n' "${VERSION}" "${REMAINING_COUNT}" "${ATTEMPT}" + + # Dealt out like cards rather than cut into blocks, so that every part gets a share of the large, + # slow plugins at the top of a popularity list. + rm -f "${RESULTS_DIR}"/todo-*.txt + remaining_plugins "${RESULTS_DIR}" | awk -v jobs="${JOBS}" -v dir="${RESULTS_DIR}" '{ print > ( dir "/todo-" ( ( NR - 1 ) % jobs + 1 ) ".txt" ) }' + + RUNNING_PIDS=() + + for TODO_FILE in "${RESULTS_DIR}"/todo-*.txt; do + PART="$( basename "${TODO_FILE}" .txt )" + PART="${PART#todo-}" + + "${SCRIPT_DIR}/test-plugins.sh" \ + --wp-version="${VERSION}" \ + --plugins-file="${TODO_FILE}" \ + --results="${RESULTS_DIR}/part-${PART}.tsv" \ + --cache-dir="${OUT_DIR}/cache" \ + ${PASSTHROUGH_ARGS[@]+"${PASSTHROUGH_ARGS[@]}"} \ + >> "${RESULTS_DIR}/part-${PART}.log" 2>&1 & + RUNNING_PIDS=( ${RUNNING_PIDS[@]+"${RUNNING_PIDS[@]}"} "$!" ) + + # Staggered, so that the copies are not all downloading WordPress into the shared cache at once. + sleep 5 + done + + # The copies exit 1 whenever a plugin fails, which is expected here. Progress is measured by what is + # left, not by exit codes. + wait "${RUNNING_PIDS[@]}" + RUNNING_PIDS=() + rm -f "${RESULTS_DIR}"/todo-*.txt + + PREVIOUS_COUNT="${REMAINING_COUNT}" + REMAINING_COUNT="$( remaining_plugins "${RESULTS_DIR}" | grep -c . )" + + # A copy that stops early, because a plugin left the site in a state it could not be reset from, has + # still recorded that plugin, so starting again gets past it. One that made no progress at all will not. + if [ "${REMAINING_COUNT}" -ge "${PREVIOUS_COUNT}" ]; then + break + fi + + ATTEMPT=$(( ATTEMPT + 1 )) + done + + cat "${RESULTS_DIR}"/part-*.tsv 2>/dev/null > "${OUT_DIR}/${LABEL}.tsv" + + printf 'WordPress %s: %s passed, %s failed, %s skipped. Results in %s.\n' "${VERSION}" \ + "$( awk -F '\t' '$3 == "PASS" { n++ } END { print n + 0 }' "${OUT_DIR}/${LABEL}.tsv" )" \ + "$( awk -F '\t' '$3 == "FAIL" { n++ } END { print n + 0 }' "${OUT_DIR}/${LABEL}.tsv" )" \ + "$( awk -F '\t' '$3 == "SKIPPED" { n++ } END { print n + 0 }' "${OUT_DIR}/${LABEL}.tsv" )" \ + "${OUT_DIR}/${LABEL}.tsv" + + if [ "${REMAINING_COUNT}" -gt 0 ]; then + printf 'Warning: %s plugins have no result against WordPress %s. See the logs in %s.\n' "${REMAINING_COUNT}" "${VERSION}" "${RESULTS_DIR}" >&2 + fi + + MERGED_RESULTS="${OUT_DIR}/${LABEL}.tsv" +} + +if [ "${BASELINE_VERSION}" = "none" ]; then + run_version "${WP_VERSION}" + printf '\nNo baseline was tested, so there is nothing to compare. Failures are listed in %s.\n' "${MERGED_RESULTS}" + awk -F '\t' '$3 == "FAIL" { found = 1 } END { exit found }' "${MERGED_RESULTS}" + exit +fi + +run_version "${BASELINE_VERSION}" +BASELINE_RESULTS="${MERGED_RESULTS}" + +run_version "${WP_VERSION}" +CANDIDATE_RESULTS="${MERGED_RESULTS}" + +printf '\n' +"${SCRIPT_DIR}/compare-results.sh" "${BASELINE_RESULTS}" "${CANDIDATE_RESULTS}" | tee "${OUT_DIR}/report.md" +COMPARE_EXIT_CODE="${PIPESTATUS[0]}" + +printf '\nThe report is in %s.\n' "${OUT_DIR}/report.md" +exit "${COMPARE_EXIT_CODE}" diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh new file mode 100755 index 0000000000000..fa29cc99758be --- /dev/null +++ b/tools/plugin-compatibility/test-plugins.sh @@ -0,0 +1,1205 @@ +#!/usr/bin/env bash +## +# Checks that plugins from the WordPress.org directory can be activated against a version of WordPress +# without fataling. +# +# Each plugin is installed on its own, activated, exercised over HTTP and through WP-CLI, and then the site is +# reset to the fresh install - files and database - before the next one is installed, so that one broken +# plugin cannot mask (or break) the next. A plugin that declares `Requires Plugins` has those dependencies +# installed and activated alongside it, because core refuses to activate it otherwise. +# +# What a real request does is what decides whether a plugin passed. The WP-CLI check runs last and only ever +# adds a note, because WP-CLI loads WordPress in a way a visitor never does. +# +# This is the same code the Plugin Compatibility Tests workflow runs. Running it here is how to reproduce a +# failure from a workflow run, or to check a plugin against a release candidate, without waiting on Actions. +# +# By default everything is provisioned in throwaway containers - a database, and a PHP with WP-CLI to run the +# checks in - so nothing but Docker is needed and nothing is left behind: +# +# tools/plugin-compatibility/test-plugins.sh --wp-version=nightly woocommerce classic-editor +# +# Pass `--no-docker` to run the checks in the current environment instead, against a database that is already +# running. That is the path the workflow takes, where the runner already has PHP, WP-CLI and a database +# service. +# +# Usage: test-plugins.sh [options] [slug...] +# +# Run `test-plugins.sh --help` for the options. +## + +# `set -e` is deliberately not used: a plugin that fatals must not stop the remaining plugins from being +# tested. Failures are checked where they matter instead. +set -uo pipefail + +# Written for Bash 3.2 so that it runs on a stock macOS as well as on a runner. + +SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null 2>&1 && pwd )/$( basename "${BASH_SOURCE[0]}" )" + +# Defaults. Every one of these can be overridden with the matching option. +WP_VERSION="nightly" +PLUGIN_LIST="" +USE_DOCKER="yes" +# The official WP-CLI image carries PHP, WP-CLI, `mysqli` and curl, which is everything the checks need. +PHP_IMAGE="wordpress:cli-php8.5" +# Matches the database the workflow runs against by default. +DB_IMAGE="mysql:9.7" +WP_DIR="" +DB_HOST="127.0.0.1" +DB_NAME="test_db" +DB_USER="root" +DB_PASS="password" +SERVER_PORT="8889" +KEEP_INSTALL="no" +FIXTURES_DIR="" +EXPECT_FILE="" +RESULTS_FILE="" +CACHE_DIR="" +# Long enough for the slowest download or activation seen in the directory, short enough that a plugin which +# hangs does not stall a run that is meant to finish overnight. +PLUGIN_TIMEOUT=300 + +# How many levels of `Requires Plugins` to follow. Chains in the directory are one level deep in practice, so +# the bound is only here to stop a circular declaration from looping forever. +MAX_DEPENDENCY_DEPTH=3 + +usage() { + cat <<'USAGE_EOF' +Usage: test-plugins.sh [options] [slug...] + +Checks that WordPress.org plugins can be activated against a version of WordPress without fataling. + +Options: + --plugins= Plugin slugs to test, separated by commas or spaces. Slugs can also be passed as + positional arguments. + --plugins-file= Read plugin slugs from a file, one per line, such as the output of list-plugins.sh. + Lines starting with # are ignored. + --wp-version= Version of WordPress to test against. Accepts "latest", "nightly", or a version + number such as a beta or RC. Default: nightly. + --php-image= Container image the checks run in. Default: wordpress:cli-php8.5. + --db-image= Container image the database runs in. Default: mysql:9.7. + --no-docker Run the checks in the current environment rather than in a container. Requires PHP, + WP-CLI, curl and a database that is already running. + --dir= Where to install WordPress. Only used with --no-docker. Default: a temporary + directory, which is removed again on exit. + --db-host= Database host, with an optional port. Only used with --no-docker. Default: 127.0.0.1. + --db-name= Database name. Only used with --no-docker. Default: test_db. + --db-user= Database user. Only used with --no-docker. Default: root. + --db-pass= Database password. Only used with --no-docker. Default: password. + --port= Port the PHP built-in server listens on. Default: 8889. + --keep Leave the WordPress install in place on exit rather than removing it. Only used with + --no-docker. + --fixtures= Install a plugin from / instead of WordPress.org when that directory exists. + --expect= Check each plugin's result against the one listed in , and exit 0 only when they + all match. Plugins are taken from the file when none are named. + --results= Append each plugin's result to as it is recorded, and skip plugins that already + have a row in it. Running the same command again after an interruption carries on + where it stopped. + --cache-dir= Keep WP-CLI's download cache in , so that a second run, against another version + of WordPress for example, does not download every plugin again. + --timeout= Give up on a plugin's download, activation or WP-CLI check after this long. A plugin + whose activation does not finish in time fails. 0 turns this off. Default: 300. + -h, --help Print this message. + +Examples: + # Check two plugins against nightly, provisioning everything in containers. + test-plugins.sh --wp-version=nightly woocommerce classic-editor + + # Reproduce a failure from a workflow run against the release it was seen on. + test-plugins.sh --wp-version=7.1-RC1 eps-301-redirects + + # Run the checks here, against a database that is already running. + test-plugins.sh --no-docker --db-host=127.0.0.1:3306 --db-pass=root hello-dolly + + # Test a long list overnight, resuming if interrupted, and sharing downloads with later runs. + test-plugins.sh --plugins-file=plugins.txt --results=nightly.tsv --cache-dir=~/.cache/plugin-compat + + # Check that plugins built to break are still caught. + test-plugins.sh --fixtures=tools/plugin-compatibility/fixtures \ + --expect=tools/plugin-compatibility/fixtures/expected.txt + +Exits 0 when no plugin fataled, and 1 when one did. A plugin that could not be downloaded, or that core +declined to activate because its declared requirements are not met, is reported as skipped and does not +fail the run. +USAGE_EOF +} + +# error +# +# Prints a message to stderr and stops. +error() { + printf 'Error: %s\n' "${1}" >&2 + exit 1 +} + +# exit_on_signal +# +# Exits on an interrupt rather than carrying on, so that the EXIT trap cleans up once and nothing runs against +# the server or container it removed. A trap that only cleans up hands control back to whatever was running. +exit_on_signal() { + trap 'exit 130' INT + trap 'exit 143' TERM +} + +while [ "$#" -gt 0 ]; do + case "${1}" in + --plugins=* ) PLUGIN_LIST="${PLUGIN_LIST} ${1#*=}" ;; + --wp-version=* ) WP_VERSION="${1#*=}" ;; + --php-image=* ) PHP_IMAGE="${1#*=}" ;; + --db-image=* ) DB_IMAGE="${1#*=}" ;; + --no-docker ) USE_DOCKER="no" ;; + --dir=* ) WP_DIR="${1#*=}" ;; + --db-host=* ) DB_HOST="${1#*=}" ;; + --db-name=* ) DB_NAME="${1#*=}" ;; + --db-user=* ) DB_USER="${1#*=}" ;; + --db-pass=* ) DB_PASS="${1#*=}" ;; + --port=* ) SERVER_PORT="${1#*=}" ;; + --keep ) KEEP_INSTALL="yes" ;; + --fixtures=* ) FIXTURES_DIR="${1#*=}" ;; + --expect=* ) EXPECT_FILE="${1#*=}" ;; + --plugins-file=* ) + PLUGINS_FILE="${1#*=}" + [ -f "${PLUGINS_FILE}" ] || error "The plugins file ${PLUGINS_FILE} does not exist." + PLUGIN_LIST="${PLUGIN_LIST} +$( grep -v '^[[:space:]]*#' "${PLUGINS_FILE}" )" + ;; + --results=* ) RESULTS_FILE="${1#*=}" ;; + --cache-dir=* ) CACHE_DIR="${1#*=}" ;; + --timeout=* ) PLUGIN_TIMEOUT="${1#*=}" ;; + -h | --help ) usage; exit 0 ;; + -* ) usage >&2; error "Unknown option ${1}." ;; + * ) PLUGIN_LIST="${PLUGIN_LIST} ${1}" ;; + esac + shift +done + +# Both are resolved now, while relative paths still mean what they meant to whoever passed them. +if [ -n "${FIXTURES_DIR}" ]; then + FIXTURES_DIR="$( cd "${FIXTURES_DIR}" > /dev/null 2>&1 && pwd )" || error "The fixtures directory does not exist." +fi + +if [ -n "${RESULTS_FILE}" ]; then + RESULTS_DIR="$( cd "$( dirname "${RESULTS_FILE}" )" > /dev/null 2>&1 && pwd )" || error "The directory for the results file does not exist." + RESULTS_FILE="${RESULTS_DIR%/}/$( basename "${RESULTS_FILE}" )" + touch "${RESULTS_FILE}" || error "The results file ${RESULTS_FILE} could not be written." +fi + +if [ -n "${CACHE_DIR}" ]; then + mkdir -p "${CACHE_DIR}" || error "The cache directory could not be created." + CACHE_DIR="$( cd "${CACHE_DIR}" > /dev/null 2>&1 && pwd )" +fi + +printf '%s' "${PLUGIN_TIMEOUT}" | grep -Eq '^[0-9]+$' || error "--timeout must be a number of seconds, got \"${PLUGIN_TIMEOUT}\"." + +if [ -n "${EXPECT_FILE}" ]; then + [ -f "${EXPECT_FILE}" ] || error "The expected results file does not exist." + EXPECT_DIR="$( cd "$( dirname "${EXPECT_FILE}" )" > /dev/null 2>&1 && pwd )" + EXPECT_FILE="${EXPECT_DIR%/}/$( basename "${EXPECT_FILE}" )" + + # The expected results double as the list of plugins to test, in the order they are listed. + if [ -z "$( printf '%s' "${PLUGIN_LIST}" | tr -d '[:space:],' )" ]; then + PLUGIN_LIST="$( awk '$1 !~ /^#/ && NF > 0 { print $1 }' "${EXPECT_FILE}" )" + fi +fi + +# Commas, spaces and newlines all separate slugs, so a list can be pasted in however it was written down. +PLUGIN_SLUGS="$( printf '%s' "${PLUGIN_LIST}" | tr ',[:space:]' '\n' | grep -v '^$' | awk '!seen[$0]++' )" + +if [ -z "${PLUGIN_SLUGS}" ]; then + usage >&2 + error "No plugins were named." +fi + +# A value outside the character set the directory uses is a typo worth stopping for, rather than a plugin +# reported as impossible to download several minutes from now. +INVALID_SLUGS="$( printf '%s\n' "${PLUGIN_SLUGS}" | grep -Ev '^[a-z0-9-]+$' )" + +if [ -n "${INVALID_SLUGS}" ]; then + printf 'These values are not WordPress.org plugin slugs:\n%s\n' "${INVALID_SLUGS}" >&2 + exit 1 +fi + +# Plugins that already have a result were tested by an earlier run that was interrupted. +if [ -n "${RESULTS_FILE}" ] && [ -s "${RESULTS_FILE}" ]; then + REQUESTED_COUNT="$( printf '%s\n' "${PLUGIN_SLUGS}" | wc -l | tr -d ' ' )" + PLUGIN_SLUGS="$( printf '%s\n' "${PLUGIN_SLUGS}" | awk -F '\t' 'NR == FNR { done[$1] = 1; next } !( $0 in done )' "${RESULTS_FILE}" - )" + REMAINING_COUNT="$( printf '%s' "${PLUGIN_SLUGS}" | grep -c . )" + + if [ "${REMAINING_COUNT}" -lt "${REQUESTED_COUNT}" ]; then + printf '%s of %s plugins already have a result in %s.\n' "$(( REQUESTED_COUNT - REMAINING_COUNT ))" "${REQUESTED_COUNT}" "${RESULTS_FILE}" + fi + + if [ "${REMAINING_COUNT}" -eq 0 ]; then + printf 'Nothing left to test.\n' + exit 0 + fi +fi + +## +# Provisions a database and a PHP, and runs this same script inside them. +# +# The checks themselves are identical either way: the container is handed this file and runs it with +# `--no-docker`, so there is only one copy of the logic that decides whether a plugin passed. +## +run_in_docker() { + command -v docker > /dev/null 2>&1 || error "Docker is required, or pass --no-docker to run the checks here." + + # Everything is named after the process so that two runs at once do not collide. + DOCKER_NETWORK="plugin-compatibility-$$" + DB_CONTAINER="plugin-compatibility-db-$$" + CHECKS_CONTAINER="plugin-compatibility-checks-$$" + + # Both are removed however this exits, an interrupt included. `--volumes` takes the anonymous volume the + # database image keeps its data in along with the container, which would otherwise pile up run after run. + trap 'docker rm --force "${CHECKS_CONTAINER}" > /dev/null 2>&1; docker rm --force --volumes "${DB_CONTAINER}" > /dev/null 2>&1; docker network rm "${DOCKER_NETWORK}" > /dev/null 2>&1; rm -f "${PLUGINS_LIST_FILE:-}"' EXIT + exit_on_signal + + printf 'Starting %s.\n' "${DB_IMAGE}" + + docker network create "${DOCKER_NETWORK}" > /dev/null || error "The Docker network could not be created." + + docker run --detach --name "${DB_CONTAINER}" --network "${DOCKER_NETWORK}" \ + --env MYSQL_ROOT_PASSWORD="${DB_PASS}" \ + --env MYSQL_DATABASE="${DB_NAME}" \ + "${DB_IMAGE}" > /dev/null || error "The database container could not be started." + + # A first run has to initialise the data directory, which takes appreciably longer than a later one. + # + # The image initialises it with a temporary server that answers on the socket and then restarts, so the + # ping goes over TCP, which only the real server listens on. A socket ping can succeed in that window and + # leave the next step to find the database gone. + DB_READY="no" + + for _ in $( seq 1 90 ); do + if docker exec "${DB_CONTAINER}" mysqladmin ping --silent --protocol=tcp --host=127.0.0.1 --user=root --password="${DB_PASS}" > /dev/null 2>&1; then + DB_READY="yes" + break + fi + sleep 2 + done + + if [ "${DB_READY}" != "yes" ]; then + docker logs "${DB_CONTAINER}" 2>&1 | tail -n 20 + error "The database did not become available." + fi + + printf 'The database is ready. Running the checks in %s.\n\n' "${PHP_IMAGE}" + + # Fixtures and expected results live on the host, so they are mounted read only and passed on by their + # path inside the container. + EXTRA_DOCKER_ARGS=() + EXTRA_SCRIPT_ARGS=() + + if [ -n "${FIXTURES_DIR}" ]; then + EXTRA_DOCKER_ARGS=( "${EXTRA_DOCKER_ARGS[@]+"${EXTRA_DOCKER_ARGS[@]}"}" --volume "${FIXTURES_DIR}:/fixtures:ro" ) + EXTRA_SCRIPT_ARGS=( "${EXTRA_SCRIPT_ARGS[@]+"${EXTRA_SCRIPT_ARGS[@]}"}" --fixtures=/fixtures ) + fi + + if [ -n "${EXPECT_FILE}" ]; then + EXTRA_DOCKER_ARGS=( "${EXTRA_DOCKER_ARGS[@]+"${EXTRA_DOCKER_ARGS[@]}"}" --volume "${EXPECT_FILE}:/expected.txt:ro" ) + EXTRA_SCRIPT_ARGS=( "${EXTRA_SCRIPT_ARGS[@]+"${EXTRA_SCRIPT_ARGS[@]}"}" --expect=/expected.txt ) + fi + + # The results file and the cache are written to from inside the container, so it runs as the user who + # owns them. Without that a Linux host would refuse the image's own user. + if [ -n "${RESULTS_FILE}" ] || [ -n "${CACHE_DIR}" ]; then + EXTRA_DOCKER_ARGS=( "${EXTRA_DOCKER_ARGS[@]+"${EXTRA_DOCKER_ARGS[@]}"}" --user "$( id -u ):$( id -g )" ) + fi + + if [ -n "${RESULTS_FILE}" ]; then + EXTRA_DOCKER_ARGS=( "${EXTRA_DOCKER_ARGS[@]+"${EXTRA_DOCKER_ARGS[@]}"}" --volume "${RESULTS_DIR}:/results" ) + EXTRA_SCRIPT_ARGS=( "${EXTRA_SCRIPT_ARGS[@]+"${EXTRA_SCRIPT_ARGS[@]}"}" "--results=/results/$( basename "${RESULTS_FILE}" )" ) + fi + + if [ -n "${CACHE_DIR}" ]; then + EXTRA_DOCKER_ARGS=( "${EXTRA_DOCKER_ARGS[@]+"${EXTRA_DOCKER_ARGS[@]}"}" --volume "${CACHE_DIR}:/wp-cli-cache" ) + EXTRA_SCRIPT_ARGS=( "${EXTRA_SCRIPT_ARGS[@]+"${EXTRA_SCRIPT_ARGS[@]}"}" --cache-dir=/wp-cli-cache ) + fi + + # The list is handed over as a file, because the whole directory would not fit on a command line. + PLUGINS_LIST_FILE="$( mktemp )" + printf '%s\n' "${PLUGIN_SLUGS}" > "${PLUGINS_LIST_FILE}" + chmod 644 "${PLUGINS_LIST_FILE}" + + # WordPress is installed inside the container rather than in a mounted directory, so a plugin cannot + # leave anything behind on the host and file ownership never comes into it. `HOME` is pointed at a + # writable directory because WP-CLI caches its downloads there. + # + # It runs in the background and is waited on, because Bash holds a signal until a foreground command + # finishes. Waited on, an interrupt or a TERM from test-all-plugins.sh stops it at once, and the EXIT trap + # removes the container. + docker run --rm --name "${CHECKS_CONTAINER}" --network "${DOCKER_NETWORK}" \ + --volume "${SCRIPT_PATH}:/usr/local/bin/test-plugins.sh:ro" \ + --volume "${PLUGINS_LIST_FILE}:/plugins.txt:ro" \ + ${EXTRA_DOCKER_ARGS[@]+"${EXTRA_DOCKER_ARGS[@]}"} \ + --env HOME=/tmp \ + --entrypoint bash \ + "${PHP_IMAGE}" /usr/local/bin/test-plugins.sh \ + --no-docker \ + --dir=/tmp/wordpress \ + --wp-version="${WP_VERSION}" \ + --db-host="${DB_CONTAINER}" \ + --db-name="${DB_NAME}" \ + --db-user=root \ + --db-pass="${DB_PASS}" \ + --port="${SERVER_PORT}" \ + --timeout="${PLUGIN_TIMEOUT}" \ + ${EXTRA_SCRIPT_ARGS[@]+"${EXTRA_SCRIPT_ARGS[@]}"} \ + --plugins-file=/plugins.txt & + + wait "$!" +} + +## +# Installs WordPress, starts a web server, and checks each plugin against it. +## +run_checks() { + for REQUIRED_COMMAND in php curl; do + command -v "${REQUIRED_COMMAND}" > /dev/null 2>&1 || error "${REQUIRED_COMMAND} is required to run the checks without Docker." + done + + # `type -P` searches the path rather than resolving to the `wp` wrapper function defined below. + WP_CLI_BIN="$( type -P wp )" + + [ -n "${WP_CLI_BIN}" ] || error "WP-CLI is required to run the checks without Docker." + + # A phar or PHP entry point either has no shebang or one naming PHP. Any other shebang is a launcher. + WP_CLI_IS_LAUNCHER="no" + + case "$( head -n 1 "${WP_CLI_BIN}" 2>/dev/null | tr -d '\000' )" in + '#!'*php* ) ;; + '#!'* ) WP_CLI_IS_LAUNCHER="yes" ;; + esac + + # The steps that run a plugin's code are bounded by `timeout`, which can only run a program, so the WP-CLI + # command is kept as a list of arguments rather than only behind the `wp` function. + if [ "${WP_CLI_IS_LAUNCHER}" = "yes" ]; then + WP_CLI_COMMAND=( env "WP_CLI_PHP_ARGS=-d memory_limit=-1${WP_CLI_PHP_ARGS:+ ${WP_CLI_PHP_ARGS}}" "${WP_CLI_BIN}" ) + else + WP_CLI_COMMAND=( php -d memory_limit=-1 "${WP_CLI_BIN}" ) + fi + + TIMEOUT_COMMAND=() + + if [ "${PLUGIN_TIMEOUT}" -gt 0 ]; then + if command -v timeout > /dev/null 2>&1; then + TIMEOUT_COMMAND=( timeout "${PLUGIN_TIMEOUT}" ) + elif command -v gtimeout > /dev/null 2>&1; then + TIMEOUT_COMMAND=( gtimeout "${PLUGIN_TIMEOUT}" ) + else + printf 'Warning: timeout is not installed, so a plugin that hangs will stall the run.\n' >&2 + fi + fi + + WORK_DIR="$( mktemp -d )" + RESULTS="${RESULTS_FILE:-${WORK_DIR}/results.tsv}" + RESPONSE_BODY="${WORK_DIR}/response.html" + SERVER_LOG="${WORK_DIR}/php-server.log" + SERVER_PID="" + + # A results file that was handed over holds an earlier run's rows, which are carried on from. + [ -n "${RESULTS_FILE}" ] || : > "${RESULTS}" + + if [ -n "${CACHE_DIR}" ]; then + export WP_CLI_CACHE_DIR="${CACHE_DIR}" + fi + + if [ -z "${WP_DIR}" ]; then + WP_DIR="${WORK_DIR}/wordpress" + fi + + mkdir -p "${WP_DIR}" || error "The WordPress directory could not be created." + + # The server is stopped and the temporary files are removed however this exits. The WordPress install is + # only removed when this script created it, so that `--dir` never deletes a directory it was handed. + cleanup_environment() { + if [ -n "${SERVER_PID}" ]; then + stop_server + fi + + if [ "${KEEP_INSTALL}" = "yes" ]; then + printf '\nThe WordPress install was left in %s.\n' "${WP_DIR}" + else + rm -rf "${WORK_DIR}" + fi + } + + # The built-in server forks its workers, and they keep answering on the port after the process that + # started them is gone. The server runs in its own process group (see below) so that the whole group can + # be stopped at once, and this waits for the last of them to exit so that a run which follows straight + # after finds the port free. + stop_server() { + kill -TERM -- "-${SERVER_PID}" > /dev/null 2>&1 + + for _ in $( seq 1 50 ); do + kill -0 -- "-${SERVER_PID}" > /dev/null 2>&1 || break + sleep 0.1 + done + + kill -KILL -- "-${SERVER_PID}" > /dev/null 2>&1 + wait "${SERVER_PID}" > /dev/null 2>&1 + SERVER_PID="" + } + + trap cleanup_environment EXIT + exit_on_signal + + cd "${WP_DIR}" || error "The WordPress directory could not be entered." + + # A relative `--dir` would be resolved against the wrong directory from here on. + WP_DIR="$( pwd )" + + SITE_URL="http://127.0.0.1:${SERVER_PORT}" + + printf 'Downloading WordPress %s.\n' "${WP_VERSION}" + wp core download --version="${WP_VERSION}" || error "WordPress ${WP_VERSION} could not be downloaded." + + wp config create --dbname="${DB_NAME}" --dbuser="${DB_USER}" --dbpass="${DB_PASS}" --dbhost="${DB_HOST}" \ + || error "wp-config.php could not be written." + + # Errors need to reach `wp-content/debug.log` so that a white screen of death is still detectable. + # + # `WP_DEBUG_DISPLAY` is left off on purpose: this should behave the way a production site does, where a + # fatal error is an empty page and an HTTP 500 rather than a printed stack trace. + # + # The fatal error handler is disabled so that a fatal is reported as a fatal instead of being swallowed + # by recovery mode, which would also deactivate the plugin mid-test. + # + # WP-Cron is disabled because WordPress spawns it as a loopback request during a front end request. The + # loopback lands back on the same PHP built-in server that is still busy serving the request that + # spawned it, and the two deadlock until curl gives up. + wp config set WP_DEBUG true --raw + wp config set WP_DEBUG_LOG true --raw + wp config set WP_DEBUG_DISPLAY false --raw + wp config set WP_DISABLE_FATAL_ERROR_HANDLER true --raw + wp config set DISABLE_WP_CRON true --raw + + wp core install --url="${SITE_URL}" --title="Plugin Compatibility Test" --admin_user=admin \ + --admin_password=password --admin_email=me@example.org --skip-email \ + || error "WordPress could not be installed. Check that the database is reachable at ${DB_HOST}." + + rm -f "${WP_DIR}/wp-content/debug.log" + snapshot_site || error "The freshly installed site could not be recorded for resetting between plugins." + + # The site needs to answer real requests so that fatals which only happen on a front end or admin page + # load are caught. The built-in server is enough for that and needs nothing installed. + # + # `PHP_CLI_SERVER_WORKERS` is set because the built-in server is single threaded by default. Plenty of + # plugins make a loopback request to the site they are running on, and a single threaded server cannot + # answer one while it is still serving the request that made it. + # + # Job control is switched on just long enough to start the server, because that is what gives a + # background job a process group of its own, whose ID is the PID of the server. + set -m + PHP_CLI_SERVER_WORKERS=4 php -S "127.0.0.1:${SERVER_PORT}" -t "${WP_DIR}" > "${SERVER_LOG}" 2>&1 & + SERVER_PID=$! + set +m + + # Any server already listening on the port would answer a request for a WordPress page just as well, and + # every check after this would then be made against that site instead. So readiness means this server is + # still running and is serving a file that only exists in this install. + READY_FILE="plugin-compatibility-ready-$$-${RANDOM}.txt" + READY_TOKEN="$$-${RANDOM}-$( date +%s )" + printf '%s' "${READY_TOKEN}" > "${WP_DIR}/${READY_FILE}" + + SERVER_READY="no" + + for _ in $( seq 1 30 ); do + kill -0 "${SERVER_PID}" > /dev/null 2>&1 || break + + if [ "$( curl -sf "${SITE_URL}/${READY_FILE}" 2>/dev/null )" = "${READY_TOKEN}" ]; then + SERVER_READY="yes" + break + fi + sleep 1 + done + + rm -f "${WP_DIR:?}/${READY_FILE}" + + if [ "${SERVER_READY}" != "yes" ]; then + cat "${SERVER_LOG}" + error "The PHP built-in server did not start on port ${SERVER_PORT}, or something else is already listening on it." + fi + + printf 'The PHP built-in server is ready on %s.\n' "${SITE_URL}" + + test_plugins + report_results + REPORT_EXIT_CODE=$? + + # Plugins built to fail are expected to fail, so with a list of expected results it is whether they matched + # that decides the exit code. + if [ -n "${EXPECT_FILE}" ]; then + check_expectations + return + fi + + return "${REPORT_EXIT_CODE}" +} + +# wp +# +# Runs WP-CLI with the memory limit lifted. Unpacking a WordPress zip needs more than the 128M a stock +# php.ini allows. The phar most environments carry is handed to PHP with the limit set, while a shell +# launcher, such as the `bin/wp` of a Composer install, is run as it is and passed the limit through the +# `WP_CLI_PHP_ARGS` it reads. Handing a shell script to PHP would print it rather than run it. +wp() { + "${WP_CLI_COMMAND[@]}" "$@" +} + +# wp_limited +# +# Runs WP-CLI like `wp`, but gives up after --timeout seconds. Used for the steps that run a plugin's code or +# download it, where a hang would otherwise stall every plugin after it. +wp_limited() { + ${TIMEOUT_COMMAND[@]+"${TIMEOUT_COMMAND[@]}"} "${WP_CLI_COMMAND[@]}" "$@" +} + +# timed_out +# +# Returns zero when the exit code is the one `timeout` uses for a command it stopped: 124 from GNU +# coreutils, 143 from BusyBox, which the WP-CLI image carries. +timed_out() { + [ "${#TIMEOUT_COMMAND[@]}" -gt 0 ] && { [ "${1}" -eq 124 ] || [ "${1}" -eq 143 ]; } +} + +# record +# +# Appends one tab separated row to the results file. The reason is flattened so that it cannot break the +# markdown table that is generated from these rows later on. +record() { + SAFE_REASON="$( printf '%s' "${5:--}" | tr '\n\t|' ' ' )" + printf '%s\t%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${4:--}" "${SAFE_REASON}" >> "${RESULTS}" +} + +# check_url +# +# Requests a path on the test site and prints a reason to stdout when the response looks broken. Prints +# nothing when the request looks healthy. +check_url() { + CURL_EXIT_CODE=0 + # The exit code is captured separately rather than falling back to a literal inside the command + # substitution, which would append to whatever curl had already written and produce a nonsense status + # like "200000" when a request returned headers and then stalled. + HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" )" || CURL_EXIT_CODE=$? + + if [ -z "${HTTP_CODE}" ] || [ "${HTTP_CODE}" = "000" ]; then + printf 'The request to %s did not complete, curl exit code %s' "${1}" "${CURL_EXIT_CODE}" + return + fi + + if [ "${HTTP_CODE}" -ge 500 ]; then + printf 'The request to %s returned HTTP %s' "${1}" "${HTTP_CODE}" + return + fi + + if [ "${CURL_EXIT_CODE}" -ne 0 ]; then + printf 'The request to %s returned HTTP %s but the response did not finish, curl exit code %s' "${1}" "${HTTP_CODE}" "${CURL_EXIT_CODE}" + return + fi + + # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors back on for + # itself. + if grep -qiE '(Fatal|Parse) error' "${RESPONSE_BODY}"; then + printf 'The response from %s contained a fatal error' "${1}" + fi +} + +# wait_for_requests +# +# Waits until the built-in server has finished every request it has accepted, for up to 30 seconds, and +# returns non zero when some are still running after that. +# +# A plugin can start a request of its own that the checks never wait on, such as a non-blocking loopback to +# run a background task. A fatal in it would be logged after the log has been checked, or after the reset, +# where it would be blamed on the next plugin. The server logs "Accepted" and "Closing" for every connection, +# so the difference between the two is the number still running. +wait_for_requests() { + for _ in $( seq 1 30 ); do + OPEN_REQUESTS="$( awk '/ Accepted$/ { open++ } / Closing$/ { open-- } END { print open + 0 }' "${SERVER_LOG}" )" + [ "${OPEN_REQUESTS}" -gt 0 ] || return 0 + sleep 1 + done + + printf '%s requests to the test site were still running after 30 seconds.\n' "${OPEN_REQUESTS}" + return 1 +} + +# logged_fatals +# +# Prints the fatal and parse errors in the debug log, one per line, and returns non zero when there are none. +# A parse error in a file included after output has started ends the request just as surely as a fatal does, +# and still answers with a 200. +logged_fatals() { + [ -f "${WP_DIR}/wp-content/debug.log" ] && grep -E 'PHP (Fatal|Parse) error' "${WP_DIR}/wp-content/debug.log" +} + +# install_plugin +# +# Installs a plugin from the fixtures directory when there is one by that name, and from WordPress.org +# otherwise. Returns non zero when the plugin could not be installed. +install_plugin() { + if [ -n "${FIXTURES_DIR}" ] && [ -d "${FIXTURES_DIR}/${1}" ]; then + printf 'Installing %s from %s.\n' "${1}" "${FIXTURES_DIR}" + cp -R "${FIXTURES_DIR}/${1}" "${WP_DIR}/wp-content/plugins/" + return + fi + + wp_limited plugin install "${1}" --skip-plugins --skip-themes +} + +# is_active +# +# Returns zero when core has recorded the plugin as active. +is_active() { + wp plugin is-active "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 +} + +# plugin_dependencies +# +# Prints the slugs named in the plugin's `Requires Plugins` header, one per line. WordPress' own header +# parser is used so that the same rules apply here as when core decides whether a plugin's dependencies are +# met. +plugin_dependencies() { + PLUGIN_FILE="$( wp plugin path "${1}" --skip-plugins --skip-themes 2>/dev/null || printf '' )" + + [ -n "${PLUGIN_FILE}" ] || return 0 + + # The single quotes are what keep the shell out of the PHP below. The path is handed over as an + # environment variable rather than interpolated so that it never reaches PHP as source code. + export PLUGIN_FILE + + # shellcheck disable=SC2016 + wp eval --skip-plugins --skip-themes ' + require_once ABSPATH . "wp-admin/includes/plugin.php"; + + $plugin_data = get_plugin_data( getenv( "PLUGIN_FILE" ), false, false ); + + foreach ( explode( ",", $plugin_data["RequiresPlugins"] ?? "" ) as $dependency ) { + // The same pattern as WP_Plugin_Dependencies::sanitize_dependency_slugs(). Core ignores anything + // else, so a malformed entry must not make the plugin wait on a dependency core never asks for. + if ( preg_match( "/^[a-z0-9]+(-[a-z0-9]+)*$/", trim( $dependency ) ) ) { + echo trim( $dependency ), "\n"; + } + } + ' 2>/dev/null +} + +# Prefix of the copies of the freshly installed site's tables, which every plugin's changes are rolled back to. +BASELINE_TABLE_PREFIX="pcbaseline_" + +# snapshot_site +# +# Records the freshly installed site so that it can be put back exactly after each plugin: the entries at the +# root of the install, `wp-config.php` and `wp-content` as an archive, and a copy of every database table. +snapshot_site() { + BASELINE_DIR="${WORK_DIR}/baseline" + mkdir -p "${BASELINE_DIR}" || return 1 + + ( + shopt -s dotglob nullglob + + for ENTRY in "${WP_DIR}"/*; do + printf '%s\n' "${ENTRY##*/}" + done + ) > "${BASELINE_DIR}/root-entries" || return 1 + tar -C "${WP_DIR}" -cf "${BASELINE_DIR}/files.tar" wp-config.php wp-content || return 1 + + # The table copies live in the same database, which saves depending on a database client being + # installed that can talk to whatever server version the run is using. The prefix is handed over as an + # environment variable so that it never reaches PHP as source code. + export BASELINE_TABLE_PREFIX + + # shellcheck disable=SC2016 + wp eval --skip-plugins --skip-themes ' + global $wpdb; + + $prefix = getenv( "BASELINE_TABLE_PREFIX" ); + + foreach ( $wpdb->get_col( "SHOW TABLES" ) as $table ) { + // Left over from an earlier run against the same database. + if ( 0 === strpos( $table, $prefix ) ) { + continue; + } + + $wpdb->query( "DROP TABLE IF EXISTS `{$prefix}{$table}`" ); + + if ( false === $wpdb->query( "CREATE TABLE `{$prefix}{$table}` LIKE `{$table}`" ) + || false === $wpdb->query( "INSERT INTO `{$prefix}{$table}` SELECT * FROM `{$table}`" ) ) { + exit( 1 ); + } + } + ' +} + +# reset_site +# +# Puts the site back the way `snapshot_site` found it. Removing the plugin directories is not enough on its +# own: plenty of plugins write drop-ins such as `advanced-cache.php`, add constants to `wp-config.php`, or +# create tables and options, and a drop-in left loading files that are gone would break every plugin tested +# after it. The files are restored first so that no drop-in is around to be loaded while the database is put +# back. +reset_site() { + ( + shopt -s dotglob nullglob + + for ENTRY in "${WP_DIR}"/*; do + grep -qxF -- "${ENTRY##*/}" "${BASELINE_DIR}/root-entries" || rm -rf "${ENTRY}" + done + ) + + rm -rf "${WP_DIR:?}/wp-content" || return 1 + tar -C "${WP_DIR}" -xf "${BASELINE_DIR}/files.tar" || return 1 + + # shellcheck disable=SC2016 + wp eval --skip-plugins --skip-themes ' + global $wpdb; + + $prefix = getenv( "BASELINE_TABLE_PREFIX" ); + $baseline = array(); + + $wpdb->query( "SET FOREIGN_KEY_CHECKS = 0" ); + + foreach ( $wpdb->get_results( "SHOW FULL TABLES", ARRAY_N ) as $row ) { + list( $table, $type ) = $row; + + if ( 0 === strpos( $table, $prefix ) ) { + $baseline[] = substr( $table, strlen( $prefix ) ); + } else { + $wpdb->query( ( "VIEW" === $type ? "DROP VIEW" : "DROP TABLE" ) . " `{$table}`" ); + } + } + + if ( ! $baseline ) { + exit( 1 ); + } + + foreach ( $baseline as $table ) { + if ( false === $wpdb->query( "CREATE TABLE `{$table}` LIKE `{$prefix}{$table}`" ) + || false === $wpdb->query( "INSERT INTO `{$table}` SELECT * FROM `{$prefix}{$table}`" ) ) { + exit( 1 ); + } + } + ' +} + +# group_start / group_end +# +# Collapses a plugin's output in the workflow log. The markers mean nothing to a terminal, so they are only +# printed when this is running in Actions. +group_start() { + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + printf '::group::%s\n' "${1}" + else + printf '\n----- %s -----\n' "${1}" + fi +} + +group_end() { + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + printf '::endgroup::\n' + fi +} + +# reset_failed +# +# Stops the run when the site could not be put back after a plugin. Every result after that point would be +# measured against whatever the plugin left behind, so the results so far are reported and nothing more is +# tested. +reset_failed() { + printf '\nThe site could not be reset after %s, so no further plugins were tested.\n' "${1}" >&2 + report_results + exit 1 +} + +## +# Tests each plugin in turn, writing a row per plugin to the results file. +## +test_plugins() { + while IFS= read -r SLUG; do + [ -n "${SLUG}" ] || continue + + group_start "${SLUG}" + + STATUS="PASS" + REASON="-" + VERSION="unknown" + DEPENDENCIES=() + + # Start every plugin with an empty log so that anything found in it belongs to this plugin. + rm -f "${WP_DIR}/wp-content/debug.log" + + # Step 1: download the plugin. A download failure is recorded as skipped rather than failed, since + # it generally means a network flake or a plugin that is no longer in the directory. + if ! install_plugin "${SLUG}"; then + record "${SLUG}" "unknown" "SKIPPED" "-" "The plugin could not be downloaded from WordPress.org" + reset_site || reset_failed "${SLUG}" + group_end + continue + fi + + VERSION="$( wp plugin get "${SLUG}" --field=version --skip-plugins --skip-themes 2>/dev/null || printf 'unknown' )" + + # Step 2: install whatever the plugin names in `Requires Plugins`. Core refuses to activate a plugin + # whose dependencies are missing, so without this every WooCommerce extension - a large slice of the + # most popular plugins - would go untested. + # + # Dependencies are resolved a level at a time so that a dependency declaring its own is installed + # too. Each one is resolved only once, which covers a plugin named twice in the same tree as well as a + # declaration pointing back at the plugin under test. A dependency that is already installed, such as + # the Akismet that ships with core, is not downloaded again but is still activated and followed. + PENDING=() + RESOLVED=$'\n'"${SLUG}"$'\n' + + while IFS= read -r DEPENDENCY; do + [ -n "${DEPENDENCY}" ] && PENDING[${#PENDING[@]}]="${DEPENDENCY}" + done < <( plugin_dependencies "${SLUG}" ) + + DEPTH=0 + + while [ "${#PENDING[@]}" -gt 0 ] && [ "${DEPTH}" -lt "${MAX_DEPENDENCY_DEPTH}" ]; do + NEXT=() + + for DEPENDENCY in "${PENDING[@]}"; do + case "${RESOLVED}" in + *$'\n'"${DEPENDENCY}"$'\n'* ) continue ;; + esac + + RESOLVED="${RESOLVED}${DEPENDENCY}"$'\n' + + if ! wp plugin is-installed "${DEPENDENCY}" --skip-plugins --skip-themes > /dev/null 2>&1; then + printf 'Installing %s, which %s requires.\n' "${DEPENDENCY}" "${SLUG}" + + if ! install_plugin "${DEPENDENCY}"; then + STATUS="SKIPPED" + REASON="The required plugin ${DEPENDENCY} is not available from WordPress.org" + break 2 + fi + fi + + DEPENDENCIES[${#DEPENDENCIES[@]}]="${DEPENDENCY}" + + while IFS= read -r NESTED_DEPENDENCY; do + [ -n "${NESTED_DEPENDENCY}" ] && NEXT[${#NEXT[@]}]="${NESTED_DEPENDENCY}" + done < <( plugin_dependencies "${DEPENDENCY}" ) + done + + PENDING=( ${NEXT[@]+"${NEXT[@]}"} ) + DEPTH=$(( DEPTH + 1 )) + done + + # Step 3: activate the dependencies. Core will not activate a plugin ahead of its own requirements, + # and the order they were discovered in says nothing about the order they depend on each other in - + # a plugin can require `base, extension` where `extension` requires `base` as well. So they are + # activated in passes: each pass activates whatever it can, and whatever core declined is tried + # again on the next. A pass that activates nothing means the rest can never be activated, whether + # because one of them is broken or because they require each other. + # + # A successful exit is not taken as proof of activation. A plugin that calls `exit` while activating + # ends WP-CLI with a zero status before core has recorded it as active. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + INACTIVE=( "${DEPENDENCIES[@]}" ) + + while [ "${#INACTIVE[@]}" -gt 0 ]; do + STILL_INACTIVE=() + + for DEPENDENCY in "${INACTIVE[@]}"; do + DEPENDENCY_OUTPUT="$( wp_limited plugin activate "${DEPENDENCY}" 2>&1 )" + printf '%s\n' "${DEPENDENCY_OUTPUT}" + + if ! is_active "${DEPENDENCY}"; then + STILL_INACTIVE[${#STILL_INACTIVE[@]}]="${DEPENDENCY}" + fi + done + + if [ "${#STILL_INACTIVE[@]}" -eq "${#INACTIVE[@]}" ]; then + STATUS="SKIPPED" + REASON="The required plugin ${STILL_INACTIVE[0]} could not be activated" + break + fi + + INACTIVE=( ${STILL_INACTIVE[@]+"${STILL_INACTIVE[@]}"} ) + done + fi + + # Step 4: with dependencies active the plugin under test is no longer alone on the site, so the + # baseline is checked before it is activated. A site that is already broken says something about the + # dependency rather than about the plugin being tested, and blaming the plugin for it would be the + # masking these checks are built to avoid. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + for URL_PATH in "/" "/wp-login.php"; do + BASELINE_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${BASELINE_REASON}" ]; then + STATUS="SKIPPED" + REASON="The required plugins are not healthy on their own: ${BASELINE_REASON}" + break + fi + done + + # A dependency can fatal after the page has started to render, which still answers with a 200. + # The fatal only shows up in the log, and it would show up again once the plugin under test is + # active and be blamed on that plugin, so the log is checked before it is cleared. + wait_for_requests || true + + if [ "${STATUS}" = "PASS" ] && LOGGED_FATALS="$( logged_fatals )"; then + printf '%s\n' "${LOGGED_FATALS}" + STATUS="SKIPPED" + REASON="The required plugins are not healthy on their own: $( printf '%s\n' "${LOGGED_FATALS}" | head -n 1 | cut -c 1-200 )" + fi + + # The plugin under test is activated through WP-CLI, which loads the dependencies first. One that is + # healthy over HTTP but cannot be loaded by WP-CLI, like the plugins described in step 8, would + # stop that activation and have its failure blamed on the plugin under test. + if [ "${STATUS}" = "PASS" ]; then + DEPENDENCY_EVAL_OUTPUT="$( wp_limited eval 'echo "loaded-ok";' 2>&1 )" + + if [ "${DEPENDENCY_EVAL_OUTPUT#*loaded-ok}" = "${DEPENDENCY_EVAL_OUTPUT}" ]; then + printf '%s\n' "${DEPENDENCY_EVAL_OUTPUT}" + STATUS="SKIPPED" + REASON="WP-CLI cannot load WordPress with the required plugins active, so the plugin cannot be activated" + fi + fi + + # Anything else the dependencies logged on their way up is not the responsibility of the plugin + # under test, so the log starts empty again here. + rm -f "${WP_DIR}/wp-content/debug.log" + fi + + # Step 5: activation. Activation runs the plugin's activation hooks and loads its main file. + if [ "${STATUS}" = "PASS" ]; then + ACTIVATE_EXIT_CODE=0 + ACTIVATE_OUTPUT="$( wp_limited plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? + printf '%s\n' "${ACTIVATE_OUTPUT}" + + # A visitor whose request runs into the same hang gets nothing back either. + if timed_out "${ACTIVATE_EXIT_CODE}"; then + STATUS="FAIL" + REASON="Activation did not finish within ${PLUGIN_TIMEOUT} seconds" + elif [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then + case "${ACTIVATE_OUTPUT}" in + # Core refuses to activate a plugin whose declared requirements are not met. + # Dependencies are installed above, so what is left here is a plugin asking for a + # version of PHP or WordPress this run is not using, or for a dependency that is not on + # WordPress.org. That is core working as designed rather than a fatal, so it is recorded + # as skipped. Core says "does not meet" when one of the two is unmet and "do not meet" when + # both are. + *"to be installed and activated"* | *"does not meet minimum requirements"* | *"do not meet minimum requirements"* ) + STATUS="SKIPPED" + REASON="Core declined to activate the plugin because its declared requirements are not met" + ;; + * ) + STATUS="FAIL" + REASON="The plugin could not be activated" + ;; + esac + elif ! is_active "${SLUG}"; then + # Checking the site now would be checking it without the plugin, and passing it for that. The + # check can also fail because activation broke the site itself, for example by writing a + # must-use plugin that fatals, so the site is checked before the plugin is let off. + HTTP_REASON="$( check_url "/" )" + wait_for_requests || true + + if [ -n "${HTTP_REASON}" ]; then + STATUS="FAIL" + REASON="${HTTP_REASON}" + elif LOGGED_FATALS="$( logged_fatals )"; then + printf '%s\n' "${LOGGED_FATALS}" + STATUS="FAIL" + REASON="A fatal error was logged: $( printf '%s\n' "${LOGGED_FATALS}" | head -n 1 | cut -c 1-200 )" + else + STATUS="SKIPPED" + REASON="WP-CLI reported success, but the plugin was not active afterwards, so its activation stopped early" + fi + fi + fi + + # Step 6: request the front page and the login screen through the PHP built-in server. A real request + # is what decides whether a plugin passed, because it is what a visitor gets. + if [ "${STATUS}" = "PASS" ]; then + for URL_PATH in "/" "/wp-login.php"; do + HTTP_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${HTTP_REASON}" ]; then + STATUS="FAIL" + REASON="${HTTP_REASON}" + break + fi + done + fi + + # Step 7: a fatal or parse error can be logged without changing the HTTP status, for example during + # a shutdown hook, so the debug log is checked separately, once any request the plugin started on its + # own has finished. + # A request that is still running after the wait is not a fatal on its own, so it does not change the + # result. + wait_for_requests || true + + if [ "${STATUS}" = "PASS" ] && LOGGED_FATALS="$( logged_fatals )"; then + printf '%s\n' "${LOGGED_FATALS}" + STATUS="FAIL" + REASON="A fatal error was logged: $( printf '%s\n' "${LOGGED_FATALS}" | head -n 1 | cut -c 1-200 )" + fi + + # Step 8: boot all of core plus the active plugin through WP-CLI. This runs last and cannot fail a + # plugin on its own. + # + # WP-CLI requires `wp-settings.php` from inside a method, so a plugin that assigns a variable at file + # scope and reads it back with `global` later finds nothing there. Plenty of plugins do exactly that, + # and the resulting fatal happens only under WP-CLI - a visitor never sees it. Failing a plugin for + # it would report breakage that does not exist on a real site, so it is recorded as a note against a + # plugin that is otherwise healthy. Anything that genuinely fatals on load fails the HTTP checks + # above. + # + # The fatal is written to the debug log too, which is why this comes after the log has been checked. + if [ "${STATUS}" = "PASS" ]; then + EVAL_OUTPUT="$( wp_limited eval 'echo "loaded-ok";' 2>&1 )" + + if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then + printf '%s\n' "${EVAL_OUTPUT}" + + case "${EVAL_OUTPUT}" in + *"Fatal error"* | *"Parse error"* | *"Uncaught"* ) + REASON="The site is healthy over HTTP, but WP-CLI fatals when it loads WordPress with the plugin active" + ;; + # Some plugins redirect or exit while loading, which stops WP-CLI without anything being + # broken. + * ) + printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' + ;; + esac + fi + fi + + # Step 9: record the outcome and put the site back the way it was found. + DEPENDENCY_LIST="-" + + if [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + DEPENDENCY_LIST="$( printf '%s, ' "${DEPENDENCIES[@]}" )" + DEPENDENCY_LIST="${DEPENDENCY_LIST%, }" + fi + + record "${SLUG}" "${VERSION}" "${STATUS}" "${DEPENDENCY_LIST}" "${REASON}" + + # A request still running when the site is reset could write to the next plugin's log or database. + wait_for_requests || true + reset_site || reset_failed "${SLUG}" + + printf '%s: %s\n' "${SLUG}" "${STATUS}" + group_end + done <> "${GITHUB_STEP_SUMMARY}" + fi + + # Plugins that could not be downloaded are reported but do not fail the run. + if [ "${FAIL_COUNT}" -gt 0 ]; then + printf '\nThe following plugins failed against WordPress %s:\n' "${WP_VERSION}" + awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $5 }' "${RESULTS}" + return 1 + fi + + printf '\nNo plugins fataled against WordPress %s.\n' "${WP_VERSION}" +} + +## +# Compares each plugin's result with the one the expected results file lists for it, and exits non zero when +# any of them differ. +# +# Each line of the file is a slug, the expected result, and optionally some text that the "Also active" or +# "Details" column has to contain. That last part is what tells a plugin that failed for the reason it was +# built to fail apart from one that failed for some other reason. +## +check_expectations() { + MISMATCH_COUNT=0 + MATCH_COUNT=0 + + printf '\nChecking the results against %s.\n\n' "${EXPECT_FILE}" + + # The `|| [ -n ... ]` keeps the last line when the file does not end with a newline. + while read -r EXPECTED_SLUG EXPECTED_STATUS EXPECTED_TEXT || [ -n "${EXPECTED_SLUG}" ]; do + case "${EXPECTED_SLUG}" in + '' | '#'* ) continue ;; + esac + + ROW="$( awk -F '\t' -v slug="${EXPECTED_SLUG}" '$1 == slug { print; exit }' "${RESULTS}" )" + ACTUAL_STATUS="$( printf '%s' "${ROW}" | cut -f 3 )" + ACTUAL_TEXT="$( printf '%s' "${ROW}" | cut -f 4- | tr '\t' ' ' )" + MATCHED="yes" + + if [ -z "${ROW}" ] || [ "${ACTUAL_STATUS}" != "${EXPECTED_STATUS}" ]; then + MATCHED="no" + elif [ -n "${EXPECTED_TEXT}" ]; then + case "${ACTUAL_TEXT}" in + *"${EXPECTED_TEXT}"* ) ;; + * ) MATCHED="no" ;; + esac + fi + + if [ "${MATCHED}" = "yes" ]; then + MATCH_COUNT=$(( MATCH_COUNT + 1 )) + printf 'ok %s: %s\n' "${EXPECTED_SLUG}" "${EXPECTED_STATUS}" + else + MISMATCH_COUNT=$(( MISMATCH_COUNT + 1 )) + printf 'MISMATCH %s: expected %s %s\n got %s %s\n' "${EXPECTED_SLUG}" "${EXPECTED_STATUS}" "${EXPECTED_TEXT}" "${ACTUAL_STATUS:-no result}" "${ACTUAL_TEXT}" + fi + done < "${EXPECT_FILE}" + + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + printf '%s of %s results matched %s.\n\n' "${MATCH_COUNT}" "$(( MATCH_COUNT + MISMATCH_COUNT ))" "$( basename "${EXPECT_FILE}" )" >> "${GITHUB_STEP_SUMMARY}" + fi + + if [ "${MISMATCH_COUNT}" -gt 0 ]; then + printf '\n%s result(s) did not match what was expected.\n' "${MISMATCH_COUNT}" + return 1 + fi + + printf '\nEvery result matched what was expected.\n' +} + +if [ "${USE_DOCKER}" = "yes" ]; then + run_in_docker + exit $? +fi + +run_checks