Build/Test Tools: Test the whole plugin directory locally against a baseline.

.github/workflows/reusable-plugin-compatibility.yml

@@ -0,0 +1,137 @@
+##
+# A reusable workflow that installs a version of WordPress and then checks that a list of plugins can be
+# activated against it without fataling.
+#
+# Each plugin in the `plugin-slugs` shard is tested on its own: it is installed, activated, exercised, and then
+# the site is reset to the fresh install before the next one is installed. This keeps one broken plugin from
+# masking (or breaking) the next.
+#
+# The exception is a plugin that declares a `Requires Plugins` dependency, which core will not activate until
+# that dependency is installed and active. Those dependencies are installed and activated alongside it, and the
+# site is checked with only the dependencies active first so that a dependency's own breakage is not reported
+# against the plugin under test.
+#
+# The checks themselves live in `tools/plugin-compatibility/test-plugins.sh` rather than in this file, so that
+# the same code can be run from a terminal to reproduce a result from a run here. See the comments at the top
+# of that script.
+##
+name: Plugin Compatibility Tests
+
+on:
+  workflow_call:
+    inputs:
+      wp-version:
+        description: 'The version of WordPress to test against. Accepts a version number, "latest", or "nightly".'
+        required: false
+        type: 'string'
+        default: 'nightly'
+      php-version:
+        description: 'The version of PHP to use. Expected format: X.Y.'
+        required: false
+        type: 'string'
+        default: '8.5'
+      plugin-slugs:
+        description: 'A JSON array of WordPress.org plugin slugs to test in this shard. Ignored when fixtures is true.'
+        required: false
+        type: 'string'
+        default: '[]'
+      fixtures:
+        description: 'Test the fixture plugins in tools/plugin-compatibility/fixtures, which are built to break, and pass only when each gets the result expected for it.'
+        required: false
+        type: 'boolean'
+        default: false
+      db-type:
+        description: 'Database type. Valid types are mysql and mariadb.'
+        required: false
+        type: 'string'
+        default: 'mysql'
+      db-version:
+        description: 'Database version.'
+        required: false
+        type: 'string'
+        default: '9.7'
+
+# Disable permissions for all available scopes by default.
+# Any needed permissions should be configured at the job level.
+permissions: {}
+
+jobs:
+  # Tests that each plugin in the shard can be activated against the given version of WordPress.
+  #
+  # Performs the following steps:
+  # - Checks out the repository, for the script that runs the checks.
+  # - Sets up PHP and WP-CLI.
+  # - Runs the checks against every plugin in the shard.
+  plugin-compatibility-tests:
+    name: PHP ${{ inputs.php-version }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }}
+    permissions:
+      contents: read
+    runs-on: ubuntu-24.04
+    # Installing a plugin's dependencies before testing it adds a download and an activation for each
+    # one, and a dependency the size of WooCommerce is not quick to activate.
+    timeout-minutes: 45
+
+    services:
+      database:
+        # The database type and version are inputs so that this workflow can be pointed at any supported
+        # combination, which means the image cannot be pinned to a digest. This matches how the database
+        # service is declared in install-testing.yml and reusable-upgrade-testing.yml.
+        image: ${{ inputs.db-type }}:${{ inputs.db-version }} # zizmor: ignore[unpinned-images]
+        ports:
+          - 3306
+        options: >-
+          --health-cmd="mysqladmin ping"
+          --health-interval="30s"
+          --health-timeout="10s"
+          --health-retries="5"
+          -e MYSQL_ROOT_PASSWORD="root"
+          -e MYSQL_DATABASE="test_db"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          show-progress: ${{ runner.debug == '1' && 'true' || 'false' }}
+          persist-credentials: false
+          sparse-checkout: tools/plugin-compatibility
+          sparse-checkout-cone-mode: false
+
+      - name: Set up PHP ${{ inputs.php-version }}
+        uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2
+        with:
+          php-version: '${{ inputs.php-version }}'
+          coverage: none
+          tools: wp-cli
+
+      # The runner already provides PHP, WP-CLI and a database, so the checks run here rather than in the
+      # containers the script provisions when it is run from a terminal.
+      #
+      # WordPress is installed outside the workspace so that the checkout above is not part of the site under
+      # test.
+      - name: Test each plugin in isolation
+        env:
+          PLUGIN_SLUGS: ${{ inputs.plugin-slugs }}
+          FIXTURES: ${{ inputs.fixtures }}
+          WP_VERSION: ${{ inputs.wp-version }}
+          DB_PORT: ${{ job.services.database.ports['3306'] }}
+        run: |
+          set -uo pipefail
+
+          if [ "${FIXTURES}" = "true" ]; then
+            SELECTION=(
+              --fixtures=./tools/plugin-compatibility/fixtures
+              --expect=./tools/plugin-compatibility/fixtures/expected.txt
+            )
+          else
+            SELECTION=( --plugins="$( printf '%s' "${PLUGIN_SLUGS}" | jq -r 'join( " " )' )" )
+          fi
+
+          ./tools/plugin-compatibility/test-plugins.sh \
+            --no-docker \
+            --wp-version="${WP_VERSION}" \
+            --dir="${RUNNER_TEMP}/wordpress" \
+            --db-host="127.0.0.1:${DB_PORT}" \
+            --db-name=test_db \
+            --db-user=root \
+            --db-pass=root \
+            "${SELECTION[@]}"

tools/plugin-compatibility/README.md

@@ -0,0 +1,60 @@
+# Plugin compatibility tests
+
+Checks that plugins from the WordPress.org directory can be activated against a version of WordPress without fataling. The Plugin Compatibility Tests workflow runs these scripts against the most popular plugins. The same scripts run locally, which is how to test far more plugins than CI should.
+
+| Script | What it does |
+| --- | --- |
+| `test-plugins.sh` | Tests a list of plugins against one version of WordPress. This is what each workflow shard runs. |
+| `list-plugins.sh` | Prints plugin slugs from the directory: the most popular, all of them, or those updated since a date. |
+| `compare-results.sh` | Compares a baseline run with a candidate run and reports the plugins that regressed. |
+| `test-all-plugins.sh` | Runs all of the above for a large list, in parallel, resumable. |
+| `fixtures/` | Plugins built to break in one specific way each, with the result each must get. |
+
+All of them need Docker. The list scripts also need curl and jq.
+
+## Comparing against a baseline
+
+Across the whole directory, thousands of plugins fail on a current PHP whatever version of WordPress they run on. A list of failures from one run is mostly noise. What matters before a release is the plugins that **pass on the current release and fail on the next one**.
+
+So a large run tests every plugin twice, and compares:
+
+```
+plugins.txt ──┬── test-plugins.sh --wp-version=latest  ──► latest.tsv  ──┐
+              │                                                          ├── compare-results.sh ──► report.md
+              └── test-plugins.sh --wp-version=7.2-RC1 ──► 7.2-RC1.tsv ──┘
+```
+
+The report lists plugins that regressed (passed on the baseline, failed on the candidate) and plugins the candidate fixed. Plugins that fail on both are counted but not listed. A plugin whose version changed between the two runs is flagged, since the plugin changed too.
+
+## Testing the whole directory
+
+```
+# The 5,000 most popular plugins, 8 at a time, against 7.2-RC1 compared with the current release.
+tools/plugin-compatibility/test-all-plugins.sh --count=5000 --wp-version=7.2-RC1 --jobs=8
+
+# Every plugin updated in the last two years.
+tools/plugin-compatibility/test-all-plugins.sh --updated-since=2024-11-01 --wp-version=7.2-RC1 --jobs=8
+
+# Everything.
+tools/plugin-compatibility/test-all-plugins.sh --all --wp-version=7.2-RC1 --jobs=8
+```
+
+Everything is written to `plugin-compatibility-results/`: the plugin list, a results file and log per copy, the download cache, and `report.md`. If the run is interrupted, run the same command again to carry on where it stopped. Pass an exact version rather than `nightly` for a run that may be resumed on another day, because nightly moves.
+
+Rough sizes, at the roughly 7 seconds per plugin measured on the most popular plugins. Smaller plugins are likely quicker.
+
+| List | Plugins | One version, 8 jobs | Baseline and candidate, 8 jobs |
+| --- | --- | --- | --- |
+| `--count=1000` | 1,000 | 15 minutes | 30 minutes |
+| `--updated-since` two years ago | about 42,000 | 10 hours | 20 hours |
+| `--all` | about 78,000 | 19 hours | 38 hours |
+
+Each job runs its own database and PHP containers, so memory is usually what limits `--jobs`.
+
+## Testing a few plugins
+
+```
+tools/plugin-compatibility/test-plugins.sh --wp-version=nightly woocommerce classic-editor
+```
+
+Run `test-plugins.sh --help` for every option, including `--results` and `--cache-dir` for long runs and `--no-docker` to use a PHP, WP-CLI and database you already have.

tools/plugin-compatibility/compare-results.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+##
+# Compares two plugin compatibility runs and prints the plugins whose result changed between them.
+#
+# Across the whole directory, thousands of plugins fail on a current PHP whatever version of WordPress they
+# run on, so the list of failures from a single run is mostly noise. What matters before a release is the
+# plugins that pass on the current release and fail on the next one. This finds those by comparing the
+# results files `test-plugins.sh --results` writes for each version:
+#
+#     tools/plugin-compatibility/compare-results.sh latest.tsv nightly.tsv
+#
+# Prints a markdown report, and exits 1 when at least one plugin regressed.
+##
+
+set -uo pipefail
+
+usage() {
+	cat <<'USAGE_EOF'
+Usage: compare-results.sh <baseline results> <candidate results>
+
+Compares two results files written by test-plugins.sh --results, such as one run against the current
+release and one against nightly, and prints a markdown report of what changed.
+
+  Regressed     passed on the baseline, failed on the candidate. These are the ones to look at.
+  Fixed         failed on the baseline, passed on the candidate.
+  Fail on both  failed either way, so not caused by the candidate. Counted, not listed.
+
+Plugins that were not tested in both runs, or were skipped in either, are counted but not compared. A
+regressed plugin whose version differs between the runs is flagged, since the plugin changed as well.
+
+Exits 1 when a plugin regressed and 0 otherwise.
+USAGE_EOF
+}
+
+case "${1:-}" in
+	-h | --help ) usage; exit 0 ;;
+esac
+
+if [ "$#" -ne 2 ]; then
+	usage >&2
+	exit 2
+fi
+
+BASELINE="${1}"
+CANDIDATE="${2}"
+
+for FILE in "${BASELINE}" "${CANDIDATE}"; do
+	[ -s "${FILE}" ] || { printf 'Error: %s does not exist or is empty.\n' "${FILE}" >&2; exit 2; }
+done
+
+# One row per plugin that is in both files: slug, baseline version, baseline result, candidate version,
+# candidate result, candidate reason. A plugin listed twice in a file keeps its last row, which is the one a
+# resumed run recorded.
+JOINED="$( awk -F '\t' -v OFS='\t' '
+	NR == FNR { version[$1] = $2; status[$1] = $3; next }
+	{ cand_version[$1] = $2; cand_status[$1] = $3; cand_reason[$1] = $5 }
+	END {
+		for ( slug in cand_status ) {
+			if ( slug in status ) {
+				print slug, version[slug], status[slug], cand_version[slug], cand_status[slug], cand_reason[slug]
+			}
+		}
+	}
+' "${BASELINE}" "${CANDIDATE}" | sort )"
+
+BASELINE_TOTAL="$( cut -f 1 "${BASELINE}" | sort -u | grep -c . )"
+CANDIDATE_TOTAL="$( cut -f 1 "${CANDIDATE}" | sort -u | grep -c . )"
+BOTH_TOTAL="$( printf '%s' "${JOINED}" | grep -c . )"
+
+REGRESSED="$( printf '%s\n' "${JOINED}" | awk -F '\t' '$3 == "PASS" && $5 == "FAIL"' )"
+FIXED="$( printf '%s\n' "${JOINED}" | awk -F '\t' '$3 == "FAIL" && $5 == "PASS"' )"
+REGRESSED_COUNT="$( printf '%s' "${REGRESSED}" | grep -c . )"
+FIXED_COUNT="$( printf '%s' "${FIXED}" | grep -c . )"
+BOTH_FAIL_COUNT="$( printf '%s\n' "${JOINED}" | awk -F '\t' '$3 == "FAIL" && $5 == "FAIL" { n++ } END { print n + 0 }' )"
+BOTH_PASS_COUNT="$( printf '%s\n' "${JOINED}" | awk -F '\t' '$3 == "PASS" && $5 == "PASS" { n++ } END { print n + 0 }' )"
+NOT_COMPARED_COUNT="$(( BOTH_TOTAL - REGRESSED_COUNT - FIXED_COUNT - BOTH_FAIL_COUNT - BOTH_PASS_COUNT ))"
+
+printf '## Plugin compatibility: %s compared with %s\n\n' "$( basename "${CANDIDATE}" )" "$( basename "${BASELINE}" )"
+printf '%s plugins in the baseline, %s in the candidate, %s in both.\n\n' "${BASELINE_TOTAL}" "${CANDIDATE_TOTAL}" "${BOTH_TOTAL}"
+printf '| Regressed | Fixed | Pass on both | Fail on both | Skipped in either |\n'
+printf '| --- | --- | --- | --- | --- |\n'
+printf '| %s | %s | %s | %s | %s |\n\n' "${REGRESSED_COUNT}" "${FIXED_COUNT}" "${BOTH_PASS_COUNT}" "${BOTH_FAIL_COUNT}" "${NOT_COMPARED_COUNT}"
+
+print_rows() {
+	printf '| Plugin | Version | Details |\n'
+	printf '| --- | --- | --- |\n'
+
+	printf '%s\n' "${1}" | awk -F '\t' 'NF > 0 {
+		version = ( $2 == $4 ) ? $4 : $2 " to " $4 " (plugin changed too)"
+		details = ( $6 == "-" ) ? "" : $6
+		gsub( /\|/, "\\|", details )
+		printf "| [%s](https://wordpress.org/plugins/%s/) | %s | %s |\n", $1, $1, version, details
+	}'
+
+	printf '\n'
+}
+
+if [ "${REGRESSED_COUNT}" -gt 0 ]; then
+	printf '### Regressed\n\nPassed on the baseline and failed on the candidate.\n\n'
+	print_rows "${REGRESSED}"
+else
+	printf 'No plugin that passed on the baseline failed on the candidate.\n\n'
+fi
+
+if [ "${FIXED_COUNT}" -gt 0 ]; then
+	printf '### Fixed\n\nFailed on the baseline and passed on the candidate.\n\n'
+	print_rows "${FIXED}"
+fi
+
+[ "${REGRESSED_COUNT}" -eq 0 ]

tools/plugin-compatibility/fixtures/expected.txt

@@ -0,0 +1,31 @@
+# The result each fixture plugin is expected to get from test-plugins.sh, in the order they are tested.
+#
+#     test-plugins.sh --fixtures=tools/plugin-compatibility/fixtures --expect=tools/plugin-compatibility/fixtures/expected.txt
+#
+# Each fixture is a plugin built to break in one specific way, so that a change to the checks which stops a
+# broken plugin from being caught fails a run instead of quietly passing everything.
+#
+# Format: <slug> <PASS|FAIL|SKIPPED> [text the "Also active" or "Details" column must contain]
+
+fixture-healthy                    PASS
+fixture-fatal-on-load              FAIL     The plugin could not be activated
+fixture-fatal-on-activation        FAIL     The plugin could not be activated
+fixture-exit-on-activation         SKIPPED  the plugin was not active afterwards
+fixture-breaks-site-on-activation  FAIL     The request to / returned HTTP 500
+fixture-requires-newer-php         SKIPPED  declared requirements are not met
+fixture-requires-newer-wordpress   SKIPPED  declared requirements are not met
+fixture-requires-newer-both        SKIPPED  declared requirements are not met
+fixture-fatal-on-front-end         FAIL     The request to / returned HTTP 500
+fixture-fatal-on-login             FAIL     The request to /wp-login.php returned HTTP 500
+fixture-fatal-after-output         FAIL     PHP Fatal error
+fixture-parse-error-after-output   FAIL     PHP Parse error
+fixture-fatal-in-background-request FAIL    PHP Fatal error
+fixture-wp-cli-only-fatal          PASS     WP-CLI fatals when it loads WordPress
+fixture-requires-akismet           PASS     akismet
+fixture-requires-broken-dependency SKIPPED  The required plugins are not healthy on their own
+fixture-requires-wp-cli-only-fatal SKIPPED  WP-CLI cannot load WordPress with the required plugins active
+
+# These two must stay together and in this order: the second fatals if anything the first wrote survived the
+# reset between plugins.
+fixture-leaves-files-behind        PASS
+fixture-checks-for-leftovers       PASS

tools/plugin-compatibility/fixtures/fixture-breaks-site-on-activation/fixture-breaks-site-on-activation.php

@@ -0,0 +1,14 @@
+<?php
+/**
+ * Plugin Name: Fixture: Breaks the site on activation
+ * Description: Writes a must-use plugin that fatals when it is activated, so every request fails afterwards, even one that skips regular plugins. Expected to fail.
+ * Version: 1.0.0
+ */
+
+register_activation_hook(
+	__FILE__,
+	static function () {
+		wp_mkdir_p( WPMU_PLUGIN_DIR );
+		file_put_contents( WPMU_PLUGIN_DIR . '/fixture-broken-loader.php', "<?php\nfixture_function_that_does_not_exist();\n" );
+	}
+);

tools/plugin-compatibility/fixtures/fixture-broken-dependency/fixture-broken-dependency.php

@@ -0,0 +1,8 @@
+<?php
+/**
+ * Plugin Name: Fixture: Broken dependency
+ * Description: Fatals on the front end. Only tested as a dependency of fixture-requires-broken-dependency.
+ * Version: 1.0.0
+ */
+
+add_action( 'template_redirect', 'fixture_function_that_does_not_exist' );

tools/plugin-compatibility/fixtures/fixture-checks-for-leftovers/fixture-checks-for-leftovers.php

@@ -0,0 +1,19 @@
+<?php
+/**
+ * Plugin Name: Fixture: Checks for leftovers
+ * Description: Fatals if anything from fixture-leaves-files-behind survived the reset. Expected to pass when tested straight after it.
+ * Version: 1.0.0
+ */
+
+add_action(
+	'template_redirect',
+	static function () {
+		global $wpdb;
+
+		if ( file_exists( ABSPATH . 'fixture-leftover.txt' )
+			|| get_option( 'fixture_leftover' )
+			|| $wpdb->get_var( "SHOW TABLES LIKE '{$wpdb->prefix}fixture_leftover'" ) ) {
+			fixture_function_that_does_not_exist();
+		}
+	}
+);

tools/plugin-compatibility/fixtures/fixture-exit-on-activation/fixture-exit-on-activation.php

@@ -0,0 +1,13 @@
+<?php
+/**
+ * Plugin Name: Fixture: Exit on activation
+ * Description: Exits from its activation hook, before core records it as active. Expected to be skipped rather than pass.
+ * Version: 1.0.0
+ */
+
+register_activation_hook(
+	__FILE__,
+	static function () {
+		exit;
+	}
+);