From 3d11df561636ec3953b5db870ff5447e911992b8 Mon Sep 17 00:00:00 2001 From: Eric Mann Date: Tue, 10 Nov 2026 07:21:50 -0800 Subject: [PATCH] Test the full suite below 7.2 only, and the stand-down on trunk The plugin serves WordPress below 7.2. On 7.2 and trunk it stands down and core's implementation serves every call, so the full suite on trunk only tested core with core's tests. It also failed whenever a port got ahead of the nightly build, because the trunk legs take core from the nightly zip and the tests from wordpress-develop trunk. - The test matrix, multisite, examples and smoke jobs run on 7.1.x instead of latest, plus the 6.8 floor leg. A version such as 7.1.x means the newest release on that branch, resolved from the version-check API's offers by the new bin/resolve-wp-version.sh, which both installers use. The matrix follows 7.1 security releases and stays on 7.1 when 7.2 ships. - A stand-down job runs on trunk, PHP 7.4 and 8.5: make test-stand-down runs the stand-down group. The new tests check that core declares the API, that the plugin-only layer is not loaded, that the plugin sets no store, and that the superseded notice is hooked. The no-op gate tests join the group. With SECRETS_API_TESTS_EXPECT_CORE set, the new tests fail instead of skip on a WordPress without the API. - The nightly smoke leg stays: it covers the WP-CLI commands on core's implementation (ADR 0014). --- .github/workflows/ci.yml | 73 ++++++++++++++++--- Makefile | 5 +- README.md | 4 +- bin/install-wp-tests.sh | 4 ++ bin/resolve-wp-version.sh | 47 +++++++++++++ bin/smoke-install.sh | 3 + docs/reference/ci.md | 29 +++++--- tests/phpunit/test-secrets-noop-gate.php | 1 + tests/phpunit/test-secrets-stand-down.php | 85 +++++++++++++++++++++++ 9 files changed, 231 insertions(+), 20 deletions(-) create mode 100755 bin/resolve-wp-version.sh create mode 100644 tests/phpunit/test-secrets-stand-down.php diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 699256b..326ee20 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,10 +87,13 @@ jobs: # The 7.4 leg is not optional. Core's floor is 7.4 and src/ has to run there; # `make compat` catches syntax statically, but only a running 7.4 catches a - # runtime behaviour difference. WP trunk is included so a core change that - # breaks this API surfaces here rather than on someone's site — a red trunk - # leg is worth investigating every time rather than being marked - # continue-on-error and quietly ignored. + # runtime behaviour difference. + # + # The suite runs only on WordPress below 7.2, the versions the plugin serves. + # 7.1.x is the newest 7.1 release, resolved by bin/install-wp-tests.sh, so the + # matrix follows 7.1 security releases and doesn't move to 7.2 when "latest" + # does. On a WordPress that ships the API the plugin stands down, and the + # stand-down job below checks that instead. test: name: "PHP ${{ matrix.php }} / WP ${{ matrix.wp }}" needs: static @@ -99,10 +102,10 @@ jobs: fail-fast: false matrix: php: ['7.4', '8.0', '8.3', '8.5'] - wp: ['latest', 'trunk'] + wp: ['7.1.x'] # The declared floor, `Requires at least` in secrets-api.php. src/ relies on # core's polyfills (array_all() arrived in 6.8), so the floor gets a leg of - # its own rather than being assumed from latest. + # its own rather than being assumed from 7.1.x. include: - php: '7.4' wp: '6.8' @@ -145,6 +148,58 @@ jobs: - run: make test + # On WordPress trunk, which ships the Secrets API since r64337, the plugin loads + # none of its own copy, so the full suite would only test core's implementation + # with core's tests. This checks the stand-down instead: core serves the API, + # the plugin-only layer is not loaded, and the superseded notice is hooked. + # `make test-stand-down` fails rather than skips if trunk somehow lacks the API. + # The WP-CLI commands on core's implementation are covered by the nightly smoke + # leg below. + stand-down: + name: "Stands down / PHP ${{ matrix.php }} / WP trunk" + needs: static + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + php: ['7.4', '8.5'] + services: + mysql: + image: mysql:8.0 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: 'yes' + MYSQL_DATABASE: wordpress_test + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval=10s + --health-timeout=5s + --health-retries=5 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up PHP ${{ matrix.php }} + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 + with: + php-version: ${{ matrix.php }} + extensions: sodium, mysqli + coverage: none + tools: composer + + - name: Cache Composer packages + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/composer + key: composer-${{ runner.os }}-php${{ matrix.php }}-${{ hashFiles('composer.lock') }} + restore-keys: composer-${{ runner.os }}-php${{ matrix.php }}- + + - name: Install dependencies and the WordPress test suite + run: make install WP_VERSION=trunk DB_HOST=127.0.0.1 + + - run: make test-stand-down + # Separate from the matrix above rather than crossed into it: multisite # exercises scope selection and capability wiring, not PHP-version-specific # behaviour, so proving it once on the newest supported PHP is the useful @@ -187,7 +242,7 @@ jobs: restore-keys: composer-${{ runner.os }}-php8.3- - name: Install dependencies and the WordPress test suite - run: make install WP_VERSION=latest DB_HOST=127.0.0.1 + run: make install WP_VERSION=7.1.x DB_HOST=127.0.0.1 - run: make test-ms @@ -256,7 +311,7 @@ jobs: restore-keys: composer-${{ runner.os }}-php8.3- - name: Install dependencies and the WordPress test suite - run: make install WP_VERSION=latest DB_HOST=127.0.0.1 + run: make install WP_VERSION=7.1.x DB_HOST=127.0.0.1 - name: Wait for Moto run: | @@ -288,7 +343,7 @@ jobs: fail-fast: false matrix: php: ['7.4', '8.3', '8.5'] - wp: ['latest'] + wp: ['7.1.x'] # A WordPress that ships the Secrets API: the plugin loads none of its own # copy and the commands run on core's implementation (ADR 0014). include: diff --git a/Makefile b/Makefile index dbcae07..cc849b0 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ DB_HOST ?= 127.0.0.1 SMOKE_DB_NAME ?= wordpress_smoke .DEFAULT_GOAL := help -.PHONY: help install lint lint-fix compat analyse test test-ms test-examples coverage reference reference-check ci smoke clean +.PHONY: help install lint lint-fix compat analyse test test-ms test-stand-down test-examples coverage reference reference-check ci smoke clean help: ## Show this help. @grep -hE '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) \ @@ -46,6 +46,9 @@ test: ## Run the single-site suite. test-ms: ## Run the multisite suite. WP_MULTISITE=1 $(VENDOR_BIN)/phpunit -c phpunit-multisite.xml.dist +test-stand-down: ## On a WordPress that ships the API, check that the plugin stands down. + SECRETS_API_TESTS_EXPECT_CORE=1 $(VENDOR_BIN)/phpunit --group stand-down + coverage: ## Run the single-site suite with coverage. $(VENDOR_BIN)/phpunit --coverage-html coverage --coverage-text diff --git a/README.md b/README.md index 1d2d950..2346b91 100644 --- a/README.md +++ b/README.md @@ -171,8 +171,8 @@ Security issues go through [`SECURITY.md`](SECURITY.md) rather than a public iss stores credentials, and a flaw in it is a flaw in the thing protecting everything else. CI (`.github/workflows/ci.yml`) is a thin wrapper around the `make` targets above, running on -github.com's hosted runners: static analysis gates a PHP 7.4/8.0/8.3/8.5 × WordPress latest/trunk -matrix plus a multisite job, a PHP 7.4/8.3/8.5 smoke job driving a real `wp` binary, and an +github.com's hosted runners: static analysis gates a PHP 7.4/8.0/8.3/8.5 × WordPress 7.1.x +matrix plus a multisite job, a job checking that the plugin stands down on trunk, a PHP 7.4/8.3/8.5 smoke job driving a real `wp` binary, and an `examples` job that runs the platform bindings against Moto and Vault service containers. See [`docs/reference/ci.md`](docs/reference/ci.md). diff --git a/bin/install-wp-tests.sh b/bin/install-wp-tests.sh index d29e174..5bb68d6 100755 --- a/bin/install-wp-tests.sh +++ b/bin/install-wp-tests.sh @@ -48,6 +48,10 @@ download() { fi } +# A branch with a trailing ".x", such as 7.1.x, means the newest release on that +# branch. See bin/resolve-wp-version.sh. +WP_VERSION=$("$(dirname "$0")/resolve-wp-version.sh" "$WP_VERSION") + if [[ $WP_VERSION =~ ^[0-9]+\.[0-9]+\-(beta|RC)[0-9]+$ ]]; then WP_BRANCH=${WP_VERSION%\-*} WP_TESTS_TAG="branches/$WP_BRANCH" diff --git a/bin/resolve-wp-version.sh b/bin/resolve-wp-version.sh new file mode 100755 index 0000000..47e91fb --- /dev/null +++ b/bin/resolve-wp-version.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# +# Prints the WordPress version to install for a requested version. +# +# A branch with a trailing ".x", such as 7.1.x, means the newest release on that +# branch. The version-check API lists the newest release of every supported branch +# among its offers, so a CI matrix pinned to 7.1.x follows 7.1 security releases and +# never moves to 7.2. Anything else, such as latest, trunk, nightly, 6.8 or 7.1.4, +# is printed unchanged. +# +# WP_API_BASE substituted for https://api.wordpress.org (see docs/reference/ci.md) +# +set -euo pipefail + +if [ $# -ne 1 ]; then + echo "usage: $0 " >&2 + exit 1 +fi + +REQUESTED=$1 + +if [[ ! $REQUESTED =~ ^([0-9]+)\.([0-9]+)\.x$ ]]; then + echo "$REQUESTED" + exit 0 +fi + +BRANCH_PATTERN="\"version\":\"(${BASH_REMATCH[1]}\\.${BASH_REMATCH[2]}\\.[0-9]+)\"" +API_URL="${WP_API_BASE-https://api.wordpress.org}/core/version-check/1.7/" + +if command -v curl >/dev/null 2>&1; then + OFFERS=$(curl -fsSL "$API_URL") +elif command -v wget >/dev/null 2>&1; then + OFFERS=$(wget -qO- "$API_URL") +else + echo "Neither curl nor wget is available." >&2 + exit 1 +fi + +# Matched with bash's own regex rather than grep|head, which under pipefail can be +# killed by SIGPIPE (see bin/install-wp-tests.sh). The offers are newest first, so +# the first match is the newest release on the branch. +if [[ $OFFERS =~ $BRANCH_PATTERN ]]; then + echo "${BASH_REMATCH[1]}" +else + echo "The version-check API at $API_URL lists no release for $REQUESTED." >&2 + exit 1 +fi diff --git a/bin/smoke-install.sh b/bin/smoke-install.sh index 4075544..37d4176 100755 --- a/bin/smoke-install.sh +++ b/bin/smoke-install.sh @@ -26,6 +26,9 @@ DB_USER="${DB_USER:-root}" DB_PASS="${DB_PASS:-}" DB_HOST="${DB_HOST:-127.0.0.1}" WP_VERSION="${WP_VERSION:-latest}" +# A branch with a trailing ".x", such as 7.1.x, means the newest release on that +# branch. See bin/resolve-wp-version.sh. +WP_VERSION=$("$(dirname "$0")/resolve-wp-version.sh" "$WP_VERSION") SMOKE_URL="${SMOKE_URL:-http://smoke.test}" # The PHPUnit suite drops and recreates wordpress_test on every run; this diff --git a/docs/reference/ci.md b/docs/reference/ci.md index 45edc3c..246b135 100644 --- a/docs/reference/ci.md +++ b/docs/reference/ci.md @@ -15,11 +15,16 @@ make compat # PHPCompatibilityWP, testVersion 7.4- make analyse # phpstan, level 10 make test # phpunit, single site make test-ms # phpunit, multisite +make test-stand-down # on a WordPress that ships the API: the plugin stands down make reference-check # docs/reference/ matches the source docblocks make smoke # WP-CLI end to end against a throwaway install in .smoke/ -make ci # all of the above +make ci # all of the above, except test-stand-down ``` +`make test-stand-down` needs a WordPress that ships the Secrets API, such as trunk, so it is not +part of `make ci`. It runs the `stand-down` group with `SECRETS_API_TESTS_EXPECT_CORE=1`, which +makes the stand-down tests fail instead of skip on a WordPress without the API. + ## The always-works path ```sh @@ -44,7 +49,9 @@ make ci ``` `bin/install-wp-tests.sh` takes ` [db-host] [wp-version]` and honours -`WP_TESTS_DIR` and `WP_CORE_DIR`. +`WP_TESTS_DIR` and `WP_CORE_DIR`. A version with a trailing `.x`, such as `7.1.x`, means the newest +release on that branch. `bin/resolve-wp-version.sh` looks it up in the version-check API's offers, +and `bin/smoke-install.sh` uses it too. ### The WP-CLI smoke test @@ -73,7 +80,7 @@ wordpress.org: |---|---| | `WP_MIRROR_BASE` | Substituted for `https://wordpress.org` when downloading WordPress itself | | `WP_TESTS_ZIP_URL` | Full URL to a WordPress archive, overriding version lookup entirely | -| `WP_API_BASE` | Substituted for `https://api.wordpress.org`, used only to resolve what "latest" means | +| `WP_API_BASE` | Substituted for `https://api.wordpress.org`, used only to resolve "latest" and versions such as `7.1.x` | | `WP_DEVELOP_BASE` | Substituted for `https://github.com/WordPress/wordpress-develop`, where the test suite comes from | Three separate hosts, deliberately: a mirror of wordpress.org downloads is not automatically a @@ -94,7 +101,7 @@ reliable egress to Packagist. hosted pipeline. **github.com, hosted runners.** Static analysis gates a PHP 7.4 / 8.0 / 8.3 / 8.5 × WordPress -latest / trunk matrix, plus a multisite job. `shivammathur/setup-php` provides the interpreter and +7.1.x matrix with a WordPress 6.8 floor leg, a multisite job, and a stand-down job on trunk. `shivammathur/setup-php` provides the interpreter and asks for the `sodium` extension by name. The whole API is built on libsodium, so relying on whatever the runner image happens to ship wasn't good enough. The `examples` job is the only one with service containers beyond the database: a pinned Moto instance, which the AWS Secrets Manager @@ -124,12 +131,18 @@ person pasted. | Job | PHP | WordPress | Notes | |---|---|---|---| | `static` | 8.3 | — | lint + compat + analyse. Gates everything else. | -| `test` | 7.4, 8.0, 8.3, 8.5 | latest, trunk | Single site | -| `test-multisite` | 8.3 | latest | Multisite suite | -| `smoke` | 7.4, 8.3, 8.5 | latest | WP-CLI end to end, single site then multisite | -| `examples` | 8.3 | latest | `make test-examples` against Moto (an AWS emulator) and a Vault dev server, both pinned by digest, single site then multisite. Not part of `make ci`. | +| `test` | 7.4, 8.0, 8.3, 8.5; 7.4 | 7.1.x; 6.8 | Single site | +| `stand-down` | 7.4, 8.5 | trunk | `make test-stand-down`: core serves the API, and the plugin loads none of its own layer | +| `test-multisite` | 8.3 | 7.1.x | Multisite suite | +| `smoke` | 7.4, 8.3, 8.5; 8.3 | 7.1.x; nightly | WP-CLI end to end, single site then multisite. The nightly leg runs the commands on core's implementation. | +| `examples` | 8.3 | 7.1.x | `make test-examples` against Moto (an AWS emulator) and a Vault dev server, both pinned by digest, single site then multisite. Not part of `make ci`. | | `reference-docs` | 8.3 | — | `bin/gen-reference.php --check`: the committed docs/reference/ matches the source. No Composer install. | +The full suite runs only on WordPress below 7.2, the versions the plugin serves. On 7.2 and trunk +the plugin stands down and core's implementation serves every call, so the full suite there would +only test core with core's tests, and it fails whenever a port gets ahead of the nightly build. +`7.1.x` rather than `latest` keeps the matrix on 7.1 when 7.2 ships. + The 7.4 leg is not optional. Core's floor is 7.4 and `src/` must run there; PHPCompatibilityWP catches syntax statically, but only a running 7.4 catches runtime behaviour differences. diff --git a/tests/phpunit/test-secrets-noop-gate.php b/tests/phpunit/test-secrets-noop-gate.php index 16a8e04..90031be 100644 --- a/tests/phpunit/test-secrets-noop-gate.php +++ b/tests/phpunit/test-secrets-noop-gate.php @@ -12,6 +12,7 @@ * wp_secrets_api_bootstrap() again exercises the real branches with no simulation. * * @group secrets + * @group stand-down */ class Tests_Secrets_NoopGate extends WP_UnitTestCase { diff --git a/tests/phpunit/test-secrets-stand-down.php b/tests/phpunit/test-secrets-stand-down.php new file mode 100644 index 0000000..3ff1c4b --- /dev/null +++ b/tests/phpunit/test-secrets-stand-down.php @@ -0,0 +1,85 @@ +assertSame( + realpath( ABSPATH . WPINC . '/secrets.php' ), + realpath( ( new ReflectionFunction( 'wp_get_secret' ) )->getFileName() ), + 'wp_get_secret() should be declared by core.' + ); + + foreach ( array( 'WP_Secret', 'WP_Secrets_Provider', 'WP_Secrets_Keyring', 'WP_Secrets_Key_Manager', 'WP_Secrets_Libsodium_Provider' ) as $name ) { + $file = ( new ReflectionClass( $name ) )->getFileName(); + + $this->assertIsString( $file, "{$name} should be declared in a file." ); + $this->assertStringStartsWith( $core_dir . DIRECTORY_SEPARATOR, realpath( $file ), "{$name} should be declared by core, not the plugin." ); + } + } + + /** + * Ensure that the plugin loads none of its plugin-only layer. + * + * @covers ::wp_secrets_api_bootstrap + */ + public function test_the_plugin_only_layer_is_not_loaded(): void { + foreach ( array( 'Secrets_API_Migrator', 'Secrets_API_Legacy_Reader', 'Secrets_API_Prototype_Fallback_Store' ) as $name ) { + $this->assertFalse( class_exists( $name, false ), "{$name} should not be loaded when core ships the API." ); + } + } + + /** + * Ensure that the plugin installs no store of its own in front of core's. + * + * @covers ::wp_secrets_api_bootstrap + */ + public function test_the_plugin_sets_no_store(): void { + $store = $GLOBALS['wp_secrets_store'] ?? null; + + $this->assertNull( $store, 'The plugin should leave the store global to core and the drop-in.' ); + } + + /** + * Ensure that the plugin says it has been superseded, and reports no conflict. + * + * @covers ::wp_secrets_api_bootstrap + */ + public function test_the_superseded_notice_is_hooked(): void { + $this->assertSame( 10, has_action( 'admin_notices', 'wp_secrets_api_notice_superseded' ), 'The superseded notice should be hooked.' ); + $this->assertFalse( has_action( 'admin_notices', 'wp_secrets_api_notice_conflict' ), 'No conflict notice should be hooked.' ); + } +}