From 429bc8bf7f147152c324f045e0e483c3b6feecf0 Mon Sep 17 00:00:00 2001 From: Jonathan Hefner Date: Tue, 29 Sep 2026 13:19:19 -0700 Subject: [PATCH 1/3] Record the suite version in saved reports Saved reports identify the specification target but not the suite version that evaluated their evidence. Read the installed reporting plugin manifest relative to its module and include its version as suiteVersion, so a saved result can be traced to the evaluator version used. Keep schemaVersion 1 and accept older reports without the additive field. Verify that copied plugins report their own manifest version, and check that all four plugin versions match the root development package. --- plugins/agent-plugins-conformance/README.md | 3 ++ .../src/report-format.mjs | 1 + .../agent-plugins-conformance/src/report.mjs | 6 ++++ test/record-report.test.mjs | 29 +++++++++++++++++-- test/report.test.mjs | 14 +++++++++ 5 files changed, 51 insertions(+), 2 deletions(-) diff --git a/plugins/agent-plugins-conformance/README.md b/plugins/agent-plugins-conformance/README.md index 28e389b..bdbb892 100644 --- a/plugins/agent-plugins-conformance/README.md +++ b/plugins/agent-plugins-conformance/README.md @@ -26,6 +26,7 @@ This abbreviated example shows three results and their summary counts; observati ```json { "schemaVersion": 1, + "suiteVersion": "0.1.0", "specVersion": "1.0.0", "results": [ { @@ -59,6 +60,8 @@ This abbreviated example shows three results and their summary counts; observati } ``` +`suiteVersion` records the reporting plugin's version to help investigate saved results; `specVersion` identifies the specification being tested, and `schemaVersion` identifies the report format. Older reports may omit `suiteVersion`. + Each object in `results` identifies its case through `id` and `label`, explains the outcome in `detail`, and cites the relevant specification sections in `specSections`. A result object may also contain a `warning` describing a probe cleanup failure, including the probe file's path and error. This warning does not change the result's `status`. The three statuses mean: diff --git a/plugins/agent-plugins-conformance/src/report-format.mjs b/plugins/agent-plugins-conformance/src/report-format.mjs index a294e3f..abb967f 100644 --- a/plugins/agent-plugins-conformance/src/report-format.mjs +++ b/plugins/agent-plugins-conformance/src/report-format.mjs @@ -22,6 +22,7 @@ export function validateSavedReport(report) { object(report, 'report'); if (report.schemaVersion !== 1) invalid('report.schemaVersion', 'expected 1'); string(report.specVersion, 'report.specVersion'); + if (Object.hasOwn(report, 'suiteVersion')) string(report.suiteVersion, 'report.suiteVersion'); if (!Array.isArray(report.observations)) invalid('report.observations', 'expected an array'); if (!Array.isArray(report.results)) invalid('report.results', 'expected an array'); const counts = { pass: 0, fail: 0, not_verified: 0, total: report.results.length }; diff --git a/plugins/agent-plugins-conformance/src/report.mjs b/plugins/agent-plugins-conformance/src/report.mjs index a303dcd..fb5f55a 100644 --- a/plugins/agent-plugins-conformance/src/report.mjs +++ b/plugins/agent-plugins-conformance/src/report.mjs @@ -1,7 +1,12 @@ +import { readFileSync } from 'node:fs'; import path from 'node:path'; import { CASES, MCP_CWD_VARIANTS } from './cases.mjs'; export { CASES, CASE_IDS } from './cases.mjs'; +const SUITE_VERSION = JSON.parse(readFileSync(new URL('../plugin.json', import.meta.url), 'utf8')).version; +if (typeof SUITE_VERSION !== 'string' || !SUITE_VERSION.trim()) { + throw new TypeError('plugin.json.version: expected a nonempty string'); +} const CORE_SERVERS = Object.keys(MCP_CWD_VARIANTS); const COMMAND_TOKEN_SERVERS = Object.freeze({ 'command-token-posix': Object.freeze({ @@ -847,6 +852,7 @@ export function buildReport(input) { return { schemaVersion: 1, specVersion: '1.0.0', + suiteVersion: SUITE_VERSION, observations: [ ...Object.keys(SKILLS).filter((skill) => skills.has(skill)).map((skill) => ({ kind: 'skill', skill, marker: skills.get(skill), diff --git a/test/record-report.test.mjs b/test/record-report.test.mjs index 6536cc5..dfad025 100644 --- a/test/record-report.test.mjs +++ b/test/record-report.test.mjs @@ -668,6 +668,8 @@ test('malformed saved state is rejected even when its bad observation would be r await mkdir(join(f.directory, 'nested directory')); const malformed = [ '{', 'null', JSON.stringify({ schemaVersion: 1, observations: [] }), + JSON.stringify({ ...expected([]), suiteVersion: '' }), + JSON.stringify({ ...expected([]), suiteVersion: 123 }), JSON.stringify({ ...expected([]), summary: { pass: 999, fail: 0, not_verified: 0, total: 999 } }), JSON.stringify({ ...expected([skill()]), observations: [{ ...skill(), unexpected: true }] }), ]; @@ -680,6 +682,21 @@ test('malformed saved state is rejected even when its bad observation would be r } }); +test('legacy saved reports remain readable and acquire suite version when recorded again', async (t) => { + const f = await fixture(t); + await mkdir(join(f.directory, 'nested directory')); + const legacy = expected([skill('alpha')]); + delete legacy.suiteVersion; + await writeFile(f.outputPath, `${JSON.stringify(legacy)}\n`); + const summaryScript = fileURLToPath(new URL( + '../plugins/agent-plugins-conformance/skills/run-conformance/scripts/summarize.mjs', import.meta.url)); + const summary = spawnSync(process.execPath, [summaryScript, f.outputPath], { encoding: 'utf8' }); + assert.equal(summary.status, 0, summary.stderr); + assert.equal(summary.stdout.trimEnd(), formatReport(legacy)); + f.record(skill('beta')); + assert.deepEqual(await f.read(), expected([skill('alpha'), skill('beta')])); +}); + test('write errors report failure and remove sibling temporary files', async (t) => { const f = await fixture(t); await mkdir(f.outputPath, { recursive: true }); @@ -699,14 +716,19 @@ test('copied primary plugin records and summarizes without sibling plugins or ru const f = await fixture(t); const copiedPlugin = join(f.directory, 'copied plugin'); await cp(new URL('../plugins/agent-plugins-conformance', import.meta.url), copiedPlugin, { recursive: true }); + const manifestPath = join(copiedPlugin, 'plugin.json'); + const manifest = JSON.parse(await readFile(manifestPath, 'utf8')); + const copiedVersion = '9.8.7'; + await writeFile(manifestPath, JSON.stringify({ ...manifest, version: copiedVersion })); assert.equal((await readdir(copiedPlugin)).includes('node_modules'), false); const copiedScript = join(copiedPlugin, 'skills/run-conformance/scripts/report.mjs'); f.success(start, copiedScript); + assert.equal((await f.read()).suiteVersion, copiedVersion); f.success({ action: 'record', observation: skill('alpha') }, copiedScript); f.success({ action: 'record', observation: skill('beta') }, copiedScript); f.success({ action: 'record', observation: discovery() }, copiedScript); const report = await f.read(); - assert.deepEqual(report, expected([skill('alpha'), skill('beta'), discovery()])); + assert.deepEqual(report, { ...expected([skill('alpha'), skill('beta'), discovery()]), suiteVersion: copiedVersion }); assert.equal(report.results.find(({ id }) => id === 'skills.discovery.immediate-children').status, 'pass'); const summary = spawnSync(process.execPath, [join(copiedPlugin, 'skills/run-conformance/scripts/summarize.mjs'), f.outputPath], { cwd: f.directory, encoding: 'utf8', timeout: 10_000, @@ -718,7 +740,10 @@ test('copied primary plugin records and summarizes without sibling plugins or ru const result = health.run({ action: 'record', observation: missingHttp() }, copiedScript); assert.equal(result.status, 0, result.stderr); const withHttp = await f.read(); - assert.deepEqual(withHttp, expected([skill('alpha'), skill('beta'), discovery(), { ...missingHttp(), serverHealthCheck: 'passed' }])); + assert.deepEqual(withHttp, { + ...expected([skill('alpha'), skill('beta'), discovery(), { ...missingHttp(), serverHealthCheck: 'passed' }]), + suiteVersion: copiedVersion, + }); // Reading saved evidence must remain deterministic after fixture state changes. await health.setResponse({ error: 'fixture stopped after collection' }); const savedSummary = health.run('', join(copiedPlugin, 'skills/run-conformance/scripts/summarize.mjs')); diff --git a/test/report.test.mjs b/test/report.test.mjs index 537aaf4..68d8c8f 100644 --- a/test/report.test.mjs +++ b/test/report.test.mjs @@ -1,8 +1,21 @@ import test from 'node:test'; import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; import path from 'node:path'; import { buildReport, CASE_IDS } from '../plugins/agent-plugins-conformance/src/report.mjs'; +const rootVersion = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version; + +test('all four plugin manifests use the root package version', () => { + for (const name of [ + 'agent-plugins-conformance', 'agent-plugins-conformance-core', + 'agent-plugins-conformance-recovery', 'agent-plugins-conformance-invalid-mcp', + ]) { + const manifest = JSON.parse(readFileSync(new URL(`../plugins/${name}/plugin.json`, import.meta.url), 'utf8')); + assert.equal(manifest.version, rootVersion, name); + } +}); + function input(root = '/fixture/plugin', data = '/state/plugin') { const flavor = root.startsWith('/') ? path.posix : path.win32; const platform = flavor === path.win32 ? 'windows' : 'posix'; @@ -95,6 +108,7 @@ test('complete stdio and skill core evidence passes its cases and preserves cano assert.deepEqual(report.summary, { pass: 19, fail: 0, not_verified: 39, total: 58 }); assert.deepEqual(report.results.map(({ id }) => id), CASE_IDS); assert.equal(report.specVersion, '1.0.0'); + assert.equal(report.suiteVersion, rootVersion); assert.deepEqual(result(report, 'mcp.stdio.env.plugin-root').specSections, ['9.1']); assert.deepEqual(report.observations, value.observations); runtime(value).argv.push('after report'); From 44ad7262c6d14f6430907c9477dc73ef27f2ced1 Mon Sep 17 00:00:00 2001 From: Jonathan Hefner Date: Tue, 29 Sep 2026 13:19:23 -0700 Subject: [PATCH 2/3] Define coordinated suite releases Release the four plugins together under a shared suite version and immutable annotated tags, independently of the specification version. Each release targets one exact published specification release. Define stable compatibility around documented report fields, check IDs, and consumer prerequisites while keeping collection protocols internal. Replacing the specification target is breaking because clients supporting the older schema identifiers need not recognize the newer ones. Pin exact suite releases for repeatable acceptance testing, since added coverage and corrective fixes can change findings. --- README.md | 2 +- docs/release-policy.md | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 docs/release-policy.md diff --git a/README.md b/README.md index 7ccd421..c397f16 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ The suite includes these plugins: To include the optional Streamable HTTP and legacy HTTP+SSE checks, follow the [Core fixture's HTTP setup](plugins/agent-plugins-conformance-core/#optional-http-setup) **before the client loads the plugins**. This starts a bundled local server in a separate terminal; CI can launch the same command. Without the server, HTTP checks remain `not_verified` and the agent continues collecting the other checks. -Install and load the plugins through the client's Agent Plugins support, then ask the agent: +Install and load the plugins from the same [release tag](docs/release-policy.md#release-tags) through the client's Agent Plugins support, then ask the agent: > Run the run-conformance skill from Agent Plugins Conformance and save the JSON report to /tmp/agent-plugins-conformance/report.json. diff --git a/docs/release-policy.md b/docs/release-policy.md new file mode 100644 index 0000000..6041029 --- /dev/null +++ b/docs/release-policy.md @@ -0,0 +1,31 @@ +# Release policy + +The suite’s plugins are released together as one suite. Their `plugin.json` versions and the root development package version match the suite release. Install the plugins from the same release tag. + +Suite versions advance independently of Agent Plugins specification versions. Each suite release targets one exact published specification release; several suite releases can improve coverage of the same specification. + +## Versioning + +The suite uses [Semantic Versioning](https://semver.org/). Before `1.0.0`, consumer-facing interfaces are considered unstable. For stable releases: + +| Change | Increment | +| --- | --- | +| Correct a check, improve collection guidance, or fix reporting without breaking a documented consumer interface | Patch | +| Add checks or other functionality while preserving existing consumer interfaces and prerequisites | Minor | +| Break the documented saved-report format, remove or redefine check IDs, change required installation or runtime prerequisites incompatibly, or replace the targeted specification version | Major | + +The collection interface between the guide and reporter, and the protocols between suite components, are internal. They can change together without requiring a major release. The documented saved-report fields and check IDs are interfaces that consumers use for acceptance queries. + +A corrective patch can change a verdict. New checks can change the result of an acceptance query that selects an entire ID prefix. Therefore, pin an exact suite release for reproducible acceptance testing. + +## Specification releases + +A new specification tag does not automatically retarget or release the suite. Adopting it requires validating the suite against that published release and keeping the declared schema targets, evaluated requirements, and report's `specVersion` consistent. + +Replacing the target specification is a breaking change because clients supporting the previous target are not required to recognize the new schema identifiers. For stable suite versions, this requires a major increment. Earlier suite tags remain available for testing earlier specification versions. Fixes can be backported when needed; publishing a new target does not promise ongoing maintenance of every older release line. + +## Release tags + +Each suite release has an annotated `vX.Y.Z` tag matching the root package and all plugin versions. Its GitHub release identifies the exact specification tag being targeted. + +Release tags and released plugin contents are immutable; corrections require a new suite version. `main` is development state; use release tags for repeatable runs. From 2af24e7dc700f12dcbbea33ffe544877c5c36f4a Mon Sep 17 00:00:00 2001 From: Jonathan Hefner Date: Tue, 29 Sep 2026 13:26:35 -0700 Subject: [PATCH 3/3] Retry transient Windows report lock removal failures Windows Node 22 and 24 CI exposed EPERM while concurrent writers recovered an expired report lock. Retry removal of the exact owner file and empty lock directory briefly, preserving successor owners and surfacing persistent permission errors. Keep directory removal nonrecursive and add deterministic regressions that fail with the previous implementation and verify that recording recovers without losing observations. --- .../skills/run-conformance/scripts/report.mjs | 40 ++++++++++++++--- test/report-concurrency.test.mjs | 44 ++++++++++++++++++- 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/plugins/agent-plugins-conformance/skills/run-conformance/scripts/report.mjs b/plugins/agent-plugins-conformance/skills/run-conformance/scripts/report.mjs index face5ee..a276587 100644 --- a/plugins/agent-plugins-conformance/skills/run-conformance/scripts/report.mjs +++ b/plugins/agent-plugins-conformance/skills/run-conformance/scripts/report.mjs @@ -82,7 +82,7 @@ async function withReportLock(outputPath, update) { if (owners.length) { const ownerPath = join(lockPath, owners[0]); try { - if (Date.now() - (await stat(ownerPath)).mtimeMs > staleAfter) await unlink(ownerPath); + if (Date.now() - (await stat(ownerPath)).mtimeMs > staleAfter) await removeLockOwner(ownerPath); } catch (cleanupError) { if (cleanupError.code !== 'ENOENT') throw cleanupError; } @@ -97,7 +97,7 @@ async function withReportLock(outputPath, update) { try { await update(); } finally { - await rm(join(lockPath, owner), { force: true }); + await removeLockOwner(join(lockPath, owner)); await removeEmptyLock(lockPath); } } finally { @@ -105,11 +105,39 @@ async function withReportLock(outputPath, update) { } } +async function removeLockOwner(ownerPath) { + for (let attempt = 0; ; attempt++) { + try { + await unlink(ownerPath); + return; + } catch (error) { + if (error.code === 'ENOENT') return; + if (error.code !== 'EPERM' || attempt === 10) throw error; + // Retry Windows permission errors briefly on this exact owner path. + await setTimeout(100); + } + } +} + async function removeEmptyLock(lockPath) { - try { - await rmdir(lockPath); - } catch (error) { - if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error.code)) throw error; + for (let attempt = 0; ; attempt++) { + try { + await rmdir(lockPath); + return; + } catch (error) { + if (['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error.code)) return; + if (error.code !== 'EPERM') throw error; + // Retry Windows permission errors briefly when removing the empty lock. + // A new owner may also have filled the directory in the meantime. + try { + if ((await readdir(lockPath)).length > 0) return; + } catch (readError) { + if (readError.code === 'ENOENT') return; + if (readError.code !== 'EPERM') throw readError; + } + if (attempt === 10) throw error; + await setTimeout(100); + } } } diff --git a/test/report-concurrency.test.mjs b/test/report-concurrency.test.mjs index b5fe7ea..60980c4 100644 --- a/test/report-concurrency.test.mjs +++ b/test/report-concurrency.test.mjs @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; import { randomUUID } from 'node:crypto'; -import { mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, utimes, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { basename, dirname, join } from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; @@ -315,6 +315,48 @@ test('a writer waits for a held report lock and proceeds after release', async ( await assertUnlocked(fixture); }); +for (const operation of ['unlink', 'rmdir']) test(`a transient ${operation} permission error while removing an expired lock is retried`, async (t) => { + const oldObservation = skill('alpha'); + const newObservation = skill('beta'); + const fixture = await makeFixture(t, [oldObservation]); + const destination = join(await realpath(dirname(fixture.outputPath)), basename(fixture.outputPath)); + const lockPath = `${destination}.lock`; + await mkdir(lockPath); + const ownerPath = join(lockPath, randomUUID()); + await writeFile(ownerPath, ''); + const expired = new Date(Date.now() - 20_000); + await utimes(ownerPath, expired, expired); + + const preload = join(fixture.directory, `transient-${operation}.mjs`); + await writeFile(preload, ` + import fs from 'node:fs/promises'; + import { syncBuiltinESMExports } from 'node:module'; + + const operation = ${JSON.stringify(operation)}; + const target = ${JSON.stringify(operation === 'unlink' ? ownerPath : lockPath)}; + const originalRemove = fs[operation].bind(fs); + let failed = false; + fs[operation] = async function remove(path, ...args) { + if (!failed && String(path) === target) { + failed = true; + const error = new Error('operation not permitted'); + error.code = 'EPERM'; + throw error; + } + return originalRemove(path, ...args); + }; + syncBuiltinESMExports(); + `); + + assertSuccess(await runReporter(fixture, { + action: 'record', observation: newObservation, + }, ['--import', pathToFileURL(preload).href])); + assert.deepEqual(await readReport(fixture), buildReport({ + schemaVersion: 1, observations: [oldObservation, newObservation], + })); + await assertUnlocked(fixture); +}); + test('writers recover an expired lock after its owner is killed and preserve every record', { timeout: 30_000 }, async (t) => { const oldObservation = skill('alpha'); const fixture = await makeFixture(t, [oldObservation]);