Define suite releases and record report version provenance

README.md

@@ -17,7 +17,7 @@ The suite includes these plugins:
 
 To include the optional Streamable HTTP and legacy HTTP+SSE checks, follow the [Core fixture's HTTP setup](plugins/agent-plugins-conformance-core/#optional-http-setup) **before the client loads the plugins**. This starts a bundled local server in a separate terminal; CI can launch the same command. Without the server, HTTP checks remain `not_verified` and the agent continues collecting the other checks.
 
-Install and load the plugins through the client's Agent Plugins support, then ask the agent:
+Install and load the plugins from the same [release tag](docs/release-policy.md#release-tags) through the client's Agent Plugins support, then ask the agent:
 
 > Run the run-conformance skill from Agent Plugins Conformance and save the JSON report to /tmp/agent-plugins-conformance/report.json.
 

docs/release-policy.md

@@ -0,0 +1,31 @@
+# Release policy
+
+The suite’s plugins are released together as one suite. Their `plugin.json` versions and the root development package version match the suite release. Install the plugins from the same release tag.
+
+Suite versions advance independently of Agent Plugins specification versions. Each suite release targets one exact published specification release; several suite releases can improve coverage of the same specification.
+
+## Versioning
+
+The suite uses [Semantic Versioning](https://semver.org/). Before `1.0.0`, consumer-facing interfaces are considered unstable. For stable releases:
+
+| Change | Increment |
+| --- | --- |
+| Correct a check, improve collection guidance, or fix reporting without breaking a documented consumer interface | Patch |
+| Add checks or other functionality while preserving existing consumer interfaces and prerequisites | Minor |
+| Break the documented saved-report format, remove or redefine check IDs, change required installation or runtime prerequisites incompatibly, or replace the targeted specification version | Major |
+
+The collection interface between the guide and reporter, and the protocols between suite components, are internal. They can change together without requiring a major release. The documented saved-report fields and check IDs are interfaces that consumers use for acceptance queries.
+
+A corrective patch can change a verdict. New checks can change the result of an acceptance query that selects an entire ID prefix. Therefore, pin an exact suite release for reproducible acceptance testing.
+
+## Specification releases
+
+A new specification tag does not automatically retarget or release the suite. Adopting it requires validating the suite against that published release and keeping the declared schema targets, evaluated requirements, and report's `specVersion` consistent.
+
+Replacing the target specification is a breaking change because clients supporting the previous target are not required to recognize the new schema identifiers. For stable suite versions, this requires a major increment. Earlier suite tags remain available for testing earlier specification versions. Fixes can be backported when needed; publishing a new target does not promise ongoing maintenance of every older release line.
+
+## Release tags
+
+Each suite release has an annotated `vX.Y.Z` tag matching the root package and all plugin versions. Its GitHub release identifies the exact specification tag being targeted.
+
+Release tags and released plugin contents are immutable; corrections require a new suite version. `main` is development state; use release tags for repeatable runs.

plugins/agent-plugins-conformance/README.md

@@ -26,6 +26,7 @@ This abbreviated example shows three results and their summary counts; observati
 ```json
 {
   "schemaVersion": 1,
+  "suiteVersion": "0.1.0",
   "specVersion": "1.0.0",
   "results": [
     {
@@ -59,6 +60,8 @@ This abbreviated example shows three results and their summary counts; observati
 }
 ```
 
+`suiteVersion` records the reporting plugin's version to help investigate saved results; `specVersion` identifies the specification being tested, and `schemaVersion` identifies the report format. Older reports may omit `suiteVersion`.
+
 Each object in `results` identifies its case through `id` and `label`, explains the outcome in `detail`, and cites the relevant specification sections in `specSections`. A result object may also contain a `warning` describing a probe cleanup failure, including the probe file's path and error. This warning does not change the result's `status`.
 
 The three statuses mean:

plugins/agent-plugins-conformance/skills/run-conformance/scripts/report.mjs

@@ -82,7 +82,7 @@ async function withReportLock(outputPath, update) {
         if (owners.length) {
           const ownerPath = join(lockPath, owners[0]);
           try {
-            if (Date.now() - (await stat(ownerPath)).mtimeMs > staleAfter) await unlink(ownerPath);
+            if (Date.now() - (await stat(ownerPath)).mtimeMs > staleAfter) await removeLockOwner(ownerPath);
           } catch (cleanupError) {
             if (cleanupError.code !== 'ENOENT') throw cleanupError;
           }
@@ -97,19 +97,47 @@ async function withReportLock(outputPath, update) {
     try {
       await update();
     } finally {
-      await rm(join(lockPath, owner), { force: true });
+      await removeLockOwner(join(lockPath, owner));
       await removeEmptyLock(lockPath);
     }
   } finally {
     await rm(candidate, { recursive: true, force: true });
   }
 }
 
+async function removeLockOwner(ownerPath) {
+  for (let attempt = 0; ; attempt++) {
+    try {
+      await unlink(ownerPath);
+      return;
+    } catch (error) {
+      if (error.code === 'ENOENT') return;
+      if (error.code !== 'EPERM' || attempt === 10) throw error;
+      // Retry Windows permission errors briefly on this exact owner path.
+      await setTimeout(100);
+    }
+  }
+}
+
 async function removeEmptyLock(lockPath) {
-  try {
-    await rmdir(lockPath);
-  } catch (error) {
-    if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error.code)) throw error;
+  for (let attempt = 0; ; attempt++) {
+    try {
+      await rmdir(lockPath);
+      return;
+    } catch (error) {
+      if (['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes(error.code)) return;
+      if (error.code !== 'EPERM') throw error;
+      // Retry Windows permission errors briefly when removing the empty lock.
+      // A new owner may also have filled the directory in the meantime.
+      try {
+        if ((await readdir(lockPath)).length > 0) return;
+      } catch (readError) {
+        if (readError.code === 'ENOENT') return;
+        if (readError.code !== 'EPERM') throw readError;
+      }
+      if (attempt === 10) throw error;
+      await setTimeout(100);
+    }
   }
 }
 

plugins/agent-plugins-conformance/src/report-format.mjs

@@ -22,6 +22,7 @@ export function validateSavedReport(report) {
   object(report, 'report');
   if (report.schemaVersion !== 1) invalid('report.schemaVersion', 'expected 1');
   string(report.specVersion, 'report.specVersion');
+  if (Object.hasOwn(report, 'suiteVersion')) string(report.suiteVersion, 'report.suiteVersion');
   if (!Array.isArray(report.observations)) invalid('report.observations', 'expected an array');
   if (!Array.isArray(report.results)) invalid('report.results', 'expected an array');
   const counts = { pass: 0, fail: 0, not_verified: 0, total: report.results.length };

plugins/agent-plugins-conformance/src/report.mjs

@@ -1,7 +1,12 @@
+import { readFileSync } from 'node:fs';
 import path from 'node:path';
 import { CASES, MCP_CWD_VARIANTS } from './cases.mjs';
 
 export { CASES, CASE_IDS } from './cases.mjs';
+const SUITE_VERSION = JSON.parse(readFileSync(new URL('../plugin.json', import.meta.url), 'utf8')).version;
+if (typeof SUITE_VERSION !== 'string' || !SUITE_VERSION.trim()) {
+  throw new TypeError('plugin.json.version: expected a nonempty string');
+}
 const CORE_SERVERS = Object.keys(MCP_CWD_VARIANTS);
 const COMMAND_TOKEN_SERVERS = Object.freeze({
   'command-token-posix': Object.freeze({
@@ -847,6 +852,7 @@ export function buildReport(input) {
   return {
     schemaVersion: 1,
     specVersion: '1.0.0',
+    suiteVersion: SUITE_VERSION,
     observations: [
       ...Object.keys(SKILLS).filter((skill) => skills.has(skill)).map((skill) => ({
         kind: 'skill', skill, marker: skills.get(skill),

test/record-report.test.mjs

@@ -668,6 +668,8 @@ test('malformed saved state is rejected even when its bad observation would be r
   await mkdir(join(f.directory, 'nested directory'));
   const malformed = [
     '{', 'null', JSON.stringify({ schemaVersion: 1, observations: [] }),
+    JSON.stringify({ ...expected([]), suiteVersion: '' }),
+    JSON.stringify({ ...expected([]), suiteVersion: 123 }),
     JSON.stringify({ ...expected([]), summary: { pass: 999, fail: 0, not_verified: 0, total: 999 } }),
     JSON.stringify({ ...expected([skill()]), observations: [{ ...skill(), unexpected: true }] }),
   ];
@@ -680,6 +682,21 @@ test('malformed saved state is rejected even when its bad observation would be r
   }
 });
 
+test('legacy saved reports remain readable and acquire suite version when recorded again', async (t) => {
+  const f = await fixture(t);
+  await mkdir(join(f.directory, 'nested directory'));
+  const legacy = expected([skill('alpha')]);
+  delete legacy.suiteVersion;
+  await writeFile(f.outputPath, `${JSON.stringify(legacy)}\n`);
+  const summaryScript = fileURLToPath(new URL(
+    '../plugins/agent-plugins-conformance/skills/run-conformance/scripts/summarize.mjs', import.meta.url));
+  const summary = spawnSync(process.execPath, [summaryScript, f.outputPath], { encoding: 'utf8' });
+  assert.equal(summary.status, 0, summary.stderr);
+  assert.equal(summary.stdout.trimEnd(), formatReport(legacy));
+  f.record(skill('beta'));
+  assert.deepEqual(await f.read(), expected([skill('alpha'), skill('beta')]));
+});
+
 test('write errors report failure and remove sibling temporary files', async (t) => {
   const f = await fixture(t);
   await mkdir(f.outputPath, { recursive: true });
@@ -699,14 +716,19 @@ test('copied primary plugin records and summarizes without sibling plugins or ru
   const f = await fixture(t);
   const copiedPlugin = join(f.directory, 'copied plugin');
   await cp(new URL('../plugins/agent-plugins-conformance', import.meta.url), copiedPlugin, { recursive: true });
+  const manifestPath = join(copiedPlugin, 'plugin.json');
+  const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
+  const copiedVersion = '9.8.7';
+  await writeFile(manifestPath, JSON.stringify({ ...manifest, version: copiedVersion }));
   assert.equal((await readdir(copiedPlugin)).includes('node_modules'), false);
   const copiedScript = join(copiedPlugin, 'skills/run-conformance/scripts/report.mjs');
   f.success(start, copiedScript);
+  assert.equal((await f.read()).suiteVersion, copiedVersion);
   f.success({ action: 'record', observation: skill('alpha') }, copiedScript);
   f.success({ action: 'record', observation: skill('beta') }, copiedScript);
   f.success({ action: 'record', observation: discovery() }, copiedScript);
   const report = await f.read();
-  assert.deepEqual(report, expected([skill('alpha'), skill('beta'), discovery()]));
+  assert.deepEqual(report, { ...expected([skill('alpha'), skill('beta'), discovery()]), suiteVersion: copiedVersion });
   assert.equal(report.results.find(({ id }) => id === 'skills.discovery.immediate-children').status, 'pass');
   const summary = spawnSync(process.execPath, [join(copiedPlugin, 'skills/run-conformance/scripts/summarize.mjs'), f.outputPath], {
     cwd: f.directory, encoding: 'utf8', timeout: 10_000,
@@ -718,7 +740,10 @@ test('copied primary plugin records and summarizes without sibling plugins or ru
   const result = health.run({ action: 'record', observation: missingHttp() }, copiedScript);
   assert.equal(result.status, 0, result.stderr);
   const withHttp = await f.read();
-  assert.deepEqual(withHttp, expected([skill('alpha'), skill('beta'), discovery(), { ...missingHttp(), serverHealthCheck: 'passed' }]));
+  assert.deepEqual(withHttp, {
+    ...expected([skill('alpha'), skill('beta'), discovery(), { ...missingHttp(), serverHealthCheck: 'passed' }]),
+    suiteVersion: copiedVersion,
+  });
   // Reading saved evidence must remain deterministic after fixture state changes.
   await health.setResponse({ error: 'fixture stopped after collection' });
   const savedSummary = health.run('', join(copiedPlugin, 'skills/run-conformance/scripts/summarize.mjs'));

test/report-concurrency.test.mjs

@@ -1,7 +1,7 @@
 import assert from 'node:assert/strict';
 import { spawn } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
-import { mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, writeFile } from 'node:fs/promises';
+import { mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, utimes, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { basename, dirname, join } from 'node:path';
 import { setTimeout as delay } from 'node:timers/promises';
@@ -315,6 +315,48 @@ test('a writer waits for a held report lock and proceeds after release', async (
   await assertUnlocked(fixture);
 });
 
+for (const operation of ['unlink', 'rmdir']) test(`a transient ${operation} permission error while removing an expired lock is retried`, async (t) => {
+  const oldObservation = skill('alpha');
+  const newObservation = skill('beta');
+  const fixture = await makeFixture(t, [oldObservation]);
+  const destination = join(await realpath(dirname(fixture.outputPath)), basename(fixture.outputPath));
+  const lockPath = `${destination}.lock`;
+  await mkdir(lockPath);
+  const ownerPath = join(lockPath, randomUUID());
+  await writeFile(ownerPath, '');
+  const expired = new Date(Date.now() - 20_000);
+  await utimes(ownerPath, expired, expired);
+
+  const preload = join(fixture.directory, `transient-${operation}.mjs`);
+  await writeFile(preload, `
+    import fs from 'node:fs/promises';
+    import { syncBuiltinESMExports } from 'node:module';
+
+    const operation = ${JSON.stringify(operation)};
+    const target = ${JSON.stringify(operation === 'unlink' ? ownerPath : lockPath)};
+    const originalRemove = fs[operation].bind(fs);
+    let failed = false;
+    fs[operation] = async function remove(path, ...args) {
+      if (!failed && String(path) === target) {
+        failed = true;
+        const error = new Error('operation not permitted');
+        error.code = 'EPERM';
+        throw error;
+      }
+      return originalRemove(path, ...args);
+    };
+    syncBuiltinESMExports();
+  `);
+
+  assertSuccess(await runReporter(fixture, {
+    action: 'record', observation: newObservation,
+  }, ['--import', pathToFileURL(preload).href]));
+  assert.deepEqual(await readReport(fixture), buildReport({
+    schemaVersion: 1, observations: [oldObservation, newObservation],
+  }));
+  await assertUnlocked(fixture);
+});
+
 test('writers recover an expired lock after its owner is killed and preserve every record', { timeout: 30_000 }, async (t) => {
   const oldObservation = skill('alpha');
   const fixture = await makeFixture(t, [oldObservation]);

test/report.test.mjs

@@ -1,8 +1,21 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
 import path from 'node:path';
 import { buildReport, CASE_IDS } from '../plugins/agent-plugins-conformance/src/report.mjs';
 
+const rootVersion = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version;
+
+test('all four plugin manifests use the root package version', () => {
+  for (const name of [
+    'agent-plugins-conformance', 'agent-plugins-conformance-core',
+    'agent-plugins-conformance-recovery', 'agent-plugins-conformance-invalid-mcp',
+  ]) {
+    const manifest = JSON.parse(readFileSync(new URL(`../plugins/${name}/plugin.json`, import.meta.url), 'utf8'));
+    assert.equal(manifest.version, rootVersion, name);
+  }
+});
+
 function input(root = '/fixture/plugin', data = '/state/plugin') {
   const flavor = root.startsWith('/') ? path.posix : path.win32;
   const platform = flavor === path.win32 ? 'windows' : 'posix';
@@ -95,6 +108,7 @@ test('complete stdio and skill core evidence passes its cases and preserves cano
   assert.deepEqual(report.summary, { pass: 19, fail: 0, not_verified: 39, total: 58 });
   assert.deepEqual(report.results.map(({ id }) => id), CASE_IDS);
   assert.equal(report.specVersion, '1.0.0');
+  assert.equal(report.suiteVersion, rootVersion);
   assert.deepEqual(result(report, 'mcp.stdio.env.plugin-root').specSections, ['9.1']);
   assert.deepEqual(report.observations, value.observations);
   runtime(value).argv.push('after report');