diff --git a/plugins/agent-plugins-conformance-core/README.md b/plugins/agent-plugins-conformance-core/README.md index ce237c3..5239148 100644 --- a/plugins/agent-plugins-conformance-core/README.md +++ b/plugins/agent-plugins-conformance-core/README.md @@ -35,6 +35,7 @@ Replace the path with a downloaded or installed copy of this plugin; the command | `mcp.stdio.env.configured-precedence` | Configured environment values replace conflicting base-environment values. | | `mcp.stdio.args.preservation-and-expansion` | Argument boundaries, including spaces and an empty argument, are preserved; recognized placeholders expand and unknown placeholder-like text stays literal. | | `mcp.stdio.command.single-token` | A command path containing a space is preserved as a single executable token. | +| `mcp.stdio.command.plugin-relative-resolution` | Plugin-relative commands resolve against the plugin root, independently of the configured working directory. | | `mcp.stdio.env.expansion` | Repeated recognized placeholders expand in environment values and unknown placeholder-like text stays literal. | | `mcp.streamable-http.tool-availability` | The client exposes the HTTP tool and it returns a valid observation despite a conflicting, mixed-case configured `Accept` header. A completed discovery or call attempt with no observation fails this check only when the reporter confirms that the local fixture is healthy. | | `mcp.streamable-http.url.literal-route-and-query` | The configured URL path and query are preserved literally. | diff --git a/plugins/agent-plugins-conformance-core/mcp.json b/plugins/agent-plugins-conformance-core/mcp.json index 4427fbf..8a028a8 100644 --- a/plugins/agent-plugins-conformance-core/mcp.json +++ b/plugins/agent-plugins-conformance-core/mcp.json @@ -65,7 +65,8 @@ "arg with spaces", "", "literal-value" - ] + ], + "cwd": "./probe-workdir" }, "command-token-windows": { "type": "stdio", @@ -74,7 +75,8 @@ "arg with spaces", "", "literal-value" - ] + ], + "cwd": "./probe-workdir" }, "sse": { "type": "sse", diff --git a/plugins/agent-plugins-conformance-core/probe-workdir/bin/posix/probe b/plugins/agent-plugins-conformance-core/probe-workdir/bin/posix/probe new file mode 100755 index 0000000..7550df7 --- /dev/null +++ b/plugins/agent-plugins-conformance-core/probe-workdir/bin/posix/probe @@ -0,0 +1,2 @@ +#!/bin/sh +exec node "$(dirname "$0")/../../../dist/probe.mjs" command-token-posix posix-cwd-decoy split "$@" diff --git a/plugins/agent-plugins-conformance-core/probe-workdir/bin/posix/probe token.sh b/plugins/agent-plugins-conformance-core/probe-workdir/bin/posix/probe token.sh new file mode 100755 index 0000000..f7ed0f1 --- /dev/null +++ b/plugins/agent-plugins-conformance-core/probe-workdir/bin/posix/probe token.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exec node "$(dirname "$0")/../../../dist/probe.mjs" command-token-posix posix-cwd-exact intact "$@" diff --git a/plugins/agent-plugins-conformance-core/probe-workdir/bin/windows/probe token.cmd b/plugins/agent-plugins-conformance-core/probe-workdir/bin/windows/probe token.cmd new file mode 100644 index 0000000..fc1d812 --- /dev/null +++ b/plugins/agent-plugins-conformance-core/probe-workdir/bin/windows/probe token.cmd @@ -0,0 +1,2 @@ +@echo off +node "%~dp0..\..\..\dist\probe.mjs" command-token-windows windows-cwd-exact intact %* diff --git a/plugins/agent-plugins-conformance-core/probe-workdir/bin/windows/probe.cmd b/plugins/agent-plugins-conformance-core/probe-workdir/bin/windows/probe.cmd new file mode 100644 index 0000000..b1ee9a4 --- /dev/null +++ b/plugins/agent-plugins-conformance-core/probe-workdir/bin/windows/probe.cmd @@ -0,0 +1,2 @@ +@echo off +node "%~dp0..\..\..\dist\probe.mjs" command-token-windows windows-cwd-decoy split %* diff --git a/plugins/agent-plugins-conformance/src/cases.mjs b/plugins/agent-plugins-conformance/src/cases.mjs index d073d3b..3a9b917 100644 --- a/plugins/agent-plugins-conformance/src/cases.mjs +++ b/plugins/agent-plugins-conformance/src/cases.mjs @@ -29,6 +29,7 @@ export const CASES = Object.freeze([ ['mcp.stdio.env.configured-value', 'Configured environment', ['9.1']], ['mcp.stdio.env.configured-precedence', 'Configured environment precedence', ['9.1']], ['mcp.stdio.command.single-token', 'Command token preservation', ['7.2.1']], + ['mcp.stdio.command.plugin-relative-resolution', 'Plugin-relative command resolution', ['7.2.1']], ['mcp.stdio.args.preservation-and-expansion', 'Argument preservation and expansion', ['7.2.1', '9.2']], ['mcp.stdio.env.expansion', 'Environment expansion', ['9.2']], ['mcp.streamable-http.tool-availability', 'Streamable HTTP MCP tool evidence', ['6.1', '7.2.1']], diff --git a/plugins/agent-plugins-conformance/src/report.mjs b/plugins/agent-plugins-conformance/src/report.mjs index 58f96c5..5c347e4 100644 --- a/plugins/agent-plugins-conformance/src/report.mjs +++ b/plugins/agent-plugins-conformance/src/report.mjs @@ -4,8 +4,14 @@ import { CASES, MCP_CWD_VARIANTS } from './cases.mjs'; export { CASES, CASE_IDS } from './cases.mjs'; const CORE_SERVERS = Object.keys(MCP_CWD_VARIANTS); const COMMAND_TOKEN_SERVERS = Object.freeze({ - 'command-token-posix': Object.freeze({ exactOrigin: 'posix-exact', decoyOrigin: 'posix-decoy', splitTail: 'token.sh' }), - 'command-token-windows': Object.freeze({ exactOrigin: 'windows-exact', decoyOrigin: 'windows-decoy', splitTail: 'token.cmd' }), + 'command-token-posix': Object.freeze({ + rootExactOrigin: 'posix-exact', rootDecoyOrigin: 'posix-decoy', + cwdExactOrigin: 'posix-cwd-exact', cwdDecoyOrigin: 'posix-cwd-decoy', splitTail: 'token.sh', + }), + 'command-token-windows': Object.freeze({ + rootExactOrigin: 'windows-exact', rootDecoyOrigin: 'windows-decoy', + cwdExactOrigin: 'windows-cwd-exact', cwdDecoyOrigin: 'windows-cwd-decoy', splitTail: 'token.cmd', + }), }); const INVALID_STDIO_SERVERS = Object.freeze({ 'recovery-cwd-invalid-form': 'mcp.stdio.cwd.invalid-form', @@ -569,6 +575,7 @@ export function buildReport(input) { } const defaultEvidence = runtime.get('default'); const commandTokenId = 'mcp.stdio.command.single-token'; + const commandResolutionId = 'mcp.stdio.command.plugin-relative-resolution'; if (defaultEvidence) { const defaultFlavor = pathFlavor(defaultEvidence.root); const platform = defaultFlavor === path.win32 ? 'windows' : 'posix'; @@ -577,22 +584,48 @@ export function buildReport(input) { const evidence = runtime.get(server); if (!evidence) { set(commandTokenId, 'not_verified', `No attributable ${platform} command-token observation was supplied.`); + set(commandResolutionId, 'not_verified', + `No attributable ${platform} plugin-relative command observation was supplied.`); } else if (!samePath(evidence.root, defaultEvidence.root, defaultFlavor)) { set(commandTokenId, 'not_verified', `The ${platform} command-token observation did not identify the same installed plugin root as the Core default observation.`); + set(commandResolutionId, 'not_verified', + `The ${platform} plugin-relative command observation did not identify the same installed plugin root as the Core default observation.`); } else { - const split = evidence.argv[0] === server && evidence.argv[1] === variant.decoyOrigin && + const rootIntact = evidence.argv[0] === server && evidence.argv[1] === variant.rootExactOrigin && + evidence.argv[2] === 'intact'; + const cwdIntact = evidence.argv[0] === server && evidence.argv[1] === variant.cwdExactOrigin && + evidence.argv[2] === 'intact'; + const split = evidence.argv[0] === server && + [variant.rootDecoyOrigin, variant.cwdDecoyOrigin].includes(evidence.argv[1]) && evidence.argv[2] === 'split' && evidence.argv[3] === variant.splitTail; - const intact = evidence.argv[0] === server && evidence.argv[1] === variant.exactOrigin && evidence.argv[2] === 'intact'; if (split) { set(commandTokenId, 'fail', `The ${platform} decoy wrapper received ${JSON.stringify(variant.splitTail)} as an argument, showing that the configured command was split.`); - } else if (intact) { + } else if (rootIntact || cwdIntact) { set(commandTokenId, 'pass', `The ${platform} exact-name wrapper was selected.`); } else { set(commandTokenId, 'not_verified', `The ${platform} command-token observation did not identify either the exact-name wrapper or an attributable split-name decoy.`); } + + const expectedCwd = defaultFlavor.join(defaultEvidence.root, 'probe-workdir'); + if (!samePath(evidence.cwd, expectedCwd, defaultFlavor)) { + set(commandResolutionId, 'not_verified', + `The ${platform} plugin-relative command observation ran in ${JSON.stringify(evidence.cwd)} instead of the expected working directory ${JSON.stringify(expectedCwd)}.`); + } else if (split) { + set(commandResolutionId, 'not_verified', + `The ${platform} command was split before resolution, so plugin-relative resolution was not evaluated.`); + } else if (rootIntact) { + set(commandResolutionId, 'pass', + `The ${platform} plugin-root exact-name wrapper was selected while the configured working directory remained active.`); + } else if (cwdIntact) { + set(commandResolutionId, 'fail', + `The ${platform} working-directory exact-name wrapper was selected, showing that the plugin-relative command was resolved against the subprocess working directory.`); + } else { + set(commandResolutionId, 'not_verified', + `The ${platform} command observation did not identify an intact exact-name wrapper, so plugin-relative resolution was not evaluated.`); + } } } if (skills.has('conformance-recovery-valid')) { diff --git a/scripts/smoke-codex.mjs b/scripts/smoke-codex.mjs index 746aee0..9c5323a 100644 --- a/scripts/smoke-codex.mjs +++ b/scripts/smoke-codex.mjs @@ -271,12 +271,12 @@ try { : invalidSseServers.includes(server) ? 'mcp-sse' : 'mcp-stdio', `${server}: missing observation`); assert.equal(observation.server, server, `${server}: unexpected observation server`); + record({ action: 'record', observation }); if (server === commandServer) { assert.deepEqual(observation.evidence.argv, [commandServer, `${commandPlatform}-exact`, 'intact', 'arg with spaces', '', 'literal-value'], 'Native launch did not preserve the command token and configured arguments'); } - record({ action: 'record', observation }); } // SSE is optional. Preserve native diagnostics even when the loader omits entries. const sseObservations = []; diff --git a/test/command-token.test.mjs b/test/command-token.test.mjs index c447fda..f6da6df 100644 --- a/test/command-token.test.mjs +++ b/test/command-token.test.mjs @@ -1,19 +1,22 @@ import assert from 'node:assert/strict'; -import { cp, lstat, mkdtemp, readFile, realpath, readdir, rm, writeFile } from 'node:fs/promises'; +import { cp, lstat, mkdir, mkdtemp, readFile, realpath, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import test from 'node:test'; import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js'; +import { buildReport } from '../plugins/agent-plugins-conformance/src/report.mjs'; const configuredArgs = ['arg with spaces', '', 'literal-value']; async function fixture(t) { const parent = await mkdtemp(join(tmpdir(), 'apc command token ')); const root = join(await realpath(parent), 'installed plugin with spaces'); + const data = join(await realpath(parent), 'plugin data with spaces'); const clients = []; const protocolErrors = []; await cp(new URL('../plugins/agent-plugins-conformance-core/', import.meta.url), root, { recursive: true }); + await mkdir(data); t.after(async () => { try { const results = await Promise.allSettled(clients.map((client) => client.close())); @@ -25,14 +28,46 @@ async function fixture(t) { } }); const config = JSON.parse(await readFile(join(root, 'mcp.json'), 'utf8')); - return { parent, root, clients, config, protocolErrors }; + return { parent, root, data, clients, config, protocolErrors }; } -async function connect(files, server, { split = false, batch = false } = {}) { +async function connectTransport(files, options) { + const client = new Client({ name: 'command-token-reference-test', version: '1' }); + client.onerror = (error) => files.protocolErrors.push(error); + files.clients.push(client); + await client.connect(new StdioClientTransport({ ...options, stderr: 'pipe' })); + return client; +} + +function baseEnvironment() { + const env = Object.fromEntries(Object.entries(process.env)); + for (const name of ['PLUGIN_ROOT', 'PLUGIN_DATA', 'APC_VALUE', 'APC_EXPANSION', 'APC_LITERAL']) delete env[name]; + return env; +} + +async function connectDefault(files) { + return connectTransport(files, { + command: process.execPath, + args: [join(files.root, 'dist/probe.mjs'), 'default'], + cwd: files.root, + env: { ...baseEnvironment(), PLUGIN_ROOT: files.root, PLUGIN_DATA: files.data }, + }); +} + +async function connect(files, server, { base = 'root', split = false, batch = false } = {}) { const configured = files.config.mcpServers[server]; - let command = resolve(files.root, configured.command.slice(2)); + const tail = process.platform === 'win32' ? 'token.cmd' : 'token.sh'; + const cwd = resolve(files.root, configured.cwd.slice(2)); + let command = resolve(base === 'cwd' ? cwd : files.root, configured.command.slice(2)); let args = configured.args; - if (split) { + if (split === 'root-harness') { + // Simulate the attributable result of splitting the command while resolving + // its first token from the plugin root. Native clients choose the resolution + // base before the fixture can observe it, so this is intentionally a harness. + command = resolve(files.root, configured.command.slice(2).replace(` ${tail}`, '')) + + (process.platform === 'win32' ? '.cmd' : ''); + args = [tail, ...configured.args]; + } else if (split === 'cwd-shell') { if (process.platform === 'win32') { command = join(files.parent, 'bad command token launcher.cmd'); await writeFile(command, '@echo off\r\n.\\bin\\windows\\probe token.cmd %*\r\n'); @@ -44,15 +79,11 @@ async function connect(files, server, { split = false, batch = false } = {}) { if (batch) { // Preserve ordinary batch echo settings; the fixture must keep stdout protocol-clean. command = join(files.parent, 'ordinary batch launcher.cmd'); - await writeFile(command, '@call ".\\bin\\windows\\probe token.cmd" %*\r\n'); + await writeFile(command, + `@call "${resolve(files.root, configured.command.slice(2))}" %*\r\n`); } - const env = Object.fromEntries(Object.entries(process.env)); - for (const name of ['PLUGIN_ROOT', 'PLUGIN_DATA', 'APC_VALUE', 'APC_EXPANSION', 'APC_LITERAL']) delete env[name]; - const client = new Client({ name: 'command-token-reference-test', version: '1' }); - client.onerror = (error) => files.protocolErrors.push(error); - files.clients.push(client); - await client.connect(new StdioClientTransport({ command, args, cwd: files.root, env, stderr: 'pipe' })); - return client; + const env = baseEnvironment(); + return connectTransport(files, { command, args, cwd, env }); } async function observe(client) { @@ -64,22 +95,41 @@ async function observe(client) { return result.structuredContent; } -test('the copied Core fixture preserves the platform command token and configured arguments', +test('the copied Core fixture completes MCP exchanges from all four attributable command origins', { timeout: 30_000 }, async (t) => { const files = await fixture(t); assert.equal((await readdir(files.root)).includes('node_modules'), false); const platform = process.platform === 'win32' ? 'windows' : 'posix'; + const tail = process.platform === 'win32' ? 'token.cmd' : 'token.sh'; const server = `command-token-${platform}`; - const observation = await observe(await connect(files, server)); - assert.equal(observation.kind, 'mcp-stdio'); - assert.equal(observation.server, server); - assert.equal(observation.evidence.server, server); - assert.equal(observation.evidence.root, await realpath(files.root)); - assert.equal(observation.evidence.cwd, await realpath(files.root)); - assert.equal(observation.evidence.resolvedData, null); - assert.deepEqual(observation.evidence.env, {}); - assert.deepEqual(observation.evidence.argv, - [server, `${platform}-exact`, 'intact', ...configuredArgs]); + const expectedCwd = await realpath(join(files.root, 'probe-workdir')); + const defaultObservation = await observe(await connectDefault(files)); + for (const [variant, origin, marker, prefix, tokenStatus, resolutionStatus] of [ + [undefined, `${platform}-exact`, 'intact', [], 'pass', 'pass'], + ['root-harness', `${platform}-decoy`, 'split', [tail], 'fail', 'not_verified'], + ['cwd-exact', `${platform}-cwd-exact`, 'intact', [], 'pass', 'fail'], + ['cwd-shell', `${platform}-cwd-decoy`, 'split', [tail], 'fail', 'not_verified'], + ]) { + const options = variant === 'cwd-exact' ? { base: 'cwd' } : { split: variant }; + const observation = await observe(await connect(files, server, options)); + assert.equal(observation.kind, 'mcp-stdio', variant); + assert.equal(observation.server, server, variant); + assert.equal(observation.evidence.server, server, variant); + assert.equal(observation.evidence.root, await realpath(files.root), variant); + assert.equal(observation.evidence.cwd, expectedCwd, variant); + assert.equal(observation.evidence.resolvedData, null, variant); + assert.deepEqual(observation.evidence.env, {}, variant); + assert.deepEqual(observation.evidence.argv, + [server, origin, marker, ...prefix, ...configuredArgs], variant); + const report = buildReport({ + schemaVersion: 1, + observations: [defaultObservation, observation], + }); + assert.equal(report.results.find(({ id }) => + id === 'mcp.stdio.command.single-token').status, tokenStatus, variant); + assert.equal(report.results.find(({ id }) => + id === 'mcp.stdio.command.plugin-relative-resolution').status, resolutionStatus, variant); + } }); test('deliberate unquoted platform-shell parsing reaches the split-name decoy', @@ -88,11 +138,11 @@ test('deliberate unquoted platform-shell parsing reaches the split-name decoy', const platform = process.platform === 'win32' ? 'windows' : 'posix'; const tail = process.platform === 'win32' ? 'token.cmd' : 'token.sh'; const server = `command-token-${platform}`; - const observation = await observe(await connect(files, server, { split: true })); + const observation = await observe(await connect(files, server, { split: 'cwd-shell' })); assert.equal(observation.kind, 'mcp-stdio'); assert.equal(observation.server, server); assert.deepEqual(observation.evidence.argv, - [server, `${platform}-decoy`, 'split', tail, ...configuredArgs]); + [server, `${platform}-cwd-decoy`, 'split', tail, ...configuredArgs]); }); test('explicit Windows batch execution preserves the command and arguments without non-MCP stdout', @@ -102,13 +152,19 @@ test('explicit Windows batch execution preserves the command and arguments witho const observation = await observe(await connect(files, server, { batch: true })); assert.deepEqual(observation.evidence.argv, [server, 'windows-exact', 'intact', ...configuredArgs]); + assert.equal(observation.evidence.cwd, await realpath(join(files.root, 'probe-workdir'))); }); test('Core packages paired native wrappers with the intended executable modes', { skip: process.platform === 'win32' }, async () => { const root = new URL('../plugins/agent-plugins-conformance-core/bin/', import.meta.url); + const cwd = new URL('../plugins/agent-plugins-conformance-core/probe-workdir/bin/', import.meta.url); assert.notEqual((await lstat(new URL('posix/probe token.sh', root))).mode & 0o111, 0); assert.notEqual((await lstat(new URL('posix/probe', root))).mode & 0o111, 0); assert.equal((await lstat(new URL('windows/probe token.cmd', root))).mode & 0o111, 0); assert.equal((await lstat(new URL('windows/probe.cmd', root))).mode & 0o111, 0); + assert.notEqual((await lstat(new URL('posix/probe token.sh', cwd))).mode & 0o111, 0); + assert.notEqual((await lstat(new URL('posix/probe', cwd))).mode & 0o111, 0); + assert.equal((await lstat(new URL('windows/probe token.cmd', cwd))).mode & 0o111, 0); + assert.equal((await lstat(new URL('windows/probe.cmd', cwd))).mode & 0o111, 0); }); diff --git a/test/package.test.mjs b/test/package.test.mjs index ec490c4..a391513 100644 --- a/test/package.test.mjs +++ b/test/package.test.mjs @@ -26,6 +26,19 @@ test('plugin and MCP target the same published specification version', async () assert.equal((await load('mcp')).$schema, 'https://agent-plugins.org/schemas/1.0.0/mcp.schema.json'); }); +test('command-token MCP servers use the shared plugin-relative working directory', async () => { + const mcp = JSON.parse(await readFile( + new URL('../plugins/agent-plugins-conformance-core/mcp.json', import.meta.url), 'utf8')); + for (const platform of ['posix', 'windows']) { + assert.deepEqual(mcp.mcpServers[`command-token-${platform}`], { + type: 'stdio', + command: `./bin/${platform}/probe token.${platform === 'posix' ? 'sh' : 'cmd'}`, + args: ['arg with spaces', '', 'literal-value'], + cwd: './probe-workdir', + }); + } +}); + test('primary run skill is separate from the core discovery fixture layout', async () => { const primary = new URL('../plugins/agent-plugins-conformance/', import.meta.url); const core = new URL('../plugins/agent-plugins-conformance-core/', import.meta.url); diff --git a/test/probe.test.mjs b/test/probe.test.mjs index 104c210..c00ae97 100644 --- a/test/probe.test.mjs +++ b/test/probe.test.mjs @@ -106,7 +106,7 @@ test('copied plugin runs only MCP observation tools without node_modules', { tim const direct = buildReport(reportInput); assert.equal(direct.summary.fail, 0); assert.equal(direct.summary.pass, 16); - assert.equal(direct.summary.not_verified, 41); + assert.equal(direct.summary.not_verified, 42); }); test('faulty ambient precedence becomes a normal reporter failure for both platform variables', { timeout: 30_000 }, async (t) => { diff --git a/test/record-report.test.mjs b/test/record-report.test.mjs index 70ce358..6536cc5 100644 --- a/test/record-report.test.mjs +++ b/test/record-report.test.mjs @@ -35,12 +35,13 @@ const commandToken = (platform = 'posix', { data = platform === 'posix' ? '/data' : 'D:\\data', origin = `${platform}-exact`, marker = 'intact', args = ['arg with spaces', '', 'literal-value'], + cwd = platform === 'posix' ? `${root}/probe-workdir` : `${root}\\probe-workdir`, } = {}) => { const server = `command-token-${platform}`; return { kind: 'mcp-stdio', server, evidence: { - version: 1, server, root, cwd: root, resolvedData: data, + version: 1, server, root, cwd, resolvedData: data, argv: [server, origin, marker, ...args], env: {}, }, }; @@ -425,6 +426,8 @@ test('command-token evidence records canonically and replacement changes only it f.record(exact); let report = await f.read(); assert.equal(report.results.find(({ id }) => id === 'mcp.stdio.command.single-token').status, 'pass'); + assert.equal(report.results.find(({ id }) => + id === 'mcp.stdio.command.plugin-relative-resolution').status, 'pass'); assert.deepEqual(report.observations, [mcp(), exact]); exact.evidence.argv.push('changed after recording'); @@ -436,13 +439,25 @@ test('command-token evidence records canonically and replacement changes only it f.record(split); report = await f.read(); assert.equal(report.results.find(({ id }) => id === 'mcp.stdio.command.single-token').status, 'fail'); + assert.equal(report.results.find(({ id }) => + id === 'mcp.stdio.command.plugin-relative-resolution').status, 'not_verified'); assert.deepEqual(report.observations, [mcp(), split]); + const cwdExact = commandToken('posix', { origin: 'posix-cwd-exact' }); + f.record(cwdExact); + report = await f.read(); + assert.equal(report.results.find(({ id }) => id === 'mcp.stdio.command.single-token').status, 'pass'); + assert.equal(report.results.find(({ id }) => + id === 'mcp.stdio.command.plugin-relative-resolution').status, 'fail'); + assert.deepEqual(report.observations, [mcp(), cwdExact]); + const windows = commandToken('windows'); f.record(windows); report = await f.read(); - assert.equal(report.results.find(({ id }) => id === 'mcp.stdio.command.single-token').status, 'fail'); - assert.deepEqual(report.observations, [mcp(), split, windows]); + assert.equal(report.results.find(({ id }) => id === 'mcp.stdio.command.single-token').status, 'pass'); + assert.equal(report.results.find(({ id }) => + id === 'mcp.stdio.command.plugin-relative-resolution').status, 'fail'); + assert.deepEqual(report.observations, [mcp(), cwdExact, windows]); }); test('recovery observations have distinct recorder keys and repeated keys replace canonically', async (t) => { diff --git a/test/report-human.test.mjs b/test/report-human.test.mjs index ef98ab5..60d4dea 100644 --- a/test/report-human.test.mjs +++ b/test/report-human.test.mjs @@ -28,7 +28,7 @@ function input() { { kind: 'mcp-stdio', server: 'command-token-posix', evidence: { - version: 1, server: 'command-token-posix', root, cwd: root, resolvedData: data, + version: 1, server: 'command-token-posix', root, cwd: `${root}/probe-workdir`, resolvedData: data, argv: ['command-token-posix', 'posix-exact', 'intact', 'arg with spaces', '', 'literal-value'], env: {}, }, }, @@ -63,9 +63,9 @@ test('complete stdio and recovery evidence leaves optional HTTP and SSE checks u '', 'Checks Passed Failed Not verified', 'Skills 3 0 0', - 'MCP 23 0 26', + 'MCP 24 0 26', 'Filesystem 4 0 1', - 'Total 30 0 27', + 'Total 31 0 27', ].join('\n')); const missing = human.split('\n\nNot verified\n')[1]; for (const id of [ @@ -104,7 +104,7 @@ test('all-unverified human report preserves every saved missing-evidence result' value.observations = []; const human = formatReport(buildReport(value)); assert.match(human, /Skills\s+0\s+0\s+3/); - assert.match(human, /MCP\s+0\s+0\s+49/); + assert.match(human, /MCP\s+0\s+0\s+50/); assert.match(human, /Filesystem\s+0\s+0\s+5/); assert.match(human, /Missing skill observations: conformance-alpha, conformance-beta\./); for (const result of buildReport(value).results) assert.ok(human.includes(`(${result.id})`)); @@ -116,7 +116,7 @@ test('mixed human report puts failures before missing-evidence details', () => { value.observations = [value.observations[2]]; value.observations[0].evidence.env.APC_VALUE = 'incorrect configured value'; const human = formatReport(buildReport(value)); - assert.match(human, /MCP\s+6\s+1\s+42/); + assert.match(human, /MCP\s+6\s+1\s+43/); assert.match(human, /Filesystem\s+1\s+0\s+4/); assert.match(human, /Configured environment \(mcp.stdio.env.configured-value\)/); assert.match(human, /"fixture value with spaces"/); @@ -173,7 +173,7 @@ test('human report renders saved warnings independently of result status', () => const writable = value.results.find(({ id }) => id === 'filesystem.data.writable'); writable.status = 'not_verified'; writable.warning = 'Saved cleanup warning for /state/plugin/leftover.'; - value.summary = { pass: 17, fail: 0, not_verified: 40, total: 57 }; + value.summary = { pass: 18, fail: 0, not_verified: 40, total: 58 }; value.observations = []; const human = formatReport(value); diff --git a/test/report.test.mjs b/test/report.test.mjs index bcb41fd..537aaf4 100644 --- a/test/report.test.mjs +++ b/test/report.test.mjs @@ -32,12 +32,13 @@ function input(root = '/fixture/plugin', data = '/state/plugin') { } const commandTokenObservation = (platform, root = '/fixture/plugin', data = '/state/plugin', { origin = `${platform}-exact`, marker = 'intact', args = ['arg with spaces', '', 'literal-value'], + cwd = (root.startsWith('/') ? path.posix : path.win32).join(root, 'probe-workdir'), } = {}) => { const server = `command-token-${platform}`; return { kind: 'mcp-stdio', server, evidence: { - version: 1, server, root, cwd: root, resolvedData: data, + version: 1, server, root, cwd, resolvedData: data, argv: [server, origin, marker, ...args], env: {}, }, }; @@ -91,7 +92,7 @@ const invalidSseServerCases = [ test('complete stdio and skill core evidence passes its cases and preserves canonical evidence', () => { const value = input(); const report = buildReport(value); - assert.deepEqual(report.summary, { pass: 18, fail: 0, not_verified: 39, total: 57 }); + assert.deepEqual(report.summary, { pass: 19, fail: 0, not_verified: 39, total: 58 }); assert.deepEqual(report.results.map(({ id }) => id), CASE_IDS); assert.equal(report.specVersion, '1.0.0'); assert.deepEqual(result(report, 'mcp.stdio.env.plugin-root').specSections, ['9.1']); @@ -104,19 +105,19 @@ test('complete stdio and skill core evidence passes its cases and preserves cano test('recovery witnesses extend the canonical report without changing core-only results', () => { const coreOnly = buildReport(input()); - assert.deepEqual(coreOnly.summary, { pass: 18, fail: 0, not_verified: 39, total: 57 }); + assert.deepEqual(coreOnly.summary, { pass: 19, fail: 0, not_verified: 39, total: 58 }); assert.equal(result(coreOnly, 'skills.recovery.valid-skill-available').status, 'not_verified'); assert.equal(result(coreOnly, 'mcp.stdio.recovery.valid-server-available').status, 'not_verified'); const recoveryOnly = buildReport({ schemaVersion: 1, observations: recoveryObservations() }); - assert.deepEqual(recoveryOnly.summary, { pass: 2, fail: 0, not_verified: 55, total: 57 }); + assert.deepEqual(recoveryOnly.summary, { pass: 2, fail: 0, not_verified: 56, total: 58 }); assert.equal(result(recoveryOnly, 'skills.recovery.valid-skill-available').status, 'pass'); assert.equal(result(recoveryOnly, 'mcp.stdio.recovery.valid-server-available').status, 'pass'); const combinedInput = input(); combinedInput.observations.push(...recoveryObservations()); const combined = buildReport(combinedInput); - assert.deepEqual(combined.summary, { pass: 21, fail: 0, not_verified: 36, total: 57 }); + assert.deepEqual(combined.summary, { pass: 22, fail: 0, not_verified: 36, total: 58 }); assert.deepEqual(combined.results.map(({ id }) => id), CASE_IDS); const recoveryIds = new Set([ 'skills.recovery.valid-skill-available', @@ -139,7 +140,7 @@ test('recovery witnesses extend the canonical report without changing core-only assert.equal(JSON.stringify(buildReport(reordered)), JSON.stringify(combined)); const missing = buildReport({ schemaVersion: 1, observations: [] }); - assert.deepEqual(missing.summary, { pass: 0, fail: 0, not_verified: 57, total: 57 }); + assert.deepEqual(missing.summary, { pass: 0, fail: 0, not_verified: 58, total: 58 }); }); test('an incorrect recovery skill marker fails its availability check independently', () => { @@ -150,7 +151,7 @@ test('an incorrect recovery skill marker fails its availability check independen assert.equal(availability.status, 'fail'); assert.match(availability.detail, /expected "APC_RECOVERY_VALID_V1"; observed "wrong recovery marker"/); assert.equal(result(report, 'mcp.stdio.recovery.valid-server-available').status, 'pass'); - assert.deepEqual(report.summary, { pass: 1, fail: 1, not_verified: 55, total: 57 }); + assert.deepEqual(report.summary, { pass: 1, fail: 1, not_verified: 56, total: 58 }); }); test('report bytes are deterministic across observation and property orders', () => { @@ -165,11 +166,11 @@ test('missing observations remain unverified and every result identifies its hie const value = input(); value.observations = []; const report = buildReport(value); - assert.deepEqual(report.summary, { pass: 0, fail: 0, not_verified: 57, total: 57 }); + assert.deepEqual(report.summary, { pass: 0, fail: 0, not_verified: 58, total: 58 }); const skills = report.results.filter(({ id }) => id.startsWith('skills.')); const mcp = report.results.filter(({ id }) => id.startsWith('mcp.')); assert.deepEqual(skills.map(({ id }) => id), ['skills.discovery.immediate-children', 'skills.recovery.valid-skill-available', 'skills.recovery.invalid-mcp-document']); - assert.equal(mcp.length, 49); + assert.equal(mcp.length, 50); assert.equal(report.results.filter(({ id }) => id.startsWith('filesystem.')).length, 5); assert.ok(mcp.some(({ id }) => id === 'mcp.stdio.env.plugin-root')); for (const result of report.results) { @@ -294,7 +295,7 @@ test('data cwd follows resolved aliases while expansion preserves the original e runtime(value).dataWrite.path = '/private/tmp/data/.agent-plugins-conformance-write-test'; runtime(value, 'data').cwd = '/private/tmp/data'; const report = buildReport(value); - assert.equal(report.summary.pass, 18); + assert.equal(report.summary.pass, 19); assert.equal(runtime(report, 'data').resolvedData, '/private/tmp/data'); assert.equal(runtime(report).env.PLUGIN_DATA, '/tmp/data'); runtime(value, 'data').resolvedData = null; @@ -415,6 +416,64 @@ test('the platform command fixture establishes exact execution and attributable } }); +test('plugin-relative command resolution is independent from command token preservation', () => { + const resolutionId = 'mcp.stdio.command.plugin-relative-resolution'; + const tokenId = 'mcp.stdio.command.single-token'; + for (const [root, data, platform, tail] of [ + ['/fixture/plugin', '/state/plugin', 'posix', 'token.sh'], + ['C:\\fixture\\plugin', 'D:\\state\\plugin', 'windows', 'token.cmd'], + ]) { + const value = input(root, data); + const observation = value.observations.find(({ server }) => server === `command-token-${platform}`); + const expectedCwd = (platform === 'posix' ? path.posix : path.win32).join(root, 'probe-workdir'); + const status = (id) => result(buildReport(value), id).status; + + assert.equal(status(tokenId), 'pass'); + assert.equal(status(resolutionId), 'pass'); + + observation.evidence.argv = [observation.server, `${platform}-cwd-exact`, 'intact', + 'arg with spaces', '', 'literal-value']; + assert.equal(status(tokenId), 'pass'); + assert.equal(status(resolutionId), 'fail'); + + for (const origin of [`${platform}-decoy`, `${platform}-cwd-decoy`]) { + observation.evidence.argv = [observation.server, origin, 'split', tail, + 'arg with spaces', '', 'literal-value']; + assert.equal(status(tokenId), 'fail'); + assert.equal(status(resolutionId), 'not_verified'); + } + + observation.evidence.argv = [observation.server, 'unexpected-origin', 'intact']; + assert.equal(status(tokenId), 'not_verified'); + assert.equal(status(resolutionId), 'not_verified'); + + observation.evidence.argv = [observation.server, `${platform}-exact`, 'intact']; + observation.evidence.cwd = root; + assert.equal(status(tokenId), 'pass'); + assert.equal(status(resolutionId), 'not_verified'); + + observation.evidence.cwd = platform === 'posix' ? '/unexpected/cwd' : 'C:\\unexpected\\cwd'; + assert.equal(status(tokenId), 'pass'); + assert.equal(status(resolutionId), 'not_verified'); + + observation.evidence.cwd = expectedCwd; + observation.evidence.root = platform === 'posix' ? '/different/plugin' : 'C:\\different\\plugin'; + assert.equal(status(tokenId), 'not_verified'); + assert.equal(status(resolutionId), 'not_verified'); + } +}); + +test('missing command-resolution dependencies remain unverified', () => { + const resolutionId = 'mcp.stdio.command.plugin-relative-resolution'; + const value = input(); + const defaultObservation = value.observations.find(({ server }) => server === 'default'); + const commandObservation = value.observations.find(({ server }) => server === 'command-token-posix'); + for (const observations of [[], [defaultObservation], [commandObservation]]) { + assert.equal(result(buildReport({ schemaVersion: 1, observations }), resolutionId).status, + 'not_verified'); + } +}); + test('command-token attribution requires independent matching Core root evidence', () => { const id = 'mcp.stdio.command.single-token'; const core = input(); @@ -463,7 +522,7 @@ test('command-token observations stay outside cwd and plugin-data aggregate chec test('path comparisons follow producing OS and normalize path components', () => { for (const [root, data] of [['/plugin with spaces', '/data with spaces'], ['C:\\plugin', 'D:\\data'], ['\\\\host\\share\\plugin', '\\\\host\\share\\data']]) { const value = input(root, data); - assert.equal(buildReport(value).summary.pass, 18); + assert.equal(buildReport(value).summary.pass, 19); const flavor = root.startsWith('/') ? path.posix : path.win32; runtime(value).cwd = `${root}${flavor.sep}sub${flavor.sep}..`; assert.equal(result(buildReport(value), 'mcp.stdio.cwd.omitted').status, 'pass'); @@ -721,7 +780,7 @@ test('malformed MCP skill evidence affects only its own result for missing, corr assert.deepEqual(result(report, id).specSections, ['7.2.2']); assert.deepEqual(report.results.filter((item) => item.id !== id), baseline.results.filter((item) => item.id !== id)); assert.deepEqual(report.observations.find(({ skill }) => skill === observation.skill), observation); - assert.deepEqual(report.summary, { pass: status === 'pass' ? 22 : 21, fail: status === 'fail' ? 1 : 0, not_verified: 35, total: 57 }); + assert.deepEqual(report.summary, { pass: status === 'pass' ? 23 : 22, fail: status === 'fail' ? 1 : 0, not_verified: 35, total: 58 }); if (status === 'fail') assert.match(result(report, id).detail, /expected "APC_INVALID_MCP_VALID_V1"; observed "incorrect marker"/); const reversed = { schemaVersion: 1, observations: [...value.observations, observation].reverse() }; assert.equal(JSON.stringify(buildReport(reversed)), JSON.stringify(report));