Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions plugins/agent-plugins-conformance-core/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ Replace the path with a downloaded or installed copy of this plugin; the command
| `mcp.stdio.env.configured-precedence` | Configured environment values replace conflicting base-environment values. |
| `mcp.stdio.args.preservation-and-expansion` | Argument boundaries, including spaces and an empty argument, are preserved; recognized placeholders expand and unknown placeholder-like text stays literal. |
| `mcp.stdio.command.single-token` | A command path containing a space is preserved as a single executable token. |
| `mcp.stdio.command.plugin-relative-resolution` | Plugin-relative commands resolve against the plugin root, independently of the configured working directory. |
| `mcp.stdio.env.expansion` | Repeated recognized placeholders expand in environment values and unknown placeholder-like text stays literal. |
| `mcp.streamable-http.tool-availability` | The client exposes the HTTP tool and it returns a valid observation despite a conflicting, mixed-case configured `Accept` header. A completed discovery or call attempt with no observation fails this check only when the reporter confirms that the local fixture is healthy. |
| `mcp.streamable-http.url.literal-route-and-query` | The configured URL path and query are preserved literally. |
Expand Down
6 changes: 4 additions & 2 deletions plugins/agent-plugins-conformance-core/mcp.json
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,8 @@
"arg with spaces",
"",
"literal-value"
]
],
"cwd": "./probe-workdir"
},
"command-token-windows": {
"type": "stdio",
Expand All @@ -74,7 +75,8 @@
"arg with spaces",
"",
"literal-value"
]
],
"cwd": "./probe-workdir"
},
"sse": {
"type": "sse",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
#!/bin/sh
exec node "$(dirname "$0")/../../../dist/probe.mjs" command-token-posix posix-cwd-decoy split "$@"
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
#!/bin/sh
exec node "$(dirname "$0")/../../../dist/probe.mjs" command-token-posix posix-cwd-exact intact "$@"
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
@echo off
node "%~dp0..\..\..\dist\probe.mjs" command-token-windows windows-cwd-exact intact %*
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
@echo off
node "%~dp0..\..\..\dist\probe.mjs" command-token-windows windows-cwd-decoy split %*
1 change: 1 addition & 0 deletions plugins/agent-plugins-conformance/src/cases.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ export const CASES = Object.freeze([
['mcp.stdio.env.configured-value', 'Configured environment', ['9.1']],
['mcp.stdio.env.configured-precedence', 'Configured environment precedence', ['9.1']],
['mcp.stdio.command.single-token', 'Command token preservation', ['7.2.1']],
['mcp.stdio.command.plugin-relative-resolution', 'Plugin-relative command resolution', ['7.2.1']],
['mcp.stdio.args.preservation-and-expansion', 'Argument preservation and expansion', ['7.2.1', '9.2']],
['mcp.stdio.env.expansion', 'Environment expansion', ['9.2']],
['mcp.streamable-http.tool-availability', 'Streamable HTTP MCP tool evidence', ['6.1', '7.2.1']],
Expand Down
43 changes: 38 additions & 5 deletions plugins/agent-plugins-conformance/src/report.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,14 @@ import { CASES, MCP_CWD_VARIANTS } from './cases.mjs';
export { CASES, CASE_IDS } from './cases.mjs';
const CORE_SERVERS = Object.keys(MCP_CWD_VARIANTS);
const COMMAND_TOKEN_SERVERS = Object.freeze({
'command-token-posix': Object.freeze({ exactOrigin: 'posix-exact', decoyOrigin: 'posix-decoy', splitTail: 'token.sh' }),
'command-token-windows': Object.freeze({ exactOrigin: 'windows-exact', decoyOrigin: 'windows-decoy', splitTail: 'token.cmd' }),
'command-token-posix': Object.freeze({
rootExactOrigin: 'posix-exact', rootDecoyOrigin: 'posix-decoy',
cwdExactOrigin: 'posix-cwd-exact', cwdDecoyOrigin: 'posix-cwd-decoy', splitTail: 'token.sh',
}),
'command-token-windows': Object.freeze({
rootExactOrigin: 'windows-exact', rootDecoyOrigin: 'windows-decoy',
cwdExactOrigin: 'windows-cwd-exact', cwdDecoyOrigin: 'windows-cwd-decoy', splitTail: 'token.cmd',
}),
});
const INVALID_STDIO_SERVERS = Object.freeze({
'recovery-cwd-invalid-form': 'mcp.stdio.cwd.invalid-form',
Expand Down Expand Up @@ -569,6 +575,7 @@ export function buildReport(input) {
}
const defaultEvidence = runtime.get('default');
const commandTokenId = 'mcp.stdio.command.single-token';
const commandResolutionId = 'mcp.stdio.command.plugin-relative-resolution';
if (defaultEvidence) {
const defaultFlavor = pathFlavor(defaultEvidence.root);
const platform = defaultFlavor === path.win32 ? 'windows' : 'posix';
Expand All @@ -577,22 +584,48 @@ export function buildReport(input) {
const evidence = runtime.get(server);
if (!evidence) {
set(commandTokenId, 'not_verified', `No attributable ${platform} command-token observation was supplied.`);
set(commandResolutionId, 'not_verified',
`No attributable ${platform} plugin-relative command observation was supplied.`);
} else if (!samePath(evidence.root, defaultEvidence.root, defaultFlavor)) {
set(commandTokenId, 'not_verified',
`The ${platform} command-token observation did not identify the same installed plugin root as the Core default observation.`);
set(commandResolutionId, 'not_verified',
`The ${platform} plugin-relative command observation did not identify the same installed plugin root as the Core default observation.`);
} else {
const split = evidence.argv[0] === server && evidence.argv[1] === variant.decoyOrigin &&
const rootIntact = evidence.argv[0] === server && evidence.argv[1] === variant.rootExactOrigin &&
evidence.argv[2] === 'intact';
const cwdIntact = evidence.argv[0] === server && evidence.argv[1] === variant.cwdExactOrigin &&
evidence.argv[2] === 'intact';
const split = evidence.argv[0] === server &&
[variant.rootDecoyOrigin, variant.cwdDecoyOrigin].includes(evidence.argv[1]) &&
evidence.argv[2] === 'split' && evidence.argv[3] === variant.splitTail;
const intact = evidence.argv[0] === server && evidence.argv[1] === variant.exactOrigin && evidence.argv[2] === 'intact';
if (split) {
set(commandTokenId, 'fail',
`The ${platform} decoy wrapper received ${JSON.stringify(variant.splitTail)} as an argument, showing that the configured command was split.`);
} else if (intact) {
} else if (rootIntact || cwdIntact) {
set(commandTokenId, 'pass', `The ${platform} exact-name wrapper was selected.`);
} else {
set(commandTokenId, 'not_verified',
`The ${platform} command-token observation did not identify either the exact-name wrapper or an attributable split-name decoy.`);
}

const expectedCwd = defaultFlavor.join(defaultEvidence.root, 'probe-workdir');
if (!samePath(evidence.cwd, expectedCwd, defaultFlavor)) {
set(commandResolutionId, 'not_verified',
`The ${platform} plugin-relative command observation ran in ${JSON.stringify(evidence.cwd)} instead of the expected working directory ${JSON.stringify(expectedCwd)}.`);
} else if (split) {
set(commandResolutionId, 'not_verified',
`The ${platform} command was split before resolution, so plugin-relative resolution was not evaluated.`);
} else if (rootIntact) {
set(commandResolutionId, 'pass',
`The ${platform} plugin-root exact-name wrapper was selected while the configured working directory remained active.`);
} else if (cwdIntact) {
set(commandResolutionId, 'fail',
`The ${platform} working-directory exact-name wrapper was selected, showing that the plugin-relative command was resolved against the subprocess working directory.`);
} else {
set(commandResolutionId, 'not_verified',
`The ${platform} command observation did not identify an intact exact-name wrapper, so plugin-relative resolution was not evaluated.`);
}
}
}
if (skills.has('conformance-recovery-valid')) {
Expand Down
2 changes: 1 addition & 1 deletion scripts/smoke-codex.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -271,12 +271,12 @@ try {
: invalidSseServers.includes(server) ? 'mcp-sse' : 'mcp-stdio',
`${server}: missing observation`);
assert.equal(observation.server, server, `${server}: unexpected observation server`);
record({ action: 'record', observation });
if (server === commandServer) {
assert.deepEqual(observation.evidence.argv,
[commandServer, `${commandPlatform}-exact`, 'intact', 'arg with spaces', '', 'literal-value'],
'Native launch did not preserve the command token and configured arguments');
}
record({ action: 'record', observation });
}
// SSE is optional. Preserve native diagnostics even when the loader omits entries.
const sseObservations = [];
Expand Down
108 changes: 82 additions & 26 deletions test/command-token.test.mjs
Original file line number Diff line number Diff line change
@@ -1,19 +1,22 @@
import assert from 'node:assert/strict';
import { cp, lstat, mkdtemp, readFile, realpath, readdir, rm, writeFile } from 'node:fs/promises';
import { cp, lstat, mkdir, mkdtemp, readFile, realpath, readdir, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import test from 'node:test';
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
import { buildReport } from '../plugins/agent-plugins-conformance/src/report.mjs';

const configuredArgs = ['arg with spaces', '', 'literal-value'];

async function fixture(t) {
const parent = await mkdtemp(join(tmpdir(), 'apc command token '));
const root = join(await realpath(parent), 'installed plugin with spaces');
const data = join(await realpath(parent), 'plugin data with spaces');
const clients = [];
const protocolErrors = [];
await cp(new URL('../plugins/agent-plugins-conformance-core/', import.meta.url), root, { recursive: true });
await mkdir(data);
t.after(async () => {
try {
const results = await Promise.allSettled(clients.map((client) => client.close()));
Expand All @@ -25,14 +28,46 @@ async function fixture(t) {
}
});
const config = JSON.parse(await readFile(join(root, 'mcp.json'), 'utf8'));
return { parent, root, clients, config, protocolErrors };
return { parent, root, data, clients, config, protocolErrors };
}

async function connect(files, server, { split = false, batch = false } = {}) {
async function connectTransport(files, options) {
const client = new Client({ name: 'command-token-reference-test', version: '1' });
client.onerror = (error) => files.protocolErrors.push(error);
files.clients.push(client);
await client.connect(new StdioClientTransport({ ...options, stderr: 'pipe' }));
return client;
}

function baseEnvironment() {
const env = Object.fromEntries(Object.entries(process.env));
for (const name of ['PLUGIN_ROOT', 'PLUGIN_DATA', 'APC_VALUE', 'APC_EXPANSION', 'APC_LITERAL']) delete env[name];
return env;
}

async function connectDefault(files) {
return connectTransport(files, {
command: process.execPath,
args: [join(files.root, 'dist/probe.mjs'), 'default'],
cwd: files.root,
env: { ...baseEnvironment(), PLUGIN_ROOT: files.root, PLUGIN_DATA: files.data },
});
}

async function connect(files, server, { base = 'root', split = false, batch = false } = {}) {
const configured = files.config.mcpServers[server];
let command = resolve(files.root, configured.command.slice(2));
const tail = process.platform === 'win32' ? 'token.cmd' : 'token.sh';
const cwd = resolve(files.root, configured.cwd.slice(2));
let command = resolve(base === 'cwd' ? cwd : files.root, configured.command.slice(2));
let args = configured.args;
if (split) {
if (split === 'root-harness') {
// Simulate the attributable result of splitting the command while resolving
// its first token from the plugin root. Native clients choose the resolution
// base before the fixture can observe it, so this is intentionally a harness.
command = resolve(files.root, configured.command.slice(2).replace(` ${tail}`, '')) +
(process.platform === 'win32' ? '.cmd' : '');
args = [tail, ...configured.args];
} else if (split === 'cwd-shell') {
if (process.platform === 'win32') {
command = join(files.parent, 'bad command token launcher.cmd');
await writeFile(command, '@echo off\r\n.\\bin\\windows\\probe token.cmd %*\r\n');
Expand All @@ -44,15 +79,11 @@ async function connect(files, server, { split = false, batch = false } = {}) {
if (batch) {
// Preserve ordinary batch echo settings; the fixture must keep stdout protocol-clean.
command = join(files.parent, 'ordinary batch launcher.cmd');
await writeFile(command, '@call ".\\bin\\windows\\probe token.cmd" %*\r\n');
await writeFile(command,
`@call "${resolve(files.root, configured.command.slice(2))}" %*\r\n`);
}
const env = Object.fromEntries(Object.entries(process.env));
for (const name of ['PLUGIN_ROOT', 'PLUGIN_DATA', 'APC_VALUE', 'APC_EXPANSION', 'APC_LITERAL']) delete env[name];
const client = new Client({ name: 'command-token-reference-test', version: '1' });
client.onerror = (error) => files.protocolErrors.push(error);
files.clients.push(client);
await client.connect(new StdioClientTransport({ command, args, cwd: files.root, env, stderr: 'pipe' }));
return client;
const env = baseEnvironment();
return connectTransport(files, { command, args, cwd, env });
}

async function observe(client) {
Expand All @@ -64,22 +95,41 @@ async function observe(client) {
return result.structuredContent;
}

test('the copied Core fixture preserves the platform command token and configured arguments',
test('the copied Core fixture completes MCP exchanges from all four attributable command origins',
{ timeout: 30_000 }, async (t) => {
const files = await fixture(t);
assert.equal((await readdir(files.root)).includes('node_modules'), false);
const platform = process.platform === 'win32' ? 'windows' : 'posix';
const tail = process.platform === 'win32' ? 'token.cmd' : 'token.sh';
const server = `command-token-${platform}`;
const observation = await observe(await connect(files, server));
assert.equal(observation.kind, 'mcp-stdio');
assert.equal(observation.server, server);
assert.equal(observation.evidence.server, server);
assert.equal(observation.evidence.root, await realpath(files.root));
assert.equal(observation.evidence.cwd, await realpath(files.root));
assert.equal(observation.evidence.resolvedData, null);
assert.deepEqual(observation.evidence.env, {});
assert.deepEqual(observation.evidence.argv,
[server, `${platform}-exact`, 'intact', ...configuredArgs]);
const expectedCwd = await realpath(join(files.root, 'probe-workdir'));
const defaultObservation = await observe(await connectDefault(files));
for (const [variant, origin, marker, prefix, tokenStatus, resolutionStatus] of [
[undefined, `${platform}-exact`, 'intact', [], 'pass', 'pass'],
['root-harness', `${platform}-decoy`, 'split', [tail], 'fail', 'not_verified'],
['cwd-exact', `${platform}-cwd-exact`, 'intact', [], 'pass', 'fail'],
['cwd-shell', `${platform}-cwd-decoy`, 'split', [tail], 'fail', 'not_verified'],
]) {
const options = variant === 'cwd-exact' ? { base: 'cwd' } : { split: variant };
const observation = await observe(await connect(files, server, options));
assert.equal(observation.kind, 'mcp-stdio', variant);
assert.equal(observation.server, server, variant);
assert.equal(observation.evidence.server, server, variant);
assert.equal(observation.evidence.root, await realpath(files.root), variant);
assert.equal(observation.evidence.cwd, expectedCwd, variant);
assert.equal(observation.evidence.resolvedData, null, variant);
assert.deepEqual(observation.evidence.env, {}, variant);
assert.deepEqual(observation.evidence.argv,
[server, origin, marker, ...prefix, ...configuredArgs], variant);
const report = buildReport({
schemaVersion: 1,
observations: [defaultObservation, observation],
});
assert.equal(report.results.find(({ id }) =>
id === 'mcp.stdio.command.single-token').status, tokenStatus, variant);
assert.equal(report.results.find(({ id }) =>
id === 'mcp.stdio.command.plugin-relative-resolution').status, resolutionStatus, variant);
}
});

test('deliberate unquoted platform-shell parsing reaches the split-name decoy',
Expand All @@ -88,11 +138,11 @@ test('deliberate unquoted platform-shell parsing reaches the split-name decoy',
const platform = process.platform === 'win32' ? 'windows' : 'posix';
const tail = process.platform === 'win32' ? 'token.cmd' : 'token.sh';
const server = `command-token-${platform}`;
const observation = await observe(await connect(files, server, { split: true }));
const observation = await observe(await connect(files, server, { split: 'cwd-shell' }));
assert.equal(observation.kind, 'mcp-stdio');
assert.equal(observation.server, server);
assert.deepEqual(observation.evidence.argv,
[server, `${platform}-decoy`, 'split', tail, ...configuredArgs]);
[server, `${platform}-cwd-decoy`, 'split', tail, ...configuredArgs]);
});

test('explicit Windows batch execution preserves the command and arguments without non-MCP stdout',
Expand All @@ -102,13 +152,19 @@ test('explicit Windows batch execution preserves the command and arguments witho
const observation = await observe(await connect(files, server, { batch: true }));
assert.deepEqual(observation.evidence.argv,
[server, 'windows-exact', 'intact', ...configuredArgs]);
assert.equal(observation.evidence.cwd, await realpath(join(files.root, 'probe-workdir')));
});

test('Core packages paired native wrappers with the intended executable modes',
{ skip: process.platform === 'win32' }, async () => {
const root = new URL('../plugins/agent-plugins-conformance-core/bin/', import.meta.url);
const cwd = new URL('../plugins/agent-plugins-conformance-core/probe-workdir/bin/', import.meta.url);
assert.notEqual((await lstat(new URL('posix/probe token.sh', root))).mode & 0o111, 0);
assert.notEqual((await lstat(new URL('posix/probe', root))).mode & 0o111, 0);
assert.equal((await lstat(new URL('windows/probe token.cmd', root))).mode & 0o111, 0);
assert.equal((await lstat(new URL('windows/probe.cmd', root))).mode & 0o111, 0);
assert.notEqual((await lstat(new URL('posix/probe token.sh', cwd))).mode & 0o111, 0);
assert.notEqual((await lstat(new URL('posix/probe', cwd))).mode & 0o111, 0);
assert.equal((await lstat(new URL('windows/probe token.cmd', cwd))).mode & 0o111, 0);
assert.equal((await lstat(new URL('windows/probe.cmd', cwd))).mode & 0o111, 0);
});
13 changes: 13 additions & 0 deletions test/package.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,19 @@ test('plugin and MCP target the same published specification version', async ()
assert.equal((await load('mcp')).$schema, 'https://agent-plugins.org/schemas/1.0.0/mcp.schema.json');
});

test('command-token MCP servers use the shared plugin-relative working directory', async () => {
const mcp = JSON.parse(await readFile(
new URL('../plugins/agent-plugins-conformance-core/mcp.json', import.meta.url), 'utf8'));
for (const platform of ['posix', 'windows']) {
assert.deepEqual(mcp.mcpServers[`command-token-${platform}`], {
type: 'stdio',
command: `./bin/${platform}/probe token.${platform === 'posix' ? 'sh' : 'cmd'}`,
args: ['arg with spaces', '', 'literal-value'],
cwd: './probe-workdir',
});
}
});

test('primary run skill is separate from the core discovery fixture layout', async () => {
const primary = new URL('../plugins/agent-plugins-conformance/', import.meta.url);
const core = new URL('../plugins/agent-plugins-conformance-core/', import.meta.url);
Expand Down
2 changes: 1 addition & 1 deletion test/probe.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ test('copied plugin runs only MCP observation tools without node_modules', { tim
const direct = buildReport(reportInput);
assert.equal(direct.summary.fail, 0);
assert.equal(direct.summary.pass, 16);
assert.equal(direct.summary.not_verified, 41);
assert.equal(direct.summary.not_verified, 42);
});

test('faulty ambient precedence becomes a normal reporter failure for both platform variables', { timeout: 30_000 }, async (t) => {
Expand Down
Loading
Loading