diff --git a/plugins/agent-plugins-conformance-core/README.md b/plugins/agent-plugins-conformance-core/README.md index cf883c1..ce237c3 100644 --- a/plugins/agent-plugins-conformance-core/README.md +++ b/plugins/agent-plugins-conformance-core/README.md @@ -31,7 +31,8 @@ Replace the path with a downloaded or installed copy of this plugin; the command | `mcp.stdio.env.plugin-data-absolute` | Supplied `PLUGIN_DATA` is an absolute path. This check does not exercise writability, storage dedication, or persistence across updates. | | `filesystem.data.writable` | The default server can create, write, and close a small temporary file directly in the supplied `PLUGIN_DATA` directory. | | `filesystem.data.consistent-within-plugin` | Servers using omitted and data-rooted working directories resolve `PLUGIN_DATA` to the same filesystem path. | -| `mcp.stdio.env.configured-value` | The configured value reaches the subprocess. The fixture does not arrange a conflicting inherited value, so it does not establish override behavior. | +| `mcp.stdio.env.configured-value` | The configured value reaches the subprocess. | +| `mcp.stdio.env.configured-precedence` | Configured environment values replace conflicting base-environment values. | | `mcp.stdio.args.preservation-and-expansion` | Argument boundaries, including spaces and an empty argument, are preserved; recognized placeholders expand and unknown placeholder-like text stays literal. | | `mcp.stdio.command.single-token` | A command path containing a space is preserved as a single executable token. | | `mcp.stdio.env.expansion` | Repeated recognized placeholders expand in environment values and unknown placeholder-like text stays literal. | diff --git a/plugins/agent-plugins-conformance-core/dist/probe.mjs b/plugins/agent-plugins-conformance-core/dist/probe.mjs index aeb08c3..31c3a10 100644 --- a/plugins/agent-plugins-conformance-core/dist/probe.mjs +++ b/plugins/agent-plugins-conformance-core/dist/probe.mjs @@ -16972,6 +16972,7 @@ if (!["default", "relative", "root", "data", "command-token-posix", "command-tok } var root = realpathSync.native(fileURLToPath(new URL("..", import.meta.url))); var environmentNames = ["PLUGIN_ROOT", "PLUGIN_DATA", "APC_VALUE", "APC_EXPANSION", "APC_LITERAL"]; +var environmentPrecedenceNames = ["USER", "USERNAME"]; var resolvedData = null; if (process.env.PLUGIN_DATA && isAbsolute2(process.env.PLUGIN_DATA)) { try { @@ -16979,6 +16980,10 @@ if (process.env.PLUGIN_DATA && isAbsolute2(process.env.PLUGIN_DATA)) { } catch { } } +var environment = Object.fromEntries(environmentNames.filter((name) => process.env[name] !== void 0).map((name) => [name, process.env[name]])); +if (["default", "data"].includes(serverName)) { + for (const name of environmentPrecedenceNames) environment[name] = process.env[name] ?? null; +} var launch = { server: serverName, root, @@ -16986,7 +16991,7 @@ var launch = { // Compare directory identity even when the OS retains a junction/alias spelling. cwd: realpathSync.native(process.cwd()), argv: process.argv.slice(2), - env: Object.fromEntries(environmentNames.filter((name) => process.env[name] !== void 0).map((name) => [name, process.env[name]])) + env: environment }; var server = new Server( { name: `agent-plugins-conformance-${serverName}`, version: "0.1.0" }, @@ -16994,7 +16999,7 @@ var server = new Server( ); var observeTool = { name: "observe", - description: "Return this process launch evidence, including only fixture environment variables." + (serverName === "default" ? " Each call also creates, writes, closes, and removes a uniquely named temporary file directly in an absolute PLUGIN_DATA directory, recording any operation or cleanup error." : "") + " Record the observation object from structuredContent (or parsed JSON text) unchanged with the run-conformance reporter; exclude the MCP result wrapper.", + description: "Return this process launch evidence, including selected environment variables." + (serverName === "default" ? " Each call also creates, writes, closes, and removes a uniquely named temporary file directly in an absolute PLUGIN_DATA directory, recording any operation or cleanup error." : "") + " Record the observation object from structuredContent (or parsed JSON text) unchanged with the run-conformance reporter; exclude the MCP result wrapper.", inputSchema: { type: "object", properties: {}, additionalProperties: false }, annotations: { readOnlyHint: serverName !== "default", destructiveHint: false, idempotentHint: serverName !== "default", openWorldHint: false } }; diff --git a/plugins/agent-plugins-conformance-core/mcp.json b/plugins/agent-plugins-conformance-core/mcp.json index 9efec50..4427fbf 100644 --- a/plugins/agent-plugins-conformance-core/mcp.json +++ b/plugins/agent-plugins-conformance-core/mcp.json @@ -18,7 +18,9 @@ "env": { "APC_VALUE": "fixture value with spaces", "APC_EXPANSION": "${PLUGIN_ROOT}|${PLUGIN_DATA}|${PLUGIN_ROOT}", - "APC_LITERAL": "${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}" + "APC_LITERAL": "${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}", + "USER": "apc-configured-environment-precedence", + "USERNAME": "apc-configured-environment-precedence" } }, "relative": { diff --git a/plugins/agent-plugins-conformance-core/src/probe.mjs b/plugins/agent-plugins-conformance-core/src/probe.mjs index eea9d8a..860907a 100644 --- a/plugins/agent-plugins-conformance-core/src/probe.mjs +++ b/plugins/agent-plugins-conformance-core/src/probe.mjs @@ -14,10 +14,16 @@ if (!['default', 'relative', 'root', 'data', 'command-token-posix', 'command-tok // This value is independent of the client-provided PLUGIN_ROOT environment. const root = realpathSync.native(fileURLToPath(new URL('..', import.meta.url))); const environmentNames = ['PLUGIN_ROOT', 'PLUGIN_DATA', 'APC_VALUE', 'APC_EXPANSION', 'APC_LITERAL']; +const environmentPrecedenceNames = ['USER', 'USERNAME']; let resolvedData = null; if (process.env.PLUGIN_DATA && isAbsolute(process.env.PLUGIN_DATA)) { try { resolvedData = realpathSync.native(process.env.PLUGIN_DATA); } catch { /* Unobservable target remains null. */ } } +const environment = Object.fromEntries(environmentNames.filter((name) => process.env[name] !== undefined) + .map((name) => [name, process.env[name]])); +if (['default', 'data'].includes(serverName)) { + for (const name of environmentPrecedenceNames) environment[name] = process.env[name] ?? null; +} const launch = { server: serverName, root, @@ -25,14 +31,13 @@ const launch = { // Compare directory identity even when the OS retains a junction/alias spelling. cwd: realpathSync.native(process.cwd()), argv: process.argv.slice(2), - env: Object.fromEntries(environmentNames.filter((name) => process.env[name] !== undefined) - .map((name) => [name, process.env[name]])), + env: environment, }; const server = new Server({ name: `agent-plugins-conformance-${serverName}`, version: '0.1.0' }, { capabilities: { tools: {} } }); const observeTool = { name: 'observe', - description: 'Return this process launch evidence, including only fixture environment variables.' + + description: 'Return this process launch evidence, including selected environment variables.' + (serverName === 'default' ? ' Each call also creates, writes, closes, and removes a uniquely named temporary file directly in an absolute PLUGIN_DATA directory, recording any operation or cleanup error.' : '') + ' Record the observation object from structuredContent (or parsed JSON text) unchanged with the run-conformance reporter; exclude the MCP result wrapper.', inputSchema: { type: 'object', properties: {}, additionalProperties: false }, diff --git a/plugins/agent-plugins-conformance/src/cases.mjs b/plugins/agent-plugins-conformance/src/cases.mjs index df64eaa..d073d3b 100644 --- a/plugins/agent-plugins-conformance/src/cases.mjs +++ b/plugins/agent-plugins-conformance/src/cases.mjs @@ -27,6 +27,7 @@ export const CASES = Object.freeze([ ['filesystem.data.distinct-across-plugins', 'Distinct data directories across plugins', ['9.1']], ['filesystem.data.consistent-within-plugin', 'Consistent data directory within a plugin', ['9.1']], ['mcp.stdio.env.configured-value', 'Configured environment', ['9.1']], + ['mcp.stdio.env.configured-precedence', 'Configured environment precedence', ['9.1']], ['mcp.stdio.command.single-token', 'Command token preservation', ['7.2.1']], ['mcp.stdio.args.preservation-and-expansion', 'Argument preservation and expansion', ['7.2.1', '9.2']], ['mcp.stdio.env.expansion', 'Environment expansion', ['9.2']], diff --git a/plugins/agent-plugins-conformance/src/report.mjs b/plugins/agent-plugins-conformance/src/report.mjs index 843d7a9..58f96c5 100644 --- a/plugins/agent-plugins-conformance/src/report.mjs +++ b/plugins/agent-plugins-conformance/src/report.mjs @@ -55,7 +55,12 @@ const SKILLS = { 'conformance-recovery-valid': 'APC_RECOVERY_VALID_V1', 'conformance-invalid-mcp-valid': 'APC_INVALID_MCP_VALID_V1', }; -const ENV_KEYS = ['PLUGIN_ROOT', 'PLUGIN_DATA', 'APC_VALUE', 'APC_EXPANSION', 'APC_LITERAL']; +const ENVIRONMENT_PRECEDENCE_KEYS = ['USER', 'USERNAME']; +const ENVIRONMENT_PRECEDENCE_VALUE = 'apc-configured-environment-precedence'; +const ENV_KEYS = [ + 'PLUGIN_ROOT', 'PLUGIN_DATA', 'APC_VALUE', 'APC_EXPANSION', 'APC_LITERAL', + ...ENVIRONMENT_PRECEDENCE_KEYS, +]; function invalid(at, reason) { throw new TypeError(`${at}: ${reason}`); @@ -295,7 +300,10 @@ export function validateInput(input, { recording = false } = {}) { array(evidence.argv, `${evidenceAt}.argv`); evidence.argv.forEach((argument, i) => string(argument, `${evidenceAt}.argv[${i}]`, true)); object(evidence.env, ENV_KEYS, [], `${evidenceAt}.env`); - for (const [key, value] of Object.entries(evidence.env)) string(value, `${evidenceAt}.env.${key}`, true); + for (const [key, value] of Object.entries(evidence.env)) { + if (ENVIRONMENT_PRECEDENCE_KEYS.includes(key) && value === null) continue; + string(value, `${evidenceAt}.env.${key}`, true); + } if (observation.server === 'default' && evidence.dataWrite !== null) { const at = `${evidenceAt}.dataWrite`; const write = evidence.dataWrite; @@ -716,6 +724,37 @@ export function buildReport(input) { `default resolved PLUGIN_DATA to ${JSON.stringify(defaultData)}. data resolved PLUGIN_DATA to ${JSON.stringify(dataCwdData)}.`); } const evidence = runtime.get('default'); + const precedenceCase = 'mcp.stdio.env.configured-precedence'; + const environmentPrecedenceEvidence = Object.fromEntries(['default', 'data'] + .map((server) => [server, runtime.get(server)])); + const missingPrecedenceServers = Object.entries(environmentPrecedenceEvidence) + .filter(([, value]) => value === undefined).map(([server]) => server); + const missingPrecedenceKeys = Object.entries(environmentPrecedenceEvidence).flatMap(([server, value]) => value === undefined + ? [] + : ENVIRONMENT_PRECEDENCE_KEYS.filter((key) => !Object.hasOwn(value.env, key)) + .map((key) => `${server}.${key}`)); + if (missingPrecedenceServers.length > 0) { + set(precedenceCase, 'not_verified', + `Missing MCP observation for: ${missingPrecedenceServers.join(', ')}.`); + } else if (missingPrecedenceKeys.length > 0) { + set(precedenceCase, 'not_verified', + `Environment precedence evidence is missing: ${missingPrecedenceKeys.join(', ')}.`); + } else { + const configuredDifferences = ENVIRONMENT_PRECEDENCE_KEYS + .filter((key) => environmentPrecedenceEvidence.default.env[key] !== ENVIRONMENT_PRECEDENCE_VALUE) + .map((key) => mismatch(key, ENVIRONMENT_PRECEDENCE_VALUE, environmentPrecedenceEvidence.default.env[key])); + if (configuredDifferences.length > 0) { + set(precedenceCase, 'fail', + `Configured environment values were not delivered. ${configuredDifferences.join(' ')}`); + } else { + const differingControls = ENVIRONMENT_PRECEDENCE_KEYS.filter((key) => + typeof environmentPrecedenceEvidence.data.env[key] === 'string' && + environmentPrecedenceEvidence.data.env[key] !== ENVIRONMENT_PRECEDENCE_VALUE); + set(precedenceCase, differingControls.length > 0 ? 'pass' : 'not_verified', differingControls.length > 0 + ? `Configured values replaced different inherited control values for: ${differingControls.join(', ')}.` + : 'No control value differed from the configured value, so replacement was not observable.'); + } + } if (evidence) { const { root, env, argv } = evidence; const flavor = pathFlavor(root); diff --git a/test/probe.test.mjs b/test/probe.test.mjs index 6d773f9..104c210 100644 --- a/test/probe.test.mjs +++ b/test/probe.test.mjs @@ -37,7 +37,9 @@ async function connect(fixture, mode, override = {}) { const client = new Client({ name: 'conformance-reference-test', version: '1' }); const transport = new StdioClientTransport({ command: process.execPath, args: config.args.map(expand), cwd, - env: { ...variables, ...Object.fromEntries(Object.entries(config.env ?? {}).map(([key, value]) => [key, expand(value)])), APC_SHOULD_NOT_LEAK: 'unrelated ambient sentinel' }, + env: { USER: 'ambient-user', USERNAME: 'ambient-username', ...variables, + ...Object.fromEntries(Object.entries(config.env ?? {}).map(([key, value]) => [key, expand(value)])), + APC_SHOULD_NOT_LEAK: 'unrelated ambient sentinel' }, stderr: 'pipe', ...override, }); fixture.clients.push(client); @@ -61,6 +63,16 @@ test('copied plugin runs only MCP observation tools without node_modules', { tim assert.deepEqual(result.structuredContent, observation); assert.equal(observation.evidence.root, files.root); assert.equal(observation.evidence.env.APC_SHOULD_NOT_LEAK, undefined); + if (mode === 'default') { + assert.equal(observation.evidence.env.USER, 'apc-configured-environment-precedence'); + assert.equal(observation.evidence.env.USERNAME, 'apc-configured-environment-precedence'); + } else if (mode === 'data') { + assert.equal(observation.evidence.env.USER, 'ambient-user'); + assert.equal(observation.evidence.env.USERNAME, 'ambient-username'); + } else { + assert.equal(Object.hasOwn(observation.evidence.env, 'USER'), false); + assert.equal(Object.hasOwn(observation.evidence.env, 'USERNAME'), false); + } if (mode === 'default') { assert.equal(dirname(observation.evidence.dataWrite.path), files.data); assert.equal(observation.evidence.dataWrite.error, null); @@ -93,10 +105,34 @@ test('copied plugin runs only MCP observation tools without node_modules', { tim const reportInput = input(observations); const direct = buildReport(reportInput); assert.equal(direct.summary.fail, 0); - assert.equal(direct.summary.pass, 15); + assert.equal(direct.summary.pass, 16); assert.equal(direct.summary.not_verified, 41); }); +test('faulty ambient precedence becomes a normal reporter failure for both platform variables', { timeout: 30_000 }, async (t) => { + const files = await fixture(t); + const variables = { PLUGIN_ROOT: files.root, PLUGIN_DATA: files.data }; + const expand = (value) => value.replace(/\$\{(PLUGIN_ROOT|PLUGIN_DATA)\}/g, (_, name) => variables[name]); + const configured = Object.fromEntries(Object.entries(files.config.mcpServers.default.env) + .map(([name, value]) => [name, expand(value)])); + for (const name of ['USER', 'USERNAME']) { + const control = await connect(files, 'data', { env: { + ...variables, USER: 'ambient-user', USERNAME: 'ambient-username', + } }); + const faulty = await connect(files, 'default', { env: { + ...variables, ...configured, [name]: `ambient-${name.toLowerCase()}`, + } }); + const [controlResult, faultyResult] = await Promise.all([ + control.callTool({ name: 'observe', arguments: {} }), + faulty.callTool({ name: 'observe', arguments: {} }), + ]); + const report = buildReport(input([faultyResult.structuredContent, controlResult.structuredContent])); + const outcome = report.results.find(({ id }) => id === 'mcp.stdio.env.configured-precedence'); + assert.equal(outcome.status, 'fail', name); + assert.match(outcome.detail, new RegExp(`${name}: expected`)); + } +}); + test('copied recovery plugin serves exact valid and invalid-server observations without runtime dependencies', { timeout: 30_000 }, async (t) => { const parent = await mkdtemp(join(tmpdir(), 'agent-plugins-conformance-recovery-')); const root = join(parent, 'copied recovery plugin'); diff --git a/test/record-report.test.mjs b/test/record-report.test.mjs index c3009a8..70ce358 100644 --- a/test/record-report.test.mjs +++ b/test/record-report.test.mjs @@ -20,9 +20,16 @@ const mcp = () => ({ env: { PLUGIN_ROOT: '/plugin', PLUGIN_DATA: '/data', APC_VALUE: 'fixture value with spaces', APC_EXPANSION: '/plugin|/data|/plugin', APC_LITERAL: '${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}', + USER: 'apc-configured-environment-precedence', USERNAME: 'apc-configured-environment-precedence', }, }, }); +const dataMcp = (env = { USER: null, USERNAME: 'ambient-username' }) => ({ + kind: 'mcp-stdio', server: 'data', evidence: { + version: 1, server: 'data', root: '/plugin', cwd: '/data', resolvedData: '/data', + argv: ['data'], env, + }, +}); const commandToken = (platform = 'posix', { root = platform === 'posix' ? '/plugin' : 'C:\\plugin', data = platform === 'posix' ? '/data' : 'D:\\data', @@ -393,6 +400,23 @@ test('distinct keys accumulate in deterministic order; repeated keys fully repla assert.deepEqual(await readdir(join(f.directory, 'nested directory')), ['report with spaces.json']); }); +test('recorder preserves nullable environment precedence evidence through canonical reloads', async (t) => { + const f = await fixture(t); + f.success(start); + f.record(dataMcp()); + f.record(mcp()); + let report = await f.read(); + assert.deepEqual(report.observations.filter(({ server }) => ['default', 'data'].includes(server)), [mcp(), dataMcp()]); + assert.equal(report.results.find(({ id }) => id === 'mcp.stdio.env.configured-precedence').status, 'pass'); + assert.equal(report.observations.find(({ server }) => server === 'data').evidence.env.USER, null); + + const oldData = dataMcp({}); + f.record(oldData); + report = await f.read(); + assert.deepEqual(report.observations.find(({ server }) => server === 'data'), oldData); + assert.equal(report.results.find(({ id }) => id === 'mcp.stdio.env.configured-precedence').status, 'not_verified'); +}); + test('command-token evidence records canonically and replacement changes only its verdict', async (t) => { const f = await fixture(t); f.success(start); diff --git a/test/report-human.test.mjs b/test/report-human.test.mjs index d984d0b..ef98ab5 100644 --- a/test/report-human.test.mjs +++ b/test/report-human.test.mjs @@ -6,6 +6,9 @@ import { formatReport } from '../plugins/agent-plugins-conformance/src/report-fo function input() { const root = '/fixture/plugin'; const data = '/state/plugin'; + const precedenceEnv = (server) => server === 'default' + ? { USER: 'apc-configured-environment-precedence', USERNAME: 'apc-configured-environment-precedence' } + : server === 'data' ? { USER: 'ambient-user', USERNAME: 'ambient-username' } : {}; return { schemaVersion: 1, observations: [ @@ -18,7 +21,8 @@ function input() { cwd: server === 'default' ? root : server === 'data' ? data : `${root}/probe-workdir`, argv: ['default', 'arg with spaces', '', root, data, '${APC_UNKNOWN}', '$APC_VALUE', '${PLUGIN_ROOT_SUFFIX}'], env: { PLUGIN_ROOT: root, PLUGIN_DATA: data, APC_VALUE: 'fixture value with spaces', - APC_EXPANSION: `${root}|${data}|${root}`, APC_LITERAL: '${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}' }, + APC_EXPANSION: `${root}|${data}|${root}`, APC_LITERAL: '${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}', + ...precedenceEnv(server) }, }, })), { @@ -59,9 +63,9 @@ test('complete stdio and recovery evidence leaves optional HTTP and SSE checks u '', 'Checks Passed Failed Not verified', 'Skills 3 0 0', - 'MCP 22 0 26', + 'MCP 23 0 26', 'Filesystem 4 0 1', - 'Total 29 0 27', + 'Total 30 0 27', ].join('\n')); const missing = human.split('\n\nNot verified\n')[1]; for (const id of [ @@ -100,7 +104,7 @@ test('all-unverified human report preserves every saved missing-evidence result' value.observations = []; const human = formatReport(buildReport(value)); assert.match(human, /Skills\s+0\s+0\s+3/); - assert.match(human, /MCP\s+0\s+0\s+48/); + assert.match(human, /MCP\s+0\s+0\s+49/); assert.match(human, /Filesystem\s+0\s+0\s+5/); assert.match(human, /Missing skill observations: conformance-alpha, conformance-beta\./); for (const result of buildReport(value).results) assert.ok(human.includes(`(${result.id})`)); @@ -112,7 +116,7 @@ test('mixed human report puts failures before missing-evidence details', () => { value.observations = [value.observations[2]]; value.observations[0].evidence.env.APC_VALUE = 'incorrect configured value'; const human = formatReport(buildReport(value)); - assert.match(human, /MCP\s+6\s+1\s+41/); + assert.match(human, /MCP\s+6\s+1\s+42/); assert.match(human, /Filesystem\s+1\s+0\s+4/); assert.match(human, /Configured environment \(mcp.stdio.env.configured-value\)/); assert.match(human, /"fixture value with spaces"/); @@ -169,7 +173,7 @@ test('human report renders saved warnings independently of result status', () => const writable = value.results.find(({ id }) => id === 'filesystem.data.writable'); writable.status = 'not_verified'; writable.warning = 'Saved cleanup warning for /state/plugin/leftover.'; - value.summary = { pass: 16, fail: 0, not_verified: 40, total: 56 }; + value.summary = { pass: 17, fail: 0, not_verified: 40, total: 57 }; value.observations = []; const human = formatReport(value); diff --git a/test/report.test.mjs b/test/report.test.mjs index ba89b6d..bcb41fd 100644 --- a/test/report.test.mjs +++ b/test/report.test.mjs @@ -6,6 +6,9 @@ import { buildReport, CASE_IDS } from '../plugins/agent-plugins-conformance/src/ function input(root = '/fixture/plugin', data = '/state/plugin') { const flavor = root.startsWith('/') ? path.posix : path.win32; const platform = flavor === path.win32 ? 'windows' : 'posix'; + const precedenceEnv = (server) => server === 'default' + ? { USER: 'apc-configured-environment-precedence', USERNAME: 'apc-configured-environment-precedence' } + : server === 'data' ? { USER: 'ambient-user', USERNAME: 'ambient-username' } : {}; return { schemaVersion: 1, observations: [ @@ -19,7 +22,8 @@ function input(root = '/fixture/plugin', data = '/state/plugin') { cwd: server === 'default' ? root : server === 'data' ? data : flavor.join(root, 'probe-workdir'), argv: ['default', 'arg with spaces', '', root, data, '${APC_UNKNOWN}', '$APC_VALUE', '${PLUGIN_ROOT_SUFFIX}'], env: { PLUGIN_ROOT: root, PLUGIN_DATA: data, APC_VALUE: 'fixture value with spaces', - APC_EXPANSION: `${root}|${data}|${root}`, APC_LITERAL: '${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}' }, + APC_EXPANSION: `${root}|${data}|${root}`, APC_LITERAL: '${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}', + ...precedenceEnv(server) }, }, })), commandTokenObservation(platform, root, data), @@ -87,7 +91,7 @@ const invalidSseServerCases = [ test('complete stdio and skill core evidence passes its cases and preserves canonical evidence', () => { const value = input(); const report = buildReport(value); - assert.deepEqual(report.summary, { pass: 17, fail: 0, not_verified: 39, total: 56 }); + assert.deepEqual(report.summary, { pass: 18, fail: 0, not_verified: 39, total: 57 }); assert.deepEqual(report.results.map(({ id }) => id), CASE_IDS); assert.equal(report.specVersion, '1.0.0'); assert.deepEqual(result(report, 'mcp.stdio.env.plugin-root').specSections, ['9.1']); @@ -100,19 +104,19 @@ test('complete stdio and skill core evidence passes its cases and preserves cano test('recovery witnesses extend the canonical report without changing core-only results', () => { const coreOnly = buildReport(input()); - assert.deepEqual(coreOnly.summary, { pass: 17, fail: 0, not_verified: 39, total: 56 }); + assert.deepEqual(coreOnly.summary, { pass: 18, fail: 0, not_verified: 39, total: 57 }); assert.equal(result(coreOnly, 'skills.recovery.valid-skill-available').status, 'not_verified'); assert.equal(result(coreOnly, 'mcp.stdio.recovery.valid-server-available').status, 'not_verified'); const recoveryOnly = buildReport({ schemaVersion: 1, observations: recoveryObservations() }); - assert.deepEqual(recoveryOnly.summary, { pass: 2, fail: 0, not_verified: 54, total: 56 }); + assert.deepEqual(recoveryOnly.summary, { pass: 2, fail: 0, not_verified: 55, total: 57 }); assert.equal(result(recoveryOnly, 'skills.recovery.valid-skill-available').status, 'pass'); assert.equal(result(recoveryOnly, 'mcp.stdio.recovery.valid-server-available').status, 'pass'); const combinedInput = input(); combinedInput.observations.push(...recoveryObservations()); const combined = buildReport(combinedInput); - assert.deepEqual(combined.summary, { pass: 20, fail: 0, not_verified: 36, total: 56 }); + assert.deepEqual(combined.summary, { pass: 21, fail: 0, not_verified: 36, total: 57 }); assert.deepEqual(combined.results.map(({ id }) => id), CASE_IDS); const recoveryIds = new Set([ 'skills.recovery.valid-skill-available', @@ -135,7 +139,7 @@ test('recovery witnesses extend the canonical report without changing core-only assert.equal(JSON.stringify(buildReport(reordered)), JSON.stringify(combined)); const missing = buildReport({ schemaVersion: 1, observations: [] }); - assert.deepEqual(missing.summary, { pass: 0, fail: 0, not_verified: 56, total: 56 }); + assert.deepEqual(missing.summary, { pass: 0, fail: 0, not_verified: 57, total: 57 }); }); test('an incorrect recovery skill marker fails its availability check independently', () => { @@ -146,7 +150,7 @@ test('an incorrect recovery skill marker fails its availability check independen assert.equal(availability.status, 'fail'); assert.match(availability.detail, /expected "APC_RECOVERY_VALID_V1"; observed "wrong recovery marker"/); assert.equal(result(report, 'mcp.stdio.recovery.valid-server-available').status, 'pass'); - assert.deepEqual(report.summary, { pass: 1, fail: 1, not_verified: 54, total: 56 }); + assert.deepEqual(report.summary, { pass: 1, fail: 1, not_verified: 55, total: 57 }); }); test('report bytes are deterministic across observation and property orders', () => { @@ -161,11 +165,11 @@ test('missing observations remain unverified and every result identifies its hie const value = input(); value.observations = []; const report = buildReport(value); - assert.deepEqual(report.summary, { pass: 0, fail: 0, not_verified: 56, total: 56 }); + assert.deepEqual(report.summary, { pass: 0, fail: 0, not_verified: 57, total: 57 }); const skills = report.results.filter(({ id }) => id.startsWith('skills.')); const mcp = report.results.filter(({ id }) => id.startsWith('mcp.')); assert.deepEqual(skills.map(({ id }) => id), ['skills.discovery.immediate-children', 'skills.recovery.valid-skill-available', 'skills.recovery.invalid-mcp-document']); - assert.equal(mcp.length, 48); + assert.equal(mcp.length, 49); assert.equal(report.results.filter(({ id }) => id.startsWith('filesystem.')).length, 5); assert.ok(mcp.some(({ id }) => id === 'mcp.stdio.env.plugin-root')); for (const result of report.results) { @@ -185,6 +189,58 @@ test('missing default environment fails independent checks and leaves dependent assert.equal(result(report, 'mcp.stdio.cwd.plugin-data').status, 'pass'); }); +test('configured environment precedence requires complete new evidence and an observable control difference', () => { + const id = 'mcp.stdio.env.configured-precedence'; + const value = input(); + assert.equal(result(buildReport(value), id).status, 'pass'); + + for (const [server, key] of [ + ['default', 'USER'], ['default', 'USERNAME'], ['data', 'USER'], ['data', 'USERNAME'], + ]) { + const partial = structuredClone(value); + delete runtime(partial, server).env[key]; + const outcome = result(buildReport(partial), id); + assert.equal(outcome.status, 'not_verified', `${server}.${key}`); + assert.match(outcome.detail, new RegExp(`${server}\\.${key}`)); + } + + for (const key of ['USER', 'USERNAME']) { + for (const configured of [null, 'ambient-value']) { + const invalid = structuredClone(value); + runtime(invalid).env[key] = configured; + const outcome = result(buildReport(invalid), id); + assert.equal(outcome.status, 'fail', `${key}: ${configured}`); + assert.match(outcome.detail, new RegExp(`${key}: expected`)); + } + } + + const nullControl = structuredClone(value); + runtime(nullControl, 'data').env.USER = null; + runtime(nullControl, 'data').env.USERNAME = null; + assert.equal(result(buildReport(nullControl), id).status, 'not_verified'); + const equalControl = structuredClone(value); + runtime(equalControl, 'data').env.USER = 'apc-configured-environment-precedence'; + runtime(equalControl, 'data').env.USERNAME = 'apc-configured-environment-precedence'; + assert.equal(result(buildReport(equalControl), id).status, 'not_verified'); + + const oneDifferent = structuredClone(nullControl); + runtime(oneDifferent, 'data').env.USERNAME = 'ambient-username'; + assert.equal(result(buildReport(oneDifferent), id).status, 'pass'); + + const old = structuredClone(value); + for (const server of ['default', 'data']) { + for (const key of ['USER', 'USERNAME']) delete runtime(old, server).env[key]; + } + const oldReport = buildReport(old); + assert.equal(result(oldReport, id).status, 'not_verified'); + for (const server of ['default', 'data']) { + for (const key of ['USER', 'USERNAME']) { + assert.equal(Object.hasOwn(runtime(oldReport, server).env, key), false); + } + } + assert.equal(runtime(buildReport(nullControl), 'data').env.USER, null); +}); + test('plugin data writability distinguishes missing evidence, failed writes, successful writes, and cleanup warnings', () => { const value = input(); const id = 'filesystem.data.writable'; @@ -238,7 +294,7 @@ test('data cwd follows resolved aliases while expansion preserves the original e runtime(value).dataWrite.path = '/private/tmp/data/.agent-plugins-conformance-write-test'; runtime(value, 'data').cwd = '/private/tmp/data'; const report = buildReport(value); - assert.equal(report.summary.pass, 17); + assert.equal(report.summary.pass, 18); assert.equal(runtime(report, 'data').resolvedData, '/private/tmp/data'); assert.equal(runtime(report).env.PLUGIN_DATA, '/tmp/data'); runtime(value, 'data').resolvedData = null; @@ -407,7 +463,7 @@ test('command-token observations stay outside cwd and plugin-data aggregate chec test('path comparisons follow producing OS and normalize path components', () => { for (const [root, data] of [['/plugin with spaces', '/data with spaces'], ['C:\\plugin', 'D:\\data'], ['\\\\host\\share\\plugin', '\\\\host\\share\\data']]) { const value = input(root, data); - assert.equal(buildReport(value).summary.pass, 17); + assert.equal(buildReport(value).summary.pass, 18); const flavor = root.startsWith('/') ? path.posix : path.win32; runtime(value).cwd = `${root}${flavor.sep}sub${flavor.sep}..`; assert.equal(result(buildReport(value), 'mcp.stdio.cwd.omitted').status, 'pass'); @@ -446,6 +502,8 @@ test('unknown keys, duplicate observations, identities, versions and types are r [(v) => { runtime(v).resolvedData = 1; }, /resolvedData: expected/], [(v) => { runtime(v).argv = [1]; }, /expected a string/], [(v) => { runtime(v).env.APC_VALUE = null; }, /expected a string/], + [(v) => { runtime(v).env.USER = 1; }, /expected a string/], + [(v) => { runtime(v, 'data').env.USERNAME = {}; }, /expected a string/], [(v) => { v.observations[0].evidence = {}; }, /unknown field/], [(v) => { v.observations[3].kind = 'unknown'; }, /expected one of: mcp-stdio, mcp-streamable-http, mcp-sse, mcp-discovery, skill/], ]; @@ -663,7 +721,7 @@ test('malformed MCP skill evidence affects only its own result for missing, corr assert.deepEqual(result(report, id).specSections, ['7.2.2']); assert.deepEqual(report.results.filter((item) => item.id !== id), baseline.results.filter((item) => item.id !== id)); assert.deepEqual(report.observations.find(({ skill }) => skill === observation.skill), observation); - assert.deepEqual(report.summary, { pass: status === 'pass' ? 21 : 20, fail: status === 'fail' ? 1 : 0, not_verified: 35, total: 56 }); + assert.deepEqual(report.summary, { pass: status === 'pass' ? 22 : 21, fail: status === 'fail' ? 1 : 0, not_verified: 35, total: 57 }); if (status === 'fail') assert.match(result(report, id).detail, /expected "APC_INVALID_MCP_VALID_V1"; observed "incorrect marker"/); const reversed = { schemaVersion: 1, observations: [...value.observations, observation].reverse() }; assert.equal(JSON.stringify(buildReport(reversed)), JSON.stringify(report));