From fce2ba0deaef23f0765c4ebd887085f66793d206 Mon Sep 17 00:00:00 2001 From: Jonathan Hefner Date: Fri, 25 Sep 2026 17:35:58 -0700 Subject: [PATCH] Check configured environment precedence Use the existing default and data probes to compare configured USER and USERNAME values with naturally inherited values. These variables exercise replacement on POSIX and Windows without changing client setup, executable lookup, or home directories. Require a differing control value for a passing result. Record absent variables as null so the reporter can distinguish observed absence from older observations that never collected these fields. Preserve old reports as not_verified and cover reversed merges through real MCP observations. --- .../agent-plugins-conformance-core/README.md | 3 +- .../dist/probe.mjs | 9 +- .../agent-plugins-conformance-core/mcp.json | 4 +- .../src/probe.mjs | 11 ++- .../agent-plugins-conformance/src/cases.mjs | 1 + .../agent-plugins-conformance/src/report.mjs | 43 +++++++++- test/probe.test.mjs | 40 ++++++++- test/record-report.test.mjs | 24 ++++++ test/report-human.test.mjs | 16 ++-- test/report.test.mjs | 82 ++++++++++++++++--- 10 files changed, 204 insertions(+), 29 deletions(-) diff --git a/plugins/agent-plugins-conformance-core/README.md b/plugins/agent-plugins-conformance-core/README.md index cf883c1..ce237c3 100644 --- a/plugins/agent-plugins-conformance-core/README.md +++ b/plugins/agent-plugins-conformance-core/README.md @@ -31,7 +31,8 @@ Replace the path with a downloaded or installed copy of this plugin; the command | `mcp.stdio.env.plugin-data-absolute` | Supplied `PLUGIN_DATA` is an absolute path. This check does not exercise writability, storage dedication, or persistence across updates. | | `filesystem.data.writable` | The default server can create, write, and close a small temporary file directly in the supplied `PLUGIN_DATA` directory. | | `filesystem.data.consistent-within-plugin` | Servers using omitted and data-rooted working directories resolve `PLUGIN_DATA` to the same filesystem path. | -| `mcp.stdio.env.configured-value` | The configured value reaches the subprocess. The fixture does not arrange a conflicting inherited value, so it does not establish override behavior. | +| `mcp.stdio.env.configured-value` | The configured value reaches the subprocess. | +| `mcp.stdio.env.configured-precedence` | Configured environment values replace conflicting base-environment values. | | `mcp.stdio.args.preservation-and-expansion` | Argument boundaries, including spaces and an empty argument, are preserved; recognized placeholders expand and unknown placeholder-like text stays literal. | | `mcp.stdio.command.single-token` | A command path containing a space is preserved as a single executable token. | | `mcp.stdio.env.expansion` | Repeated recognized placeholders expand in environment values and unknown placeholder-like text stays literal. | diff --git a/plugins/agent-plugins-conformance-core/dist/probe.mjs b/plugins/agent-plugins-conformance-core/dist/probe.mjs index aeb08c3..31c3a10 100644 --- a/plugins/agent-plugins-conformance-core/dist/probe.mjs +++ b/plugins/agent-plugins-conformance-core/dist/probe.mjs @@ -16972,6 +16972,7 @@ if (!["default", "relative", "root", "data", "command-token-posix", "command-tok } var root = realpathSync.native(fileURLToPath(new URL("..", import.meta.url))); var environmentNames = ["PLUGIN_ROOT", "PLUGIN_DATA", "APC_VALUE", "APC_EXPANSION", "APC_LITERAL"]; +var environmentPrecedenceNames = ["USER", "USERNAME"]; var resolvedData = null; if (process.env.PLUGIN_DATA && isAbsolute2(process.env.PLUGIN_DATA)) { try { @@ -16979,6 +16980,10 @@ if (process.env.PLUGIN_DATA && isAbsolute2(process.env.PLUGIN_DATA)) { } catch { } } +var environment = Object.fromEntries(environmentNames.filter((name) => process.env[name] !== void 0).map((name) => [name, process.env[name]])); +if (["default", "data"].includes(serverName)) { + for (const name of environmentPrecedenceNames) environment[name] = process.env[name] ?? null; +} var launch = { server: serverName, root, @@ -16986,7 +16991,7 @@ var launch = { // Compare directory identity even when the OS retains a junction/alias spelling. cwd: realpathSync.native(process.cwd()), argv: process.argv.slice(2), - env: Object.fromEntries(environmentNames.filter((name) => process.env[name] !== void 0).map((name) => [name, process.env[name]])) + env: environment }; var server = new Server( { name: `agent-plugins-conformance-${serverName}`, version: "0.1.0" }, @@ -16994,7 +16999,7 @@ var server = new Server( ); var observeTool = { name: "observe", - description: "Return this process launch evidence, including only fixture environment variables." + (serverName === "default" ? " Each call also creates, writes, closes, and removes a uniquely named temporary file directly in an absolute PLUGIN_DATA directory, recording any operation or cleanup error." : "") + " Record the observation object from structuredContent (or parsed JSON text) unchanged with the run-conformance reporter; exclude the MCP result wrapper.", + description: "Return this process launch evidence, including selected environment variables." + (serverName === "default" ? " Each call also creates, writes, closes, and removes a uniquely named temporary file directly in an absolute PLUGIN_DATA directory, recording any operation or cleanup error." : "") + " Record the observation object from structuredContent (or parsed JSON text) unchanged with the run-conformance reporter; exclude the MCP result wrapper.", inputSchema: { type: "object", properties: {}, additionalProperties: false }, annotations: { readOnlyHint: serverName !== "default", destructiveHint: false, idempotentHint: serverName !== "default", openWorldHint: false } }; diff --git a/plugins/agent-plugins-conformance-core/mcp.json b/plugins/agent-plugins-conformance-core/mcp.json index 9efec50..4427fbf 100644 --- a/plugins/agent-plugins-conformance-core/mcp.json +++ b/plugins/agent-plugins-conformance-core/mcp.json @@ -18,7 +18,9 @@ "env": { "APC_VALUE": "fixture value with spaces", "APC_EXPANSION": "${PLUGIN_ROOT}|${PLUGIN_DATA}|${PLUGIN_ROOT}", - "APC_LITERAL": "${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}" + "APC_LITERAL": "${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}", + "USER": "apc-configured-environment-precedence", + "USERNAME": "apc-configured-environment-precedence" } }, "relative": { diff --git a/plugins/agent-plugins-conformance-core/src/probe.mjs b/plugins/agent-plugins-conformance-core/src/probe.mjs index eea9d8a..860907a 100644 --- a/plugins/agent-plugins-conformance-core/src/probe.mjs +++ b/plugins/agent-plugins-conformance-core/src/probe.mjs @@ -14,10 +14,16 @@ if (!['default', 'relative', 'root', 'data', 'command-token-posix', 'command-tok // This value is independent of the client-provided PLUGIN_ROOT environment. const root = realpathSync.native(fileURLToPath(new URL('..', import.meta.url))); const environmentNames = ['PLUGIN_ROOT', 'PLUGIN_DATA', 'APC_VALUE', 'APC_EXPANSION', 'APC_LITERAL']; +const environmentPrecedenceNames = ['USER', 'USERNAME']; let resolvedData = null; if (process.env.PLUGIN_DATA && isAbsolute(process.env.PLUGIN_DATA)) { try { resolvedData = realpathSync.native(process.env.PLUGIN_DATA); } catch { /* Unobservable target remains null. */ } } +const environment = Object.fromEntries(environmentNames.filter((name) => process.env[name] !== undefined) + .map((name) => [name, process.env[name]])); +if (['default', 'data'].includes(serverName)) { + for (const name of environmentPrecedenceNames) environment[name] = process.env[name] ?? null; +} const launch = { server: serverName, root, @@ -25,14 +31,13 @@ const launch = { // Compare directory identity even when the OS retains a junction/alias spelling. cwd: realpathSync.native(process.cwd()), argv: process.argv.slice(2), - env: Object.fromEntries(environmentNames.filter((name) => process.env[name] !== undefined) - .map((name) => [name, process.env[name]])), + env: environment, }; const server = new Server({ name: `agent-plugins-conformance-${serverName}`, version: '0.1.0' }, { capabilities: { tools: {} } }); const observeTool = { name: 'observe', - description: 'Return this process launch evidence, including only fixture environment variables.' + + description: 'Return this process launch evidence, including selected environment variables.' + (serverName === 'default' ? ' Each call also creates, writes, closes, and removes a uniquely named temporary file directly in an absolute PLUGIN_DATA directory, recording any operation or cleanup error.' : '') + ' Record the observation object from structuredContent (or parsed JSON text) unchanged with the run-conformance reporter; exclude the MCP result wrapper.', inputSchema: { type: 'object', properties: {}, additionalProperties: false }, diff --git a/plugins/agent-plugins-conformance/src/cases.mjs b/plugins/agent-plugins-conformance/src/cases.mjs index df64eaa..d073d3b 100644 --- a/plugins/agent-plugins-conformance/src/cases.mjs +++ b/plugins/agent-plugins-conformance/src/cases.mjs @@ -27,6 +27,7 @@ export const CASES = Object.freeze([ ['filesystem.data.distinct-across-plugins', 'Distinct data directories across plugins', ['9.1']], ['filesystem.data.consistent-within-plugin', 'Consistent data directory within a plugin', ['9.1']], ['mcp.stdio.env.configured-value', 'Configured environment', ['9.1']], + ['mcp.stdio.env.configured-precedence', 'Configured environment precedence', ['9.1']], ['mcp.stdio.command.single-token', 'Command token preservation', ['7.2.1']], ['mcp.stdio.args.preservation-and-expansion', 'Argument preservation and expansion', ['7.2.1', '9.2']], ['mcp.stdio.env.expansion', 'Environment expansion', ['9.2']], diff --git a/plugins/agent-plugins-conformance/src/report.mjs b/plugins/agent-plugins-conformance/src/report.mjs index 843d7a9..58f96c5 100644 --- a/plugins/agent-plugins-conformance/src/report.mjs +++ b/plugins/agent-plugins-conformance/src/report.mjs @@ -55,7 +55,12 @@ const SKILLS = { 'conformance-recovery-valid': 'APC_RECOVERY_VALID_V1', 'conformance-invalid-mcp-valid': 'APC_INVALID_MCP_VALID_V1', }; -const ENV_KEYS = ['PLUGIN_ROOT', 'PLUGIN_DATA', 'APC_VALUE', 'APC_EXPANSION', 'APC_LITERAL']; +const ENVIRONMENT_PRECEDENCE_KEYS = ['USER', 'USERNAME']; +const ENVIRONMENT_PRECEDENCE_VALUE = 'apc-configured-environment-precedence'; +const ENV_KEYS = [ + 'PLUGIN_ROOT', 'PLUGIN_DATA', 'APC_VALUE', 'APC_EXPANSION', 'APC_LITERAL', + ...ENVIRONMENT_PRECEDENCE_KEYS, +]; function invalid(at, reason) { throw new TypeError(`${at}: ${reason}`); @@ -295,7 +300,10 @@ export function validateInput(input, { recording = false } = {}) { array(evidence.argv, `${evidenceAt}.argv`); evidence.argv.forEach((argument, i) => string(argument, `${evidenceAt}.argv[${i}]`, true)); object(evidence.env, ENV_KEYS, [], `${evidenceAt}.env`); - for (const [key, value] of Object.entries(evidence.env)) string(value, `${evidenceAt}.env.${key}`, true); + for (const [key, value] of Object.entries(evidence.env)) { + if (ENVIRONMENT_PRECEDENCE_KEYS.includes(key) && value === null) continue; + string(value, `${evidenceAt}.env.${key}`, true); + } if (observation.server === 'default' && evidence.dataWrite !== null) { const at = `${evidenceAt}.dataWrite`; const write = evidence.dataWrite; @@ -716,6 +724,37 @@ export function buildReport(input) { `default resolved PLUGIN_DATA to ${JSON.stringify(defaultData)}. data resolved PLUGIN_DATA to ${JSON.stringify(dataCwdData)}.`); } const evidence = runtime.get('default'); + const precedenceCase = 'mcp.stdio.env.configured-precedence'; + const environmentPrecedenceEvidence = Object.fromEntries(['default', 'data'] + .map((server) => [server, runtime.get(server)])); + const missingPrecedenceServers = Object.entries(environmentPrecedenceEvidence) + .filter(([, value]) => value === undefined).map(([server]) => server); + const missingPrecedenceKeys = Object.entries(environmentPrecedenceEvidence).flatMap(([server, value]) => value === undefined + ? [] + : ENVIRONMENT_PRECEDENCE_KEYS.filter((key) => !Object.hasOwn(value.env, key)) + .map((key) => `${server}.${key}`)); + if (missingPrecedenceServers.length > 0) { + set(precedenceCase, 'not_verified', + `Missing MCP observation for: ${missingPrecedenceServers.join(', ')}.`); + } else if (missingPrecedenceKeys.length > 0) { + set(precedenceCase, 'not_verified', + `Environment precedence evidence is missing: ${missingPrecedenceKeys.join(', ')}.`); + } else { + const configuredDifferences = ENVIRONMENT_PRECEDENCE_KEYS + .filter((key) => environmentPrecedenceEvidence.default.env[key] !== ENVIRONMENT_PRECEDENCE_VALUE) + .map((key) => mismatch(key, ENVIRONMENT_PRECEDENCE_VALUE, environmentPrecedenceEvidence.default.env[key])); + if (configuredDifferences.length > 0) { + set(precedenceCase, 'fail', + `Configured environment values were not delivered. ${configuredDifferences.join(' ')}`); + } else { + const differingControls = ENVIRONMENT_PRECEDENCE_KEYS.filter((key) => + typeof environmentPrecedenceEvidence.data.env[key] === 'string' && + environmentPrecedenceEvidence.data.env[key] !== ENVIRONMENT_PRECEDENCE_VALUE); + set(precedenceCase, differingControls.length > 0 ? 'pass' : 'not_verified', differingControls.length > 0 + ? `Configured values replaced different inherited control values for: ${differingControls.join(', ')}.` + : 'No control value differed from the configured value, so replacement was not observable.'); + } + } if (evidence) { const { root, env, argv } = evidence; const flavor = pathFlavor(root); diff --git a/test/probe.test.mjs b/test/probe.test.mjs index 6d773f9..104c210 100644 --- a/test/probe.test.mjs +++ b/test/probe.test.mjs @@ -37,7 +37,9 @@ async function connect(fixture, mode, override = {}) { const client = new Client({ name: 'conformance-reference-test', version: '1' }); const transport = new StdioClientTransport({ command: process.execPath, args: config.args.map(expand), cwd, - env: { ...variables, ...Object.fromEntries(Object.entries(config.env ?? {}).map(([key, value]) => [key, expand(value)])), APC_SHOULD_NOT_LEAK: 'unrelated ambient sentinel' }, + env: { USER: 'ambient-user', USERNAME: 'ambient-username', ...variables, + ...Object.fromEntries(Object.entries(config.env ?? {}).map(([key, value]) => [key, expand(value)])), + APC_SHOULD_NOT_LEAK: 'unrelated ambient sentinel' }, stderr: 'pipe', ...override, }); fixture.clients.push(client); @@ -61,6 +63,16 @@ test('copied plugin runs only MCP observation tools without node_modules', { tim assert.deepEqual(result.structuredContent, observation); assert.equal(observation.evidence.root, files.root); assert.equal(observation.evidence.env.APC_SHOULD_NOT_LEAK, undefined); + if (mode === 'default') { + assert.equal(observation.evidence.env.USER, 'apc-configured-environment-precedence'); + assert.equal(observation.evidence.env.USERNAME, 'apc-configured-environment-precedence'); + } else if (mode === 'data') { + assert.equal(observation.evidence.env.USER, 'ambient-user'); + assert.equal(observation.evidence.env.USERNAME, 'ambient-username'); + } else { + assert.equal(Object.hasOwn(observation.evidence.env, 'USER'), false); + assert.equal(Object.hasOwn(observation.evidence.env, 'USERNAME'), false); + } if (mode === 'default') { assert.equal(dirname(observation.evidence.dataWrite.path), files.data); assert.equal(observation.evidence.dataWrite.error, null); @@ -93,10 +105,34 @@ test('copied plugin runs only MCP observation tools without node_modules', { tim const reportInput = input(observations); const direct = buildReport(reportInput); assert.equal(direct.summary.fail, 0); - assert.equal(direct.summary.pass, 15); + assert.equal(direct.summary.pass, 16); assert.equal(direct.summary.not_verified, 41); }); +test('faulty ambient precedence becomes a normal reporter failure for both platform variables', { timeout: 30_000 }, async (t) => { + const files = await fixture(t); + const variables = { PLUGIN_ROOT: files.root, PLUGIN_DATA: files.data }; + const expand = (value) => value.replace(/\$\{(PLUGIN_ROOT|PLUGIN_DATA)\}/g, (_, name) => variables[name]); + const configured = Object.fromEntries(Object.entries(files.config.mcpServers.default.env) + .map(([name, value]) => [name, expand(value)])); + for (const name of ['USER', 'USERNAME']) { + const control = await connect(files, 'data', { env: { + ...variables, USER: 'ambient-user', USERNAME: 'ambient-username', + } }); + const faulty = await connect(files, 'default', { env: { + ...variables, ...configured, [name]: `ambient-${name.toLowerCase()}`, + } }); + const [controlResult, faultyResult] = await Promise.all([ + control.callTool({ name: 'observe', arguments: {} }), + faulty.callTool({ name: 'observe', arguments: {} }), + ]); + const report = buildReport(input([faultyResult.structuredContent, controlResult.structuredContent])); + const outcome = report.results.find(({ id }) => id === 'mcp.stdio.env.configured-precedence'); + assert.equal(outcome.status, 'fail', name); + assert.match(outcome.detail, new RegExp(`${name}: expected`)); + } +}); + test('copied recovery plugin serves exact valid and invalid-server observations without runtime dependencies', { timeout: 30_000 }, async (t) => { const parent = await mkdtemp(join(tmpdir(), 'agent-plugins-conformance-recovery-')); const root = join(parent, 'copied recovery plugin'); diff --git a/test/record-report.test.mjs b/test/record-report.test.mjs index c3009a8..70ce358 100644 --- a/test/record-report.test.mjs +++ b/test/record-report.test.mjs @@ -20,9 +20,16 @@ const mcp = () => ({ env: { PLUGIN_ROOT: '/plugin', PLUGIN_DATA: '/data', APC_VALUE: 'fixture value with spaces', APC_EXPANSION: '/plugin|/data|/plugin', APC_LITERAL: '${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}', + USER: 'apc-configured-environment-precedence', USERNAME: 'apc-configured-environment-precedence', }, }, }); +const dataMcp = (env = { USER: null, USERNAME: 'ambient-username' }) => ({ + kind: 'mcp-stdio', server: 'data', evidence: { + version: 1, server: 'data', root: '/plugin', cwd: '/data', resolvedData: '/data', + argv: ['data'], env, + }, +}); const commandToken = (platform = 'posix', { root = platform === 'posix' ? '/plugin' : 'C:\\plugin', data = platform === 'posix' ? '/data' : 'D:\\data', @@ -393,6 +400,23 @@ test('distinct keys accumulate in deterministic order; repeated keys fully repla assert.deepEqual(await readdir(join(f.directory, 'nested directory')), ['report with spaces.json']); }); +test('recorder preserves nullable environment precedence evidence through canonical reloads', async (t) => { + const f = await fixture(t); + f.success(start); + f.record(dataMcp()); + f.record(mcp()); + let report = await f.read(); + assert.deepEqual(report.observations.filter(({ server }) => ['default', 'data'].includes(server)), [mcp(), dataMcp()]); + assert.equal(report.results.find(({ id }) => id === 'mcp.stdio.env.configured-precedence').status, 'pass'); + assert.equal(report.observations.find(({ server }) => server === 'data').evidence.env.USER, null); + + const oldData = dataMcp({}); + f.record(oldData); + report = await f.read(); + assert.deepEqual(report.observations.find(({ server }) => server === 'data'), oldData); + assert.equal(report.results.find(({ id }) => id === 'mcp.stdio.env.configured-precedence').status, 'not_verified'); +}); + test('command-token evidence records canonically and replacement changes only its verdict', async (t) => { const f = await fixture(t); f.success(start); diff --git a/test/report-human.test.mjs b/test/report-human.test.mjs index d984d0b..ef98ab5 100644 --- a/test/report-human.test.mjs +++ b/test/report-human.test.mjs @@ -6,6 +6,9 @@ import { formatReport } from '../plugins/agent-plugins-conformance/src/report-fo function input() { const root = '/fixture/plugin'; const data = '/state/plugin'; + const precedenceEnv = (server) => server === 'default' + ? { USER: 'apc-configured-environment-precedence', USERNAME: 'apc-configured-environment-precedence' } + : server === 'data' ? { USER: 'ambient-user', USERNAME: 'ambient-username' } : {}; return { schemaVersion: 1, observations: [ @@ -18,7 +21,8 @@ function input() { cwd: server === 'default' ? root : server === 'data' ? data : `${root}/probe-workdir`, argv: ['default', 'arg with spaces', '', root, data, '${APC_UNKNOWN}', '$APC_VALUE', '${PLUGIN_ROOT_SUFFIX}'], env: { PLUGIN_ROOT: root, PLUGIN_DATA: data, APC_VALUE: 'fixture value with spaces', - APC_EXPANSION: `${root}|${data}|${root}`, APC_LITERAL: '${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}' }, + APC_EXPANSION: `${root}|${data}|${root}`, APC_LITERAL: '${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}', + ...precedenceEnv(server) }, }, })), { @@ -59,9 +63,9 @@ test('complete stdio and recovery evidence leaves optional HTTP and SSE checks u '', 'Checks Passed Failed Not verified', 'Skills 3 0 0', - 'MCP 22 0 26', + 'MCP 23 0 26', 'Filesystem 4 0 1', - 'Total 29 0 27', + 'Total 30 0 27', ].join('\n')); const missing = human.split('\n\nNot verified\n')[1]; for (const id of [ @@ -100,7 +104,7 @@ test('all-unverified human report preserves every saved missing-evidence result' value.observations = []; const human = formatReport(buildReport(value)); assert.match(human, /Skills\s+0\s+0\s+3/); - assert.match(human, /MCP\s+0\s+0\s+48/); + assert.match(human, /MCP\s+0\s+0\s+49/); assert.match(human, /Filesystem\s+0\s+0\s+5/); assert.match(human, /Missing skill observations: conformance-alpha, conformance-beta\./); for (const result of buildReport(value).results) assert.ok(human.includes(`(${result.id})`)); @@ -112,7 +116,7 @@ test('mixed human report puts failures before missing-evidence details', () => { value.observations = [value.observations[2]]; value.observations[0].evidence.env.APC_VALUE = 'incorrect configured value'; const human = formatReport(buildReport(value)); - assert.match(human, /MCP\s+6\s+1\s+41/); + assert.match(human, /MCP\s+6\s+1\s+42/); assert.match(human, /Filesystem\s+1\s+0\s+4/); assert.match(human, /Configured environment \(mcp.stdio.env.configured-value\)/); assert.match(human, /"fixture value with spaces"/); @@ -169,7 +173,7 @@ test('human report renders saved warnings independently of result status', () => const writable = value.results.find(({ id }) => id === 'filesystem.data.writable'); writable.status = 'not_verified'; writable.warning = 'Saved cleanup warning for /state/plugin/leftover.'; - value.summary = { pass: 16, fail: 0, not_verified: 40, total: 56 }; + value.summary = { pass: 17, fail: 0, not_verified: 40, total: 57 }; value.observations = []; const human = formatReport(value); diff --git a/test/report.test.mjs b/test/report.test.mjs index ba89b6d..bcb41fd 100644 --- a/test/report.test.mjs +++ b/test/report.test.mjs @@ -6,6 +6,9 @@ import { buildReport, CASE_IDS } from '../plugins/agent-plugins-conformance/src/ function input(root = '/fixture/plugin', data = '/state/plugin') { const flavor = root.startsWith('/') ? path.posix : path.win32; const platform = flavor === path.win32 ? 'windows' : 'posix'; + const precedenceEnv = (server) => server === 'default' + ? { USER: 'apc-configured-environment-precedence', USERNAME: 'apc-configured-environment-precedence' } + : server === 'data' ? { USER: 'ambient-user', USERNAME: 'ambient-username' } : {}; return { schemaVersion: 1, observations: [ @@ -19,7 +22,8 @@ function input(root = '/fixture/plugin', data = '/state/plugin') { cwd: server === 'default' ? root : server === 'data' ? data : flavor.join(root, 'probe-workdir'), argv: ['default', 'arg with spaces', '', root, data, '${APC_UNKNOWN}', '$APC_VALUE', '${PLUGIN_ROOT_SUFFIX}'], env: { PLUGIN_ROOT: root, PLUGIN_DATA: data, APC_VALUE: 'fixture value with spaces', - APC_EXPANSION: `${root}|${data}|${root}`, APC_LITERAL: '${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}' }, + APC_EXPANSION: `${root}|${data}|${root}`, APC_LITERAL: '${APC_UNKNOWN}|$APC_VALUE|${PLUGIN_ROOT_SUFFIX}', + ...precedenceEnv(server) }, }, })), commandTokenObservation(platform, root, data), @@ -87,7 +91,7 @@ const invalidSseServerCases = [ test('complete stdio and skill core evidence passes its cases and preserves canonical evidence', () => { const value = input(); const report = buildReport(value); - assert.deepEqual(report.summary, { pass: 17, fail: 0, not_verified: 39, total: 56 }); + assert.deepEqual(report.summary, { pass: 18, fail: 0, not_verified: 39, total: 57 }); assert.deepEqual(report.results.map(({ id }) => id), CASE_IDS); assert.equal(report.specVersion, '1.0.0'); assert.deepEqual(result(report, 'mcp.stdio.env.plugin-root').specSections, ['9.1']); @@ -100,19 +104,19 @@ test('complete stdio and skill core evidence passes its cases and preserves cano test('recovery witnesses extend the canonical report without changing core-only results', () => { const coreOnly = buildReport(input()); - assert.deepEqual(coreOnly.summary, { pass: 17, fail: 0, not_verified: 39, total: 56 }); + assert.deepEqual(coreOnly.summary, { pass: 18, fail: 0, not_verified: 39, total: 57 }); assert.equal(result(coreOnly, 'skills.recovery.valid-skill-available').status, 'not_verified'); assert.equal(result(coreOnly, 'mcp.stdio.recovery.valid-server-available').status, 'not_verified'); const recoveryOnly = buildReport({ schemaVersion: 1, observations: recoveryObservations() }); - assert.deepEqual(recoveryOnly.summary, { pass: 2, fail: 0, not_verified: 54, total: 56 }); + assert.deepEqual(recoveryOnly.summary, { pass: 2, fail: 0, not_verified: 55, total: 57 }); assert.equal(result(recoveryOnly, 'skills.recovery.valid-skill-available').status, 'pass'); assert.equal(result(recoveryOnly, 'mcp.stdio.recovery.valid-server-available').status, 'pass'); const combinedInput = input(); combinedInput.observations.push(...recoveryObservations()); const combined = buildReport(combinedInput); - assert.deepEqual(combined.summary, { pass: 20, fail: 0, not_verified: 36, total: 56 }); + assert.deepEqual(combined.summary, { pass: 21, fail: 0, not_verified: 36, total: 57 }); assert.deepEqual(combined.results.map(({ id }) => id), CASE_IDS); const recoveryIds = new Set([ 'skills.recovery.valid-skill-available', @@ -135,7 +139,7 @@ test('recovery witnesses extend the canonical report without changing core-only assert.equal(JSON.stringify(buildReport(reordered)), JSON.stringify(combined)); const missing = buildReport({ schemaVersion: 1, observations: [] }); - assert.deepEqual(missing.summary, { pass: 0, fail: 0, not_verified: 56, total: 56 }); + assert.deepEqual(missing.summary, { pass: 0, fail: 0, not_verified: 57, total: 57 }); }); test('an incorrect recovery skill marker fails its availability check independently', () => { @@ -146,7 +150,7 @@ test('an incorrect recovery skill marker fails its availability check independen assert.equal(availability.status, 'fail'); assert.match(availability.detail, /expected "APC_RECOVERY_VALID_V1"; observed "wrong recovery marker"/); assert.equal(result(report, 'mcp.stdio.recovery.valid-server-available').status, 'pass'); - assert.deepEqual(report.summary, { pass: 1, fail: 1, not_verified: 54, total: 56 }); + assert.deepEqual(report.summary, { pass: 1, fail: 1, not_verified: 55, total: 57 }); }); test('report bytes are deterministic across observation and property orders', () => { @@ -161,11 +165,11 @@ test('missing observations remain unverified and every result identifies its hie const value = input(); value.observations = []; const report = buildReport(value); - assert.deepEqual(report.summary, { pass: 0, fail: 0, not_verified: 56, total: 56 }); + assert.deepEqual(report.summary, { pass: 0, fail: 0, not_verified: 57, total: 57 }); const skills = report.results.filter(({ id }) => id.startsWith('skills.')); const mcp = report.results.filter(({ id }) => id.startsWith('mcp.')); assert.deepEqual(skills.map(({ id }) => id), ['skills.discovery.immediate-children', 'skills.recovery.valid-skill-available', 'skills.recovery.invalid-mcp-document']); - assert.equal(mcp.length, 48); + assert.equal(mcp.length, 49); assert.equal(report.results.filter(({ id }) => id.startsWith('filesystem.')).length, 5); assert.ok(mcp.some(({ id }) => id === 'mcp.stdio.env.plugin-root')); for (const result of report.results) { @@ -185,6 +189,58 @@ test('missing default environment fails independent checks and leaves dependent assert.equal(result(report, 'mcp.stdio.cwd.plugin-data').status, 'pass'); }); +test('configured environment precedence requires complete new evidence and an observable control difference', () => { + const id = 'mcp.stdio.env.configured-precedence'; + const value = input(); + assert.equal(result(buildReport(value), id).status, 'pass'); + + for (const [server, key] of [ + ['default', 'USER'], ['default', 'USERNAME'], ['data', 'USER'], ['data', 'USERNAME'], + ]) { + const partial = structuredClone(value); + delete runtime(partial, server).env[key]; + const outcome = result(buildReport(partial), id); + assert.equal(outcome.status, 'not_verified', `${server}.${key}`); + assert.match(outcome.detail, new RegExp(`${server}\\.${key}`)); + } + + for (const key of ['USER', 'USERNAME']) { + for (const configured of [null, 'ambient-value']) { + const invalid = structuredClone(value); + runtime(invalid).env[key] = configured; + const outcome = result(buildReport(invalid), id); + assert.equal(outcome.status, 'fail', `${key}: ${configured}`); + assert.match(outcome.detail, new RegExp(`${key}: expected`)); + } + } + + const nullControl = structuredClone(value); + runtime(nullControl, 'data').env.USER = null; + runtime(nullControl, 'data').env.USERNAME = null; + assert.equal(result(buildReport(nullControl), id).status, 'not_verified'); + const equalControl = structuredClone(value); + runtime(equalControl, 'data').env.USER = 'apc-configured-environment-precedence'; + runtime(equalControl, 'data').env.USERNAME = 'apc-configured-environment-precedence'; + assert.equal(result(buildReport(equalControl), id).status, 'not_verified'); + + const oneDifferent = structuredClone(nullControl); + runtime(oneDifferent, 'data').env.USERNAME = 'ambient-username'; + assert.equal(result(buildReport(oneDifferent), id).status, 'pass'); + + const old = structuredClone(value); + for (const server of ['default', 'data']) { + for (const key of ['USER', 'USERNAME']) delete runtime(old, server).env[key]; + } + const oldReport = buildReport(old); + assert.equal(result(oldReport, id).status, 'not_verified'); + for (const server of ['default', 'data']) { + for (const key of ['USER', 'USERNAME']) { + assert.equal(Object.hasOwn(runtime(oldReport, server).env, key), false); + } + } + assert.equal(runtime(buildReport(nullControl), 'data').env.USER, null); +}); + test('plugin data writability distinguishes missing evidence, failed writes, successful writes, and cleanup warnings', () => { const value = input(); const id = 'filesystem.data.writable'; @@ -238,7 +294,7 @@ test('data cwd follows resolved aliases while expansion preserves the original e runtime(value).dataWrite.path = '/private/tmp/data/.agent-plugins-conformance-write-test'; runtime(value, 'data').cwd = '/private/tmp/data'; const report = buildReport(value); - assert.equal(report.summary.pass, 17); + assert.equal(report.summary.pass, 18); assert.equal(runtime(report, 'data').resolvedData, '/private/tmp/data'); assert.equal(runtime(report).env.PLUGIN_DATA, '/tmp/data'); runtime(value, 'data').resolvedData = null; @@ -407,7 +463,7 @@ test('command-token observations stay outside cwd and plugin-data aggregate chec test('path comparisons follow producing OS and normalize path components', () => { for (const [root, data] of [['/plugin with spaces', '/data with spaces'], ['C:\\plugin', 'D:\\data'], ['\\\\host\\share\\plugin', '\\\\host\\share\\data']]) { const value = input(root, data); - assert.equal(buildReport(value).summary.pass, 17); + assert.equal(buildReport(value).summary.pass, 18); const flavor = root.startsWith('/') ? path.posix : path.win32; runtime(value).cwd = `${root}${flavor.sep}sub${flavor.sep}..`; assert.equal(result(buildReport(value), 'mcp.stdio.cwd.omitted').status, 'pass'); @@ -446,6 +502,8 @@ test('unknown keys, duplicate observations, identities, versions and types are r [(v) => { runtime(v).resolvedData = 1; }, /resolvedData: expected/], [(v) => { runtime(v).argv = [1]; }, /expected a string/], [(v) => { runtime(v).env.APC_VALUE = null; }, /expected a string/], + [(v) => { runtime(v).env.USER = 1; }, /expected a string/], + [(v) => { runtime(v, 'data').env.USERNAME = {}; }, /expected a string/], [(v) => { v.observations[0].evidence = {}; }, /unknown field/], [(v) => { v.observations[3].kind = 'unknown'; }, /expected one of: mcp-stdio, mcp-streamable-http, mcp-sse, mcp-discovery, skill/], ]; @@ -663,7 +721,7 @@ test('malformed MCP skill evidence affects only its own result for missing, corr assert.deepEqual(result(report, id).specSections, ['7.2.2']); assert.deepEqual(report.results.filter((item) => item.id !== id), baseline.results.filter((item) => item.id !== id)); assert.deepEqual(report.observations.find(({ skill }) => skill === observation.skill), observation); - assert.deepEqual(report.summary, { pass: status === 'pass' ? 21 : 20, fail: status === 'fail' ? 1 : 0, not_verified: 35, total: 56 }); + assert.deepEqual(report.summary, { pass: status === 'pass' ? 22 : 21, fail: status === 'fail' ? 1 : 0, not_verified: 35, total: 57 }); if (status === 'fail') assert.match(result(report, id).detail, /expected "APC_INVALID_MCP_VALID_V1"; observed "incorrect marker"/); const reversed = { schemaVersion: 1, observations: [...value.observations, observation].reverse() }; assert.equal(JSON.stringify(buildReport(reversed)), JSON.stringify(report));