diff --git a/src/index.ts b/src/index.ts index 4fdb5bcb..c42be2e3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -555,11 +555,13 @@ program '--requirements ', 'Run exactly the scenarios a spec revision requires, frozen at its release (e.g. 2026-07-28). Replaces --suite and --spec-version' ) + .option('--timeout ', 'Per-scenario timeout in milliseconds', '30000') .option('--verbose', 'Show verbose output (JSON instead of pretty print)') .action(async (options, cmd) => { try { // Validate options with Zod const validated = ServerOptionsSchema.parse(options); + const timeout = parseInt(options.timeout, 10); const verbose = options.verbose ?? false; const outputDir = options.outputDir; @@ -588,7 +590,8 @@ program validated.scenario, outputDir, specVersionFilter, - options.force ?? false + options.force ?? false, + timeout ); // Inapplicable scenario/spec-version combination (already logged by @@ -675,7 +678,8 @@ program specVersionFilter, // a requirement set decides membership, so its choice outranks a // scenario's own applicability window at the pinned revision - Boolean(requirements) + Boolean(requirements), + timeout ) ); allResults.push({ scenario: scenarioName, checks: result.checks }); diff --git a/src/runner/server.test.ts b/src/runner/server.test.ts index f2a2faa7..2f6ac645 100644 --- a/src/runner/server.test.ts +++ b/src/runner/server.test.ts @@ -53,6 +53,52 @@ describe('runServerConformanceTest spec-version applicability', () => { }, 60000); }); +describe('runServerConformanceTest per-scenario timeout', () => { + // A server that completes the TCP handshake and then never writes a byte. + // Before the runner bounded `scenario.run`, this hung the whole suite: the + // scenario had no timeout of its own, so nothing downstream ever ran. + let server: http.Server; + let url: string; + + beforeEach(async () => { + server = http.createServer(() => { + // Deliberately never respond, and never destroy the socket. + }); + await new Promise((resolve) => + server.listen(0, '127.0.0.1', resolve) + ); + const port = (server.address() as AddressInfo).port; + url = `http://127.0.0.1:${port}/mcp`; + }); + + afterEach(async () => { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + }); + + test('fails the scenario instead of hanging when the server never responds', async () => { + const start = Date.now(); + const result = await runServerConformanceTest( + url, + 'server-initialize', + undefined, + undefined, + false, + 1000 + ); + const elapsed = Date.now() - start; + + const timeoutCheck = result.checks.find((c) => c.id === 'scenario-timeout'); + expect(timeoutCheck).toBeDefined(); + expect(timeoutCheck?.status).toBe('FAILURE'); + expect(timeoutCheck?.errorMessage).toContain('1000ms'); + + // The bound is what makes this a failure rather than a stall; without it + // the call never returns and this assertion is never reached. + expect(elapsed).toBeLessThan(15000); + }, 30000); +}); + describe('runServerConformanceTest wire selection for draft-only scenarios', () => { // Regression: the CLI used to silently emit the legacy initialize+session // wire when running a draft-only scenario, producing requests with no diff --git a/src/runner/server.ts b/src/runner/server.ts index a2c624a0..21e80ec4 100644 --- a/src/runner/server.ts +++ b/src/runner/server.ts @@ -27,12 +27,59 @@ function formatMarkdown(text: string): string { ); } +/** + * Bound `scenario.run` so a server that accepts connections but never answers + * fails one scenario instead of stalling the whole suite. + * + * The losing promise is left pending on purpose: a scenario blocked on a socket + * read has no cancellation channel, so there is nothing to await. Its rejection + * is swallowed to keep a late failure from surfacing as an unhandled rejection + * against whichever scenario happens to be running by then. + */ +async function runScenarioBounded( + run: Promise, + scenarioName: string, + timeout: number +): Promise { + const timedOut = Symbol('timed-out'); + let timeoutHandle: NodeJS.Timeout | undefined; + + run.catch(() => {}); + + const result = await Promise.race([ + run, + new Promise((resolve) => { + timeoutHandle = setTimeout(() => resolve(timedOut), timeout); + }) + ]); + clearTimeout(timeoutHandle); + + if (result !== timedOut) { + return result; + } + + console.log(`\nScenario timed out after ${timeout}ms`); + return [ + { + id: 'scenario-timeout', + name: 'Scenario completes within the timeout', + description: + 'The scenario must finish within the configured timeout. A server that ' + + 'accepts the connection but never responds leaves it running forever.', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Scenario '${scenarioName}' did not complete within ${timeout}ms. The server under test accepted the connection but did not finish the exchange.` + } + ]; +} + export async function runServerConformanceTest( serverUrl: string, scenarioName: string, outputDir?: string, specVersion?: SpecVersion, - force = false + force = false, + timeout: number = 30000 ): Promise<{ checks: ConformanceCheck[]; resultDir?: string; @@ -98,7 +145,11 @@ export async function runServerConformanceTest( connect: (opts) => connectFor(resolvedSpecVersion)(serverUrl, opts) }; resetWireValidation(); - const checks = await scenario.run(ctx); + const checks = await runScenarioBounded( + scenario.run(ctx), + scenarioName, + timeout + ); checks.push(...wireSchemaChecks(resolvedSpecVersion)); if (resultDir) {