perf(sdk): set up jobs, workflows, channels, remote MCP, the ESM loader and OAuth flows on first use

apps/e2e/demo-e2e-bundle-size/e2e/helpers/initialized-modules.cjs

@@ -0,0 +1,65 @@
+// Loads the SDK's CommonJS build with esbuild's module-init helper wrapped, so the source modules it initializes are
+// recorded, then prints, as JSON, the ones initialized by loading it and by each step after:
+// node initialized-modules.cjs <dist/index.js> [read:<export>,...] [fetch:<METHOD> <path>]...
+const Module = require('node:module');
+const { readFileSync, realpathSync } = require('node:fs');
+const path = require('node:path');
+
+const ESM_INIT_HELPER =
+  'var __esm=(fn,res,err)=>function __init(){if(err)throw err[0];try{return fn&&(res=(0,fn[__getOwnPropNames(fn)[0]])(fn=0)),res}catch(e){throw err=[e],e}};';
+const RECORD_INITS =
+  'var __esmUnrecorded=__esm;__esm=(fn,res,err)=>{const source=__getOwnPropNames(fn)[0];const init=__esmUnrecorded(fn,res,err);return function(){globalThis.__sdkInitialized.add(source);return init()}};';
+
+const [entryArg, ...steps] = process.argv.slice(2);
+const entry = realpathSync(path.resolve(entryArg));
+const code = readFileSync(entry, 'utf8');
+if (!code.includes(ESM_INIT_HELPER)) throw new Error(`esbuild's __esm helper not found in ${entry}`);
+
+const initialized = (globalThis.__sdkInitialized = new Set());
+const sdkModule = new Module(entry, module);
+sdkModule.filename = entry;
+sdkModule.paths = Module._nodeModulePaths(path.dirname(entry));
+require.cache[entry] = sdkModule;
+sdkModule._compile(code.replace(ESM_INIT_HELPER, ESM_INIT_HELPER + RECORD_INITS), entry);
+sdkModule.loaded = true;
+const sdk = sdkModule.exports;
+
+let seen = new Set();
+function initializedSince() {
+  const added = [...initialized].filter((source) => !seen.has(source));
+  seen = new Set(initialized);
+  return added;
+}
+
+async function main() {
+  const report = [{ step: 'load', initialized: initializedSince() }];
+  let handler;
+  for (const step of steps) {
+    const [kind, arg] = [step.slice(0, step.indexOf(':')), step.slice(step.indexOf(':') + 1)];
+    if (kind === 'read') {
+      for (const name of arg.split(',')) void sdk[name];
+    } else if (kind === 'fetch') {
+      handler ??= await sdk.FrontMcpInstance.createFetchHandler({
+        info: { name: 'initialized-modules', version: '1.0.0' },
+        apps: [sdk.app({ id: 'empty', name: 'Empty' })],
+        logging: { level: sdk.LogLevel.Off },
+      });
+      if (arg) {
+        const [method, url] = arg.split(' ');
+        await (await handler(new Request(`http://localhost${url}`, { method }))).arrayBuffer();
+      }
+    } else {
+      throw new Error(`unknown step ${step}`);
+    }
+    report.push({ step, initialized: initializedSince() });
+  }
+  process.stdout.write(JSON.stringify(report));
+}
+
+main().then(
+  () => process.exit(0),
+  (error) => {
+    process.stderr.write(String(error?.stack ?? error));
+    process.exit(1);
+  },
+);

apps/e2e/demo-e2e-bundle-size/e2e/helpers/root-exports.mjs

@@ -0,0 +1,22 @@
+// Loads an SDK entry with require() or import() and prints, as JSON, the kind of each of its exports: node root-exports.mjs <entry> <require|import>
+import { createRequire } from 'node:module';
+import { pathToFileURL } from 'node:url';
+
+const [entry, loader] = process.argv.slice(2);
+const sdk = loader === 'require' ? createRequire(import.meta.url)(entry) : await import(pathToFileURL(entry).href);
+
+function kindOf(value) {
+  if (value === null) return 'null';
+  if (Array.isArray(value)) return 'array';
+  if (typeof value === 'function') {
+    return /^class\b/.test(Function.prototype.toString.call(value)) ? 'class' : 'function';
+  }
+  return typeof value;
+}
+
+const kinds = Object.fromEntries(
+  Object.keys(sdk)
+    .sort()
+    .map((name) => [name, kindOf(sdk[name])]),
+);
+process.stdout.write(JSON.stringify(kinds));

apps/e2e/demo-e2e-bundle-size/e2e/helpers/subsystem-hooks.mjs

@@ -0,0 +1,192 @@
+// Requires the SDK's CommonJS build, runs a job, a workflow, a channel notification, the channel list, a remote app's
+// tool, an ESM package's tool and a request to each OAuth route of a public-mode server, and prints, as JSON, the
+// flows a server plugin's hooks saw, on its last line: node subsystem-hooks.mjs <dist/index.js>
+import { createServer } from 'node:http';
+import { createRequire } from 'node:module';
+
+const [entry] = process.argv.slice(2);
+const sdk = createRequire(import.meta.url)(entry);
+const {
+  App,
+  Channel,
+  ChannelContext,
+  FlowHooksOf,
+  FrontMcpInstance,
+  Job,
+  JobContext,
+  LogLevel,
+  Plugin,
+  Tool,
+  ToolContext,
+} = sdk;
+const { Workflow, connect, z } = sdk;
+
+const OAUTH_ROUTES = {
+  'oauth:authorize': ['GET', '/oauth/authorize'],
+  'oauth:token': ['POST', '/oauth/token'],
+  'oauth:userinfo': ['GET', '/oauth/userinfo'],
+  'oauth:callback': ['GET', '/oauth/callback'],
+  'oauth:connect': ['GET', '/oauth/connect'],
+  'oauth:register': ['POST', '/oauth/register'],
+  'oauth:provider-callback': ['GET', '/oauth/provider/upstream/callback'],
+  'oauth:auth-ui-extra': ['POST', '/oauth/ui/extra'],
+};
+
+const seen = [];
+
+function decorate(target, name, decorator) {
+  decorator(target, name, Object.getOwnPropertyDescriptor(target, name));
+}
+
+class SubsystemHooksPlugin {
+  recordJob(flowCtx) {
+    const { job, workflow } = flowCtx.state;
+    seen.push(workflow ? `jobs:execute-job ${job.name} in ${workflow.name}` : `jobs:execute-job ${job.name}`);
+  }
+  recordChannelSend() {
+    seen.push('channels:send-notification');
+  }
+  recordChannelList() {
+    seen.push('channels:list');
+  }
+  recordToolCall(flowCtx) {
+    const tool = flowCtx.rawInput.request.params.name;
+    if (tool.startsWith('remote') || tool.startsWith('esm')) seen.push(`tools:call-tool ${tool}`);
+  }
+}
+decorate(SubsystemHooksPlugin.prototype, 'recordJob', FlowHooksOf('jobs:execute-job').Did('updateRunState'));
+decorate(SubsystemHooksPlugin.prototype, 'recordChannelSend', FlowHooksOf('channels:send-notification').Will('send'));
+decorate(SubsystemHooksPlugin.prototype, 'recordChannelList', FlowHooksOf('channels:list').Will('listChannels'));
+decorate(SubsystemHooksPlugin.prototype, 'recordToolCall', FlowHooksOf('tools:call-tool').Will('parseInput'));
+for (const flow of Object.keys(OAUTH_ROUTES)) {
+  const method = `record:${flow}`;
+  SubsystemHooksPlugin.prototype[method] = () => seen.push(flow);
+  decorate(SubsystemHooksPlugin.prototype, method, FlowHooksOf(flow).Will('checkIpFilter'));
+}
+Plugin({ name: 'subsystem-hooks', description: 'Records the flows of each subsystem that ran' })(SubsystemHooksPlugin);
+
+class GreetJob extends JobContext {
+  async execute({ name }) {
+    return { message: `Hello, ${name}` };
+  }
+}
+Job({ name: 'greet', inputSchema: { name: z.string() }, outputSchema: { message: z.string() } })(GreetJob);
+
+class GreetWorkflow {}
+Workflow({ name: 'greet-twice', steps: [{ id: 'greet', jobName: 'greet', input: { name: 'Workflow' } }] })(
+  GreetWorkflow,
+);
+
+class AlertChannel extends ChannelContext {
+  async onEvent(payload) {
+    return { content: `alert: ${payload.message}` };
+  }
+}
+Channel({ name: 'alerts', source: { type: 'app-event', event: 'alert' } })(AlertChannel);
+
+class AlertTool extends ToolContext {
+  async execute() {
+    this.scope.channelEventBus.emit('alert', { message: 'raised' });
+    await this.scope.runFlow('channels:list', {});
+    return 'raised';
+  }
+}
+Tool({ name: 'raise-alert', inputSchema: {} })(AlertTool);
+
+class SubsystemsApp {}
+App({
+  name: 'Subsystems',
+  jobs: [GreetJob],
+  workflows: [GreetWorkflow],
+  channels: [AlertChannel],
+  tools: [AlertTool],
+})(SubsystemsApp);
+
+class ShoutTool extends ToolContext {
+  async execute() {
+    return 'HELLO';
+  }
+}
+Tool({ name: 'shout', inputSchema: {} })(ShoutTool);
+class RemoteApp {}
+App({ name: 'Remote', tools: [ShoutTool] })(RemoteApp);
+
+function listen(server) {
+  return new Promise((resolveListen) => server.listen(0, '127.0.0.1', () => resolveListen(server.address().port)));
+}
+
+const portProbe = createServer();
+const remotePort = await listen(portProbe);
+await new Promise((resolveClose) => portProbe.close(resolveClose));
+const remote = await FrontMcpInstance.bootstrap({
+  info: { name: 'remote', version: '1.0.0' },
+  apps: [RemoteApp],
+  http: { port: remotePort },
+  logging: { level: LogLevel.Off },
+});
+
+const ESM_PACKAGE = '@frontmcp-e2e/wave-tools';
+const ESM_BUNDLE = `module.exports = { default: { name: '${ESM_PACKAGE}', version: '1.0.0', tools: [{
+  name: 'wave', description: 'Waves', inputSchema: { type: 'object', properties: {} },
+  execute: async () => ({ content: [{ type: 'text', text: 'wave' }] }),
+}] } };`;
+const esmPackages = createServer((request, response) => {
+  const path = decodeURIComponent(new URL(request.url, 'http://127.0.0.1').pathname).slice(1);
+  if (path === ESM_PACKAGE) {
+    response.writeHead(200, { 'Content-Type': 'application/json' });
+    response.end(
+      JSON.stringify({
+        name: ESM_PACKAGE,
+        'dist-tags': { latest: '1.0.0' },
+        versions: { '1.0.0': { name: ESM_PACKAGE, version: '1.0.0' } },
+        time: { '1.0.0': new Date().toISOString() },
+      }),
+    );
+  } else if (path === `${ESM_PACKAGE}@1.0.0`) {
+    response.writeHead(200, { 'Content-Type': 'application/javascript' });
+    response.end(ESM_BUNDLE);
+  } else {
+    response.writeHead(404);
+    response.end();
+  }
+});
+const esmPort = await listen(esmPackages);
+
+const config = {
+  info: { name: 'subsystem-hooks', version: '1.0.0' },
+  loader: { url: `http://127.0.0.1:${esmPort}` },
+  apps: [
+    SubsystemsApp,
+    { name: 'remote', urlType: 'url', url: `http://127.0.0.1:${remotePort}/`, namespace: 'remote', standalone: false },
+    App.esm(`${ESM_PACKAGE}@^1.0.0`, { namespace: 'esm' }),
+  ],
+  plugins: [SubsystemHooksPlugin],
+  jobs: { enabled: true },
+  channels: { enabled: true },
+  logging: { level: LogLevel.Off },
+};
+
+const client = await connect(config);
+try {
+  await client.callTool('execute_job', { name: 'greet', input: { name: 'Ada' } });
+  await client.callTool('execute_workflow', { name: 'greet-twice' });
+  await client.callTool('raise-alert', {});
+  const tools = await client.listTools();
+  for (const { name } of tools.filter((tool) => tool.name.startsWith('remote') || tool.name.startsWith('esm'))) {
+    await client.callTool(name, {});
+  }
+  await new Promise((resolveDelivery) => setTimeout(resolveDelivery, 100));
+} finally {
+  await client.close();
+}
+
+const handler = await FrontMcpInstance.createFetchHandler(config);
+for (const [method, path] of Object.values(OAUTH_ROUTES)) {
+  const response = await handler(new Request(`http://localhost${path}`, { method }));
+  await response.arrayBuffer();
+}
+
+await remote?.shutdown?.();
+esmPackages.close();
+process.stdout.write(`\n${JSON.stringify(seen)}\n`);
+process.exit(0);

apps/e2e/demo-e2e-bundle-size/e2e/sdk-lazy-subsystems.e2e.spec.ts

@@ -0,0 +1,71 @@
+/**
+ * The SDK's CommonJS build sets up the subsystems a server may never use the first time one of their exports is read,
+ * and a public-mode server loads each of its OAuth flows with the first request that reaches it (#950).
+ *
+ * Needs the packages built: `nx run-many -t build -p sdk plugin-webmcp`.
+ */
+import { execFileSync } from 'node:child_process';
+import { resolve } from 'node:path';
+
+import { workspaceRoot } from './helpers/browser-bundle';
+
+type Step = { step: string; initialized: string[] };
+
+const SUBSYSTEM_MODULE = /^libs\/sdk\/src\/(job|workflow|channel|remote-mcp|esm-loader)\//;
+// Helpers the startup path itself uses: the decorators parse `Tool.esm()` specifiers, the scope normalizes app entries
+const STARTUP_HELPERS = [
+  'libs/sdk/src/esm-loader/package-specifier.ts',
+  'libs/sdk/src/esm-loader/module-specifiers.ts',
+  'libs/sdk/src/job/job.utils.ts',
+  'libs/sdk/src/channel/channel.utils.ts',
+  'libs/sdk/src/channel/sources/completion-events.ts',
+];
+const OAUTH_FLOW_MODULE = /^libs\/sdk\/src\/auth\/flows\/oauth\.[a-z-]+\.flow\.ts$/;
+
+function initializedModules(...steps: string[]): Step[] {
+  const runner = resolve(__dirname, 'helpers', 'initialized-modules.cjs');
+  return JSON.parse(
+    execFileSync(process.execPath, [runner, resolve(workspaceRoot, 'libs/sdk/dist/index.js'), ...steps], {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'pipe'],
+      maxBuffer: 16 * 1024 * 1024,
+    }),
+  );
+}
+
+describe('the subsystems of the SDK under require()', () => {
+  it('are not set up when the SDK loads', () => {
+    const [load] = initializedModules();
+
+    expect(load.initialized.length).toBeGreaterThan(0);
+    const subsystemModules = load.initialized.filter((source) => SUBSYSTEM_MODULE.test(source));
+    expect(subsystemModules.filter((source) => !STARTUP_HELPERS.includes(source))).toEqual([]);
+  });
+
+  it.each([
+    ['JobRegistry', 'libs/sdk/src/job/job.registry.ts'],
+    ['WorkflowRegistry', 'libs/sdk/src/workflow/workflow.registry.ts'],
+    ['ChannelRegistry', 'libs/sdk/src/channel/channel.registry.ts'],
+    ['McpClientService', 'libs/sdk/src/remote-mcp/mcp-client.service.ts'],
+    ['EsmModuleLoader', 'libs/sdk/src/esm-loader/esm-module-loader.ts'],
+  ])('are set up when %s is read', (exportName, source) => {
+    const [, read] = initializedModules(`read:${exportName}`);
+
+    expect(read.initialized).toContain(source);
+  });
+});
+
+describe('the OAuth flows of a public-mode server', () => {
+  it('load with the first request that reaches each one', () => {
+    const [, start, authorize, token] = initializedModules(
+      'fetch:',
+      'fetch:GET /oauth/authorize',
+      'fetch:POST /oauth/token',
+    );
+    const oauthFlows = (step: Step) => step.initialized.filter((source) => OAUTH_FLOW_MODULE.test(source));
+
+    expect(oauthFlows(start)).toEqual([]);
+    expect(oauthFlows(authorize)).toEqual(['libs/sdk/src/auth/flows/oauth.authorize.flow.ts']);
+    expect(oauthFlows(token)).toEqual(['libs/sdk/src/auth/flows/oauth.token.flow.ts']);
+  });
+});

apps/e2e/demo-e2e-bundle-size/e2e/sdk-root-exports.e2e.spec.ts

@@ -0,0 +1,42 @@
+/**
+ * The SDK's CommonJS and ESM entry points export the same names, each of the same kind, as the list pinned in
+ * `fixtures/sdk-root-exports.json` (#950): loading a subsystem on first use changes none of them.
+ *
+ * Needs the packages built: `nx run-many -t build -p sdk plugin-webmcp`.
+ */
+import { execFileSync } from 'node:child_process';
+import { resolve } from 'node:path';
+
+import { readJSON } from '@frontmcp/utils';
+
+import { workspaceRoot } from './helpers/browser-bundle';
+
+const pinnedExports = () => readJSON<Record<string, string>>(resolve(__dirname, 'fixtures', 'sdk-root-exports.json'));
+
+function exportKinds(entry: string, loader: 'require' | 'import'): Record<string, string> {
+  const runner = resolve(__dirname, 'helpers', 'root-exports.mjs');
+  return JSON.parse(
+    execFileSync(process.execPath, [runner, resolve(workspaceRoot, entry), loader], {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'pipe'],
+      maxBuffer: 16 * 1024 * 1024,
+    }),
+  );
+}
+
+describe('the root exports of @frontmcp/sdk', () => {
+  it('are the pinned names and kinds under require()', async () => {
+    expect(exportKinds('libs/sdk/dist/index.js', 'require')).toEqual(await pinnedExports());
+  });
+
+  it("are the pinned names and kinds when an ESM import loads the CommonJS build, as Node's export detection sees it", async () => {
+    expect(exportKinds('libs/sdk/dist/index.js', 'import')).toEqual({
+      ...(await pinnedExports()),
+      'module.exports': 'object',
+    });
+  });
+
+  it('are the pinned names and kinds under import', async () => {
+    expect(exportKinds('libs/sdk/dist/esm/index.mjs', 'import')).toEqual(await pinnedExports());
+  });
+});