diff --git a/apps/e2e/demo-e2e-bundle-size/e2e/browser-bundle-size.e2e.spec.ts b/apps/e2e/demo-e2e-bundle-size/e2e/browser-bundle-size.e2e.spec.ts index bb0263729..e8a81df23 100644 --- a/apps/e2e/demo-e2e-bundle-size/e2e/browser-bundle-size.e2e.spec.ts +++ b/apps/e2e/demo-e2e-bundle-size/e2e/browser-bundle-size.e2e.spec.ts @@ -24,8 +24,8 @@ import { } from './helpers/browser-bundle'; const BUDGETS: Array<{ entry: string; label: string; maxBytes: number }> = [ - { entry: 'create-tool.ts', label: 'create() + one tool', maxBytes: 1_333_877 }, - { entry: 'create-webmcp-tool.ts', label: 'create() + WebMcpPlugin + one tool', maxBytes: 1_340_744 }, + { entry: 'create-tool.ts', label: 'create() + one tool', maxBytes: 1_334_799 }, + { entry: 'create-webmcp-tool.ts', label: 'create() + WebMcpPlugin + one tool', maxBytes: 1_341_666 }, ]; const VITE_INITIAL_CHUNK_MAX_BYTES = 960_000; diff --git a/apps/e2e/demo-e2e-dts-consumer/e2e/dts-consumer.e2e.spec.ts b/apps/e2e/demo-e2e-dts-consumer/e2e/dts-consumer.e2e.spec.ts index 2def860bf..9391d78c7 100644 --- a/apps/e2e/demo-e2e-dts-consumer/e2e/dts-consumer.e2e.spec.ts +++ b/apps/e2e/demo-e2e-dts-consumer/e2e/dts-consumer.e2e.spec.ts @@ -49,6 +49,7 @@ const fixtureFiles = [ 'http-routes-must-compile.ts', 'shared-server-must-compile.ts', 'durable-jobs-must-compile.ts', + 'job-runners-must-compile.ts', 'build-manifest-must-compile.ts', ].map((fileName) => path.join(fixtureDir, fileName)); @@ -234,6 +235,10 @@ describe('@frontmcp/sdk declarations in a strict consumer project', () => { expect(describeEach(inFixtureFile('durable-jobs-must-compile.ts'))).toEqual([]); }); + it("types @Job({ runner }), jobs.runner, the run's runner, JOB_WORKER_MARKER and the runner errors", () => { + expect(describeEach(inFixtureFile('job-runners-must-compile.ts'))).toEqual([]); + }); + it('types instructionsBudget, buildManifest(), lintServerManifest() and the manifest schema', () => { expect(describeEach(inFixtureFile('build-manifest-must-compile.ts'))).toEqual([]); }); diff --git a/apps/e2e/demo-e2e-dts-consumer/fixture/job-runners-must-compile.ts b/apps/e2e/demo-e2e-dts-consumer/fixture/job-runners-must-compile.ts new file mode 100644 index 000000000..bc33761bf --- /dev/null +++ b/apps/e2e/demo-e2e-dts-consumer/fixture/job-runners-must-compile.ts @@ -0,0 +1,84 @@ +import 'reflect-metadata'; + +import { + App, + FrontMcp, + Job, + JOB_WORKER_MARKER, + JobContext, + JobRunnerBootError, + JobRunnerUnavailableError, + z, + type JobRunner, + type JobRunnerInfo, + type JobRunRecord, + type JobStateStore, +} from '@frontmcp/sdk'; +import { reapOrphans } from '@frontmcp/utils'; + +@Job({ name: 'export', inputSchema: { dir: z.string() }, outputSchema: { files: z.number() }, runner: 'detached' }) +export class ExportJob extends JobContext { + async execute(input: { dir: string }) { + await this.progress(1, 1, `exported ${input.dir}`); + return { files: 1 }; + } +} + +@Job({ name: 'hash', inputSchema: { text: z.string() }, outputSchema: { length: z.number() }, runner: 'worker' }) +export class HashJob extends JobContext { + async execute(input: { text: string }) { + return { length: input.text.length }; + } +} + +@App({ name: 'exports', jobs: [ExportJob, HashJob] }) +class ExportsApp {} + +@FrontMcp({ + info: { name: 'job-runners', version: '1.0.0' }, + apps: [ExportsApp], + sqlite: { path: '/var/lib/exports/state.sqlite' }, + jobs: { enabled: true, runner: 'inline' }, +}) +export class JobRunnersServer {} + +export function runnerOf(run: JobRunRecord): JobRunner | undefined { + const runner: JobRunnerInfo | undefined = run.runner; + return runner?.kind; +} + +export function isRunnerError(error: unknown): boolean { + return error instanceof JobRunnerUnavailableError || error instanceof JobRunnerBootError; +} + +export function reapRunners(): Promise { + return reapOrphans(JOB_WORKER_MARKER); +} + +/** A store that keeps heartbeats and claims: its runs are reconciled on boot. */ +export function heartbeats(store: JobStateStore): Promise | undefined { + void store.touchRunner?.('runner-1', 5); + void store.claimRun?.('run-1'); + return store.isRunnerAlive?.('runner-1'); +} + +@Job({ + name: 'thread', + inputSchema: {}, + outputSchema: {}, + // @ts-expect-error a runner is 'inline', 'worker' or 'detached' + runner: 'thread', +}) +export class BadRunnerJob extends JobContext { + async execute() { + return {}; + } +} + +@FrontMcp({ + info: { name: 'job-runners-bad', version: '1.0.0' }, + apps: [ExportsApp], + // @ts-expect-error jobs.runner is 'inline', 'worker' or 'detached' + jobs: { enabled: true, runner: 'process' }, +}) +export class BadRunnerServer {} diff --git a/apps/e2e/demo-e2e-dts-consumer/fixture/tsconfig.json b/apps/e2e/demo-e2e-dts-consumer/fixture/tsconfig.json index 1f9a27f5e..ae3bd10b0 100644 --- a/apps/e2e/demo-e2e-dts-consumer/fixture/tsconfig.json +++ b/apps/e2e/demo-e2e-dts-consumer/fixture/tsconfig.json @@ -35,6 +35,7 @@ "duties-must-compile.ts", "shared-server-must-compile.ts", "durable-jobs-must-compile.ts", + "job-runners-must-compile.ts", "materialized-resources-must-compile.ts", "duties-must-compile.ts", "http-routes-must-compile.ts" diff --git a/apps/e2e/demo-e2e-jobs/e2e/jobs-detached.e2e.spec.ts b/apps/e2e/demo-e2e-jobs/e2e/jobs-detached.e2e.spec.ts new file mode 100644 index 000000000..eee1e76ee --- /dev/null +++ b/apps/e2e/demo-e2e-jobs/e2e/jobs-detached.e2e.spec.ts @@ -0,0 +1,205 @@ +/** + * E2E: detached job runs (proposal 2.2 b). A job on the detached runner runs in a process of its own and keeps + * its run in the server's SQLite store: it finishes after the server that started it was killed, a new server + * instance reads and cancels it (the runner sees the cancel and runs the job's cleanup), and a run whose runner + * died is marked `interrupted` by the next server to boot, which runs the cleanup itself. A job on the worker + * runner runs in a worker thread of the server. + */ +import * as fs from 'node:fs'; +import * as os from 'node:os'; + +import { expect, McpTestClient, TestServer } from '@frontmcp/testing'; +import { pathJoin } from '@frontmcp/utils'; + +interface RunHandle { + runId: string; + state: string; + uri: string; +} + +interface RunStatus { + runId: string; + state: string; + result?: Record; + error?: { name: string; message: string }; +} + +const SERVER = 'npx tsx apps/e2e/demo-e2e-jobs/src/main.detached.ts'; + +let dir: string; +let env: Record; +const servers: TestServer[] = []; +const clients: McpTestClient[] = []; + +async function until(read: () => Promise, done: (value: T) => boolean, what: string, timeoutMs = 60_000) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const value = await read(); + if (done(value)) return value; + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}; last: ${JSON.stringify(value)}`); + await new Promise((resolve) => setTimeout(resolve, 200)); + } +} + +async function startServer(): Promise<{ server: TestServer; mcp: McpTestClient }> { + const server = await TestServer.start({ command: SERVER, project: 'demo-e2e-jobs', env, startupTimeout: 60_000 }); + servers.push(server); + const mcp = await McpTestClient.create({ + baseUrl: server.info.baseUrl, + transport: 'streamable-http', + }).buildAndConnect(); + clients.push(mcp); + return { server, mcp }; +} + +/** Kill the server process outright, as a crash would, then let the harness release its port. */ +async function killServer(server: TestServer): Promise { + const pid = server.info.pid; + if (pid !== undefined) process.kill(pid, 'SIGKILL'); + await server.stop(); +} + +function events(): string[] { + const file = env['JOBS_EVENTS_PATH']; + return fs.existsSync(file) ? fs.readFileSync(file, 'utf8').split('\n').filter(Boolean) : []; +} + +/** The pid that wrote `::`, once it is there. */ +async function pidOf(tag: string, event: string): Promise { + const prefix = `${tag}:${event}:`; + const line = await until( + async () => events().find((entry) => entry.startsWith(prefix)), + (found) => found !== undefined, + `${prefix} in the events`, + ); + return Number(String(line).slice(prefix.length)); +} + +function isRunning(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function start(mcp: McpTestClient, name: string, input: Record): Promise { + const result = await mcp.tools.call('execute_job', { name, input, background: true }); + expect(result).toBeSuccessful(); + return result.json(); +} + +async function status(mcp: McpTestClient, runId: string): Promise { + return (await mcp.tools.call('get_job_status', { runId })).json(); +} + +async function settled(mcp: McpTestClient, runId: string): Promise { + return until( + () => status(mcp, runId), + (run) => !['pending', 'running', 'retrying'].includes(run.state), + `run ${runId} to settle`, + ); +} + +describe('Detached jobs E2E', () => { + beforeEach(() => { + dir = fs.mkdtempSync(pathJoin(os.tmpdir(), 'frontmcp-e2e-detached-')); + env = { JOBS_SQLITE_PATH: pathJoin(dir, 'jobs.sqlite'), JOBS_EVENTS_PATH: pathJoin(dir, 'events.log') }; + }); + + afterEach(async () => { + for (const mcp of clients.splice(0)) await mcp.disconnect().catch(() => undefined); + for (const server of servers.splice(0)) await server.stop().catch(() => undefined); + for (const entry of events()) { + const pid = Number(entry.split(':').pop()); + if (pid !== process.pid && isRunning(pid)) process.kill(pid, 'SIGKILL'); + } + fs.rmSync(dir, { recursive: true, force: true }); + }); + + it('finishes a run after the server that started it was killed, and a new instance reads it', async () => { + const first = await startServer(); + const handle = await start(first.mcp, 'slow-count', { tag: 'count', steps: 20, stepMs: 200 }); + expect(handle).toEqual({ runId: expect.any(String), state: 'pending', uri: `job://runs/${handle.runId}` }); + const runnerPid = await pidOf('count', 'started'); + expect(runnerPid).not.toBe(first.server.info.pid); + await until( + () => status(first.mcp, handle.runId), + (run) => run.state === 'running', + 'the run to start', + ); + + await killServer(first.server); + expect(isRunning(runnerPid)).toBe(true); + + const second = await startServer(); + const read = await second.mcp.resources.read(handle.uri); + expect(read).toBeSuccessful(); + const finished = await settled(second.mcp, handle.runId); + expect(finished).toEqual(expect.objectContaining({ state: 'completed', result: { counted: 20, pid: runnerPid } })); + await until( + async () => isRunning(runnerPid), + (running) => !running, + 'the runner to exit', + ); + }, 180_000); + + it('lets a new instance cancel a run another one started; the runner runs the cleanup', async () => { + const first = await startServer(); + const handle = await start(first.mcp, 'wait-for-cancel', { tag: 'cancel' }); + const runnerPid = await pidOf('cancel', 'started'); + await first.server.stop(); + + const second = await startServer(); + const cancelled = await second.mcp.tools.call('cancel_job', { runId: handle.runId }); + + expect(cancelled).toBeSuccessful(); + expect(cancelled.json()).toEqual({ runId: handle.runId, state: 'cancelled' }); + expect(await pidOf('cancel', 'cleanup:cancel')).toBe(runnerPid); + expect((await settled(second.mcp, handle.runId)).state).toBe('cancelled'); + await until( + async () => isRunning(runnerPid), + (running) => !running, + 'the runner to exit', + ); + }, 180_000); + + it('marks a run whose runner died interrupted when the next server boots, and runs its cleanup there', async () => { + const first = await startServer(); + const handle = await start(first.mcp, 'wait-for-cancel', { tag: 'orphan' }); + const runnerPid = await pidOf('orphan', 'started'); + await first.server.stop(); + process.kill(runnerPid, 'SIGKILL'); + await until( + async () => isRunning(runnerPid), + (running) => !running, + 'the runner to die', + ); + + const second = await startServer(); + const ended = await settled(second.mcp, handle.runId); + + expect(ended).toEqual( + expect.objectContaining({ + state: 'interrupted', + error: expect.objectContaining({ message: expect.stringContaining('its runner is gone') }), + }), + ); + expect(await pidOf('orphan', 'cleanup:interrupt')).toBe(second.server.info.pid); + }, 180_000); + + it('runs a job on the worker runner in a worker thread of the server', async () => { + const { mcp, server } = await startServer(); + + const result = await mcp.tools.call('execute_job', { name: 'crunch-in-worker', input: { n: 21 } }); + + expect(result).toBeSuccessful(); + expect(result.json<{ state: string; result: Record }>()).toEqual( + expect.objectContaining({ + state: 'completed', + result: { doubled: 42, isMainThread: false, pid: server.info.pid }, + }), + ); + }, 120_000); +}); diff --git a/apps/e2e/demo-e2e-jobs/src/apps/detached/index.ts b/apps/e2e/demo-e2e-jobs/src/apps/detached/index.ts new file mode 100644 index 000000000..ac8901de6 --- /dev/null +++ b/apps/e2e/demo-e2e-jobs/src/apps/detached/index.ts @@ -0,0 +1,11 @@ +import { App } from '@frontmcp/sdk'; + +import { CrunchInWorkerJob, SlowCountJob, WaitForCancelJob } from './jobs/detached.jobs'; + +/** Jobs that run detached, in processes of their own, and one that runs in a worker thread (proposal 2.2 b). */ +@App({ + name: 'Detached', + description: 'Detached job runs that outlive the server, and a job on the worker runner', + jobs: [SlowCountJob, WaitForCancelJob, CrunchInWorkerJob], +}) +export class DetachedApp {} diff --git a/apps/e2e/demo-e2e-jobs/src/apps/detached/jobs/detached.jobs.ts b/apps/e2e/demo-e2e-jobs/src/apps/detached/jobs/detached.jobs.ts new file mode 100644 index 000000000..a8ab0721c --- /dev/null +++ b/apps/e2e/demo-e2e-jobs/src/apps/detached/jobs/detached.jobs.ts @@ -0,0 +1,68 @@ +import { appendFileSync } from 'node:fs'; +import { isMainThread } from 'node:worker_threads'; + +import { z } from '@frontmcp/lazy-zod'; +import { Job, JobContext, type JobCleanup, type JobPermission } from '@frontmcp/sdk'; + +/** Any caller may read and cancel these runs: each test client is a different anonymous user. */ +const anyone: JobPermission[] = [{ action: 'read' }, { action: 'delete' }]; + +/** Append `::` to the file the test reads, from whichever process runs the job. */ +function record(tag: string, event: string): void { + const events = process.env['JOBS_EVENTS_PATH']; + if (events) appendFileSync(events, `${tag}:${event}:${process.pid}\n`); +} + +const recordCleanup: JobCleanup = (job, reason) => { + record((job.input as { tag: string }).tag, `cleanup:${reason}`); +}; + +@Job({ + name: 'slow-count', + description: 'Counts `steps` steps, `stepMs` apart, reporting each one as progress', + inputSchema: { tag: z.string(), steps: z.number().int().positive(), stepMs: z.number().int().nonnegative() }, + outputSchema: { counted: z.number(), pid: z.number() }, + timeout: 0, + permissions: anyone, +}) +export class SlowCountJob extends JobContext { + async execute({ tag, steps, stepMs }: { tag: string; steps: number; stepMs: number }) { + record(tag, 'started'); + for (let step = 1; step <= steps; step++) { + await new Promise((resolve) => setTimeout(resolve, stepMs)); + await this.progress(step, steps, `step ${step}`); + } + return { counted: steps, pid: process.pid }; + } +} + +@Job({ + name: 'wait-for-cancel', + description: 'Runs until it is cancelled or interrupted', + inputSchema: { tag: z.string() }, + outputSchema: {}, + timeout: 0, + permissions: anyone, + cleanup: recordCleanup, +}) +export class WaitForCancelJob extends JobContext { + async execute({ tag }: { tag: string }): Promise> { + record(tag, 'started'); + await new Promise((resolve) => this.signal?.addEventListener('abort', resolve, { once: true })); + return {}; + } +} + +@Job({ + name: 'crunch-in-worker', + description: 'Doubles a number in a worker thread of the server', + inputSchema: { n: z.number() }, + outputSchema: { doubled: z.number(), isMainThread: z.boolean(), pid: z.number() }, + runner: 'worker', +}) +export class CrunchInWorkerJob extends JobContext { + async execute({ n }: { n: number }) { + await this.progress(1, 1, 'crunched'); + return { doubled: n * 2, isMainThread, pid: process.pid }; + } +} diff --git a/apps/e2e/demo-e2e-jobs/src/main.detached.ts b/apps/e2e/demo-e2e-jobs/src/main.detached.ts new file mode 100644 index 000000000..425b526dc --- /dev/null +++ b/apps/e2e/demo-e2e-jobs/src/main.detached.ts @@ -0,0 +1,26 @@ +/** + * Jobs server whose jobs run detached by default, in processes of their own, keeping their runs in the + * top-level SQLite store, so a run outlives the server that started it. + */ +import * as os from 'node:os'; + +import { FrontMcp, LogLevel } from '@frontmcp/sdk'; +import { pathJoin } from '@frontmcp/utils'; + +import { DetachedApp } from './apps/detached'; + +const DEFAULT_PORT = 3123; +const parsedPort = Number.parseInt(process.env['PORT'] ?? '', 10); +const port = Number.isInteger(parsedPort) && parsedPort >= 1 && parsedPort <= 65_535 ? parsedPort : DEFAULT_PORT; +const sqlitePath = process.env['JOBS_SQLITE_PATH'] ?? pathJoin(os.tmpdir(), 'frontmcp-detached-jobs.sqlite'); + +@FrontMcp({ + info: { name: 'Demo E2E Detached Jobs', version: '0.1.0' }, + apps: [DetachedApp], + logging: { level: LogLevel.Warn }, + http: { port }, + auth: { mode: 'public' }, + sqlite: { path: sqlitePath }, + jobs: { enabled: true, runner: 'detached' }, +}) +export default class Server {} diff --git a/libs/sdk/src/common/decorators/job.decorator.ts b/libs/sdk/src/common/decorators/job.decorator.ts index 23f2bd43e..6b21a3c47 100644 --- a/libs/sdk/src/common/decorators/job.decorator.ts +++ b/libs/sdk/src/common/decorators/job.decorator.ts @@ -3,6 +3,7 @@ import 'reflect-metadata'; import type z from '@frontmcp/lazy-zod'; import { parsePackageSpecifier } from '../../esm-loader/package-specifier'; +import { registerWorkerJob } from '../../offload/worker-jobs'; import { type JobContext } from '../interfaces'; import { type EsmOptions, type RemoteOptions, type ToolInputType, type ToolOutputType } from '../metadata'; import { frontMcpJobMetadataSchema, type JobMetadata } from '../metadata/job.metadata'; @@ -12,15 +13,21 @@ import { frontMcpJobMetadataSchema, type JobMetadata } from '../metadata/job.met import { JobKind, type JobEsmTargetRecord, type JobRemoteRecord } from '../records/job.record'; import { extendedJobMetadata, FrontMcpJobTokens } from '../tokens'; +import { captureCallerFile } from '../utils/caller-dir.utils'; import { validateRemoteUrl } from '../utils/validate-remote-url'; import { type ToolInputOf, type ToolOutputOf } from './tool.decorator'; +/** Basenames of THIS decorator file, skipped so the captured frame is the module that declares the job. */ +const JOB_DECORATOR_BASENAMES = ['job.decorator.ts', 'job.decorator.js', 'job.decorator.mjs', 'job.decorator.cjs']; + /** * Decorator that marks a class as a Job and provides metadata. */ function _FrontMcpJob(providedMetadata: JobMetadata): ClassDecorator { + const declaringModule = captureCallerFile(JOB_DECORATOR_BASENAMES); return (target: any) => { const metadata = frontMcpJobMetadataSchema.parse(providedMetadata); + registerWorkerJob(target, providedMetadata, declaringModule); Reflect.defineMetadata(FrontMcpJobTokens.type, true, target); const extended = {}; for (const property in metadata) { @@ -51,8 +58,10 @@ function frontMcpJob< >( providedMetadata: T, ): (handler: FrontMcpJobExecuteHandler) => () => FrontMcpJobExecuteHandler { + const declaringModule = captureCallerFile(JOB_DECORATOR_BASENAMES); return (execute) => { const metadata = frontMcpJobMetadataSchema.parse(providedMetadata); + registerWorkerJob(execute, providedMetadata, declaringModule); const jobFunction = function () { return execute; }; diff --git a/libs/sdk/src/common/interfaces/job.interface.ts b/libs/sdk/src/common/interfaces/job.interface.ts index e2b12d330..4cbabfa98 100644 --- a/libs/sdk/src/common/interfaces/job.interface.ts +++ b/libs/sdk/src/common/interfaces/job.interface.ts @@ -133,6 +133,11 @@ export abstract class JobContext< return this._logs; } + /** @internal Keep the log lines the job wrote where `execute()` ran, a worker thread. */ + keepLogs(lines: readonly string[]): void { + this._logs.push(...lines); + } + /** A job (or a workflow step running it) calls tools on the `'job'` surface. */ protected override callToolSurface(): CallSurface { return 'job'; diff --git a/libs/sdk/src/common/metadata/__tests__/jobs-config-schema.spec.ts b/libs/sdk/src/common/metadata/__tests__/jobs-config-schema.spec.ts index 870e995c6..460c491a8 100644 --- a/libs/sdk/src/common/metadata/__tests__/jobs-config-schema.spec.ts +++ b/libs/sdk/src/common/metadata/__tests__/jobs-config-schema.spec.ts @@ -96,3 +96,15 @@ describe('frontMcpMetadataSchema — durable jobs config', () => { ).toThrow('cleanup.timeoutMs must be at most 2147483647 ms (about 24.8 days)'); }); }); + +describe('frontMcpMetadataSchema — job runners', () => { + it.each(['inline', 'worker', 'detached'])('keeps runner %s', (runner) => { + expect(parseJobs({ enabled: true, runner })?.['runner']).toBe(runner); + }); + + it('refuses an unknown runner with the runners it takes', () => { + expect(() => parseJobs({ enabled: true, runner: 'thread' })).toThrow( + /jobs\.runner must be 'inline', 'worker' or 'detached'/, + ); + }); +}); diff --git a/libs/sdk/src/common/metadata/front-mcp.metadata.ts b/libs/sdk/src/common/metadata/front-mcp.metadata.ts index 55cba25b4..5fd5249d8 100644 --- a/libs/sdk/src/common/metadata/front-mcp.metadata.ts +++ b/libs/sdk/src/common/metadata/front-mcp.metadata.ts @@ -433,6 +433,14 @@ export interface FrontMcpBaseMetadata { * @default 300000 */ defaultTimeoutMs?: number; + /** + * Where the attempts of a job that sets no `runner` run: `'inline'` on the server's event loop, `'worker'` in a + * worker thread of the offload pool, `'detached'` in a process of their own that survives a server restart + * (Node only, from the server's entry, with a store other processes share). + * + * @default 'inline' + */ + runner?: 'inline' | 'worker' | 'detached'; /** * When job attempts (and workflow steps) may start in this process: at most `concurrency` at once, * each claiming `memoryMb` of free memory. An attempt waits up to `maxWaitMs` (default 600000) and @@ -746,6 +754,9 @@ export const frontMcpBaseSchema = z.object({ .nonnegative() .max(MAX_JOB_TIMER_MS, timerLimitMessage('jobs.defaultTimeoutMs')) .optional(), + runner: z + .enum(['inline', 'worker', 'detached'], { message: "jobs.runner must be 'inline', 'worker' or 'detached'" }) + .optional(), admission: z .object({ concurrency: z.number().int().positive().optional(), diff --git a/libs/sdk/src/common/metadata/job.metadata.ts b/libs/sdk/src/common/metadata/job.metadata.ts index ad47c7680..d995e2e4c 100644 --- a/libs/sdk/src/common/metadata/job.metadata.ts +++ b/libs/sdk/src/common/metadata/job.metadata.ts @@ -86,6 +86,21 @@ const jobCleanupSchema = z.union([ }), ]); +// ============================================ +// Job Runner +// ============================================ + +/** + * Where an attempt's `execute()` runs: `'inline'` on the server's event loop, `'worker'` in a worker thread of the + * server's offload pool (no DI container there, as for `this.offload()`), `'detached'` in a process of its own that + * outlives the server and keeps the run in the job store. + */ +export type JobRunner = 'inline' | 'worker' | 'detached'; + +export const jobRunnerSchema = z.enum(['inline', 'worker', 'detached'], { + message: "runner must be 'inline', 'worker' or 'detached'", +}); + // ============================================ // Job Metadata // ============================================ @@ -138,6 +153,9 @@ export interface JobMetadata< */ cleanup?: JobCleanup | JobCleanupOptions; + /** Where the job's attempts run. Default: `jobs.runner`, else `'inline'`. */ + runner?: JobRunner; + /** * Retry configuration for failed executions. */ @@ -174,6 +192,7 @@ export const frontMcpJobMetadataSchema = z outputSchema: z.instanceof(Object), timeout: z.number().int().min(0).max(MAX_JOB_TIMER_MS, timerLimitMessage('timeout')).optional(), cleanup: jobCleanupSchema.optional(), + runner: jobRunnerSchema.optional(), retry: jobRetryConfigSchema.optional(), tags: z.array(z.string().min(1)).optional(), labels: z.record(z.string(), z.string()).optional(), diff --git a/libs/sdk/src/common/tokens/job.tokens.ts b/libs/sdk/src/common/tokens/job.tokens.ts index c8f871256..08e70d3c0 100644 --- a/libs/sdk/src/common/tokens/job.tokens.ts +++ b/libs/sdk/src/common/tokens/job.tokens.ts @@ -11,6 +11,7 @@ export const FrontMcpJobTokens = { outputSchema: tokenFactory.meta('jobOutputSchema'), timeout: tokenFactory.meta('jobTimeout'), cleanup: tokenFactory.meta('jobCleanup'), + runner: tokenFactory.meta('jobRunner'), retry: tokenFactory.meta('jobRetry'), tags: tokenFactory.meta('jobTags'), labels: tokenFactory.meta('jobLabels'), diff --git a/libs/sdk/src/errors/job.errors.ts b/libs/sdk/src/errors/job.errors.ts index d7f6a1f65..3ab73b461 100644 --- a/libs/sdk/src/errors/job.errors.ts +++ b/libs/sdk/src/errors/job.errors.ts @@ -50,10 +50,29 @@ export class JobCancelledError extends PublicMcpError { } } -/** The server shut down while the run was active; the run is `interrupted`. */ +/** The run stopped before it finished because its process went away; the run is `interrupted`. */ export class JobInterruptedError extends PublicMcpError { - constructor(runId: string) { - super(`Run "${runId}" was interrupted: the server shut down`, 'JOB_INTERRUPTED', 503); + constructor(runId: string, why = 'the server shut down') { + super(`Run "${runId}" was interrupted: ${why}`, 'JOB_INTERRUPTED', 503); + } +} + +/** The job's runner cannot run here: a detached run needs a Node server started from its entry and a shared store. */ +export class JobRunnerUnavailableError extends PublicMcpError { + constructor(jobName: string, why: string, remedy: string) { + super(`Job "${jobName}" cannot use its runner: ${why}`, 'JOB_RUNNER_UNAVAILABLE', 501, undefined, { + say: remedy, + }); + } +} + +/** The process of a detached run could not start, or exited before it took the run over; the run is `failed`. */ +export class JobRunnerBootError extends PublicMcpError { + constructor(runId: string, cause: string) { + super(`The detached runner of run "${runId}" did not start: ${cause}`, 'JOB_RUNNER_BOOT_FAILED', 503, undefined, { + say: 'Run the job again; if it keeps failing, check that the server starts from its entry', + tool: 'execute_job', + }); } } diff --git a/libs/sdk/src/front-mcp/detached-job.flag.ts b/libs/sdk/src/front-mcp/detached-job.flag.ts new file mode 100644 index 000000000..1abb24095 --- /dev/null +++ b/libs/sdk/src/front-mcp/detached-job.flag.ts @@ -0,0 +1,31 @@ +/** The `spawnOwned` marker of detached job runners, for `reapOrphans(JOB_WORKER_MARKER)`. */ +export const JOB_WORKER_MARKER = 'frontmcp-job'; + +/** Set in a detached runner's environment to the run it is started for. */ +export const RUN_JOB_ENV_VAR = 'FRONTMCP_RUN_JOB_ID'; + +/** Set on the config a detached runner boots, so its scopes run that run and reconcile nothing. */ +export const JOB_WORKER_FLAG = '__jobWorker' as const; + +/** Set by the entries that hand a detached runner over to its run (`bootstrap()`, `runStdio()`, `runUnixSocket()`). */ +const ENTRY_SERVER_FLAG = '__entryServer'; + +/** The run a detached runner was started for, from its environment. */ +export function detachedJobRunId(): string | undefined { + return typeof process === 'undefined' ? undefined : process.env?.[RUN_JOB_ENV_VAR]; +} + +/** Whether a scope's config was booted by a detached runner. */ +export function isJobWorker(metadata: object): boolean { + return (metadata as Record)[JOB_WORKER_FLAG] === true; +} + +/** Mark a config as served by an entry a detached runner can run again. */ +export function markEntryServer(config: object): void { + (config as Record)[ENTRY_SERVER_FLAG] = true; +} + +/** Whether a scope's server was started by an entry a detached runner can run again. */ +export function isEntryServer(metadata: object): boolean { + return (metadata as Record)[ENTRY_SERVER_FLAG] === true; +} diff --git a/libs/sdk/src/front-mcp/front-mcp.ts b/libs/sdk/src/front-mcp/front-mcp.ts index 927ba3a5d..1ad5b6155 100644 --- a/libs/sdk/src/front-mcp/front-mcp.ts +++ b/libs/sdk/src/front-mcp/front-mcp.ts @@ -28,6 +28,7 @@ import { composeCallerInstructions } from '../skill/skill-instructions.helper'; import { computeTaskCapabilities } from '../task/task-scope.helper'; import { createMcpServer } from '../transport/mcp-server.factory'; import type { FetchHandlerCtx, WebFetchHandler } from '../transport/web-fetch-handler'; +import { detachedJobRunId, markEntryServer } from './detached-job.flag'; import { markDirectOnly } from './direct-only.flag'; import { createMcpGlobalProviders } from './front-mcp.providers'; import { @@ -102,6 +103,12 @@ function parseCliConfig(options: FrontMcpConfigInput, { fileLog }: { fileLog: bo return parsedConfig; } +/** A detached job runner runs the server's entry again: it runs its one run instead of serving, then exits. */ +async function exitAfterDetachedRun(config: FrontMcpConfigInput | FrontMcpConfigType, runId: string): Promise { + const { runDetachedJob } = await import('../job/runner/detached-job.worker.js'); + process.exit(await runDetachedJob(config, runId)); +} + export class FrontMcpInstance implements FrontMcpInterface { config: FrontMcpConfigType; readonly ready: Promise; @@ -388,6 +395,12 @@ export class FrontMcpInstance implements FrontMcpInterface { : options, ); + const detachedRunId = detachedJobRunId(); + if (detachedRunId) { + await exitAfterDetachedRun(parsedConfig, detachedRunId); + return undefined; + } + // When FRONTMCP_DAEMON_SOCKET is set (e.g., SEA binary started as daemon), // run in Unix socket mode instead of normal HTTP server const daemonSocket = process.env['FRONTMCP_DAEMON_SOCKET']; @@ -396,6 +409,7 @@ export class FrontMcpInstance implements FrontMcpInterface { return undefined; } + markEntryServer(parsedConfig); const frontMcp = new FrontMcpInstance(parsedConfig); await frontMcp.ready; @@ -697,6 +711,11 @@ export class FrontMcpInstance implements FrontMcpInterface { ): Promise<{ close: () => Promise }> { if (isSchemaExtraction()) return { close: async () => undefined }; const { socketPath, sqlite, ...configInput } = options; + const detachedRunId = detachedJobRunId(); + if (detachedRunId) { + await exitAfterDetachedRun({ ...configInput, ...(sqlite ? { sqlite } : {}) }, detachedRunId); + return { close: async () => undefined }; + } // Parse config with HTTP enabled, overriding socketPath const parsedConfig = frontMcpMetadataSchema.parse({ @@ -709,6 +728,7 @@ export class FrontMcpInstance implements FrontMcpInterface { ...(sqlite ? { sqlite } : {}), }); + markEntryServer(parsedConfig); const frontMcp = new FrontMcpInstance(parsedConfig); await frontMcp.ready; @@ -784,6 +804,8 @@ export class FrontMcpInstance implements FrontMcpInterface { // redirection, env flags) — so invalid input fails fast and a failed // resolve never trips the single-connection guard below. const options = resolveConfigInput(optionsOrClass); + const detachedRunId = detachedJobRunId(); + if (detachedRunId) return exitAfterDetachedRun(options, detachedRunId); // ── Stdio stdout protection ────────────────────────────────────────── // In stdio mode, stdout is the MCP JSON-RPC channel. ANY non-protocol @@ -844,6 +866,7 @@ export class FrontMcpInstance implements FrontMcpInterface { }; } + markEntryServer(parsedConfig); const frontMcp = new FrontMcpInstance(parsedConfig); await frontMcp.ready; // Scopes are now live (sqlite/redis/channels/jobs) — shut them down if a diff --git a/libs/sdk/src/index.ts b/libs/sdk/src/index.ts index 1b2438f32..67e67875f 100644 --- a/libs/sdk/src/index.ts +++ b/libs/sdk/src/index.ts @@ -203,11 +203,13 @@ export type { export type { JobStateStore, JobRunRecord, + JobRunnerInfo, WorkflowRunRecord, JobExecutionState, JobRunProgress, JobDefinitionStore, } from './job'; +export { JOB_WORKER_MARKER } from './front-mcp/detached-job.flag'; // Job-management tool classes (issue #408 — opt-in manual registration via // @App({ tools: [...] }) when finer-grained control than auto-registration diff --git a/libs/sdk/src/job/__tests__/job-detached.spec.ts b/libs/sdk/src/job/__tests__/job-detached.spec.ts new file mode 100644 index 000000000..7861b62ac --- /dev/null +++ b/libs/sdk/src/job/__tests__/job-detached.spec.ts @@ -0,0 +1,602 @@ +/** + * Detached job runs (proposal 2.2 b): a run that lives in a process of its own, keeps its state in the job + * store, survives the server that started it, and is read, cancelled and reconciled by any server on that store. + * The runner's entry runs in this process here, as the second instance a spawned runner would boot. + */ +import 'reflect-metadata'; + +import { spawnSync } from 'child_process'; +import { EventEmitter } from 'events'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +import { z } from '@frontmcp/lazy-zod'; +import { resetRuntimeContext } from '@frontmcp/utils'; + +import { completionEventsOf } from '../../channel/sources/completion-events'; +import { type JobCompletionEvent } from '../../channel/sources/job-completion.source'; +import { + App, + Channel, + ChannelContext, + Job, + JobContext, + LogLevel, + type ChannelNotification, + type JobPermission, +} from '../../common'; +import { type DirectAuthContext, type DirectMcpServer } from '../../direct/direct.types'; +import { JobRunnerUnavailableError } from '../../errors'; +import { FrontMcpInstance } from '../../front-mcp/front-mcp'; +import { type Scope } from '../../scope/scope.instance'; +import { JobExecutionManager } from '../execution/job-execution.manager'; +import { runDetachedJob } from '../runner/detached-job.worker'; +import { spawnDetachedRunner } from '../runner/job-runner'; +import { createJobStateStore } from '../store/job-state-store.factory'; +import { type JobRunRecord, type JobStateStore } from '../store/job-state.interface'; +import { StorageJobStateStore } from '../store/storage-job-state.store'; + +// A runner of this file's servers would run jest again: every spawn is a stand-in +jest.mock('../runner/job-runner', () => ({ + ...jest.requireActual('../runner/job-runner'), + spawnDetachedRunner: jest.fn(() => ({ pid: process.pid, once: () => undefined })), +})); + +const events: string[] = []; +let openGate: () => void = () => undefined; +let gate: Promise = Promise.resolve(); + +function closeGate(): void { + gate = new Promise((resolve) => { + openGate = resolve; + }); +} + +const OPEN: JobPermission[] = [{ action: 'read' }, { action: 'delete' }]; + +@Job({ + name: 'count_job', + inputSchema: { to: z.number() }, + outputSchema: { counted: z.number() }, + runner: 'detached', + timeout: 0, + permissions: OPEN, +}) +class CountJob extends JobContext { + async execute({ to }: { to: number }) { + for (let step = 1; step <= to; step++) await this.progress(step, to, `step ${step}`); + await gate; + return { counted: to }; + } +} + +@Job({ + name: 'wait_job', + inputSchema: { tag: z.string() }, + outputSchema: {}, + runner: 'detached', + timeout: 0, + permissions: OPEN, + cleanup: (job, reason) => { + events.push(`${(job.input as { tag: string }).tag}:cleanup:${reason}`); + }, +}) +class WaitJob extends JobContext { + async execute({ tag }: { tag: string }): Promise> { + events.push(`${tag}:started`); + await new Promise((resolve) => this.signal?.addEventListener('abort', resolve, { once: true })); + return {}; + } +} + +@App({ id: 'detached', name: 'Detached', jobs: [CountJob, WaitJob] }) +class DetachedApp {} + +const connections: string[] = []; + +@Channel({ name: 'inbox', source: { type: 'service', service: 'inbox' } }) +class InboxChannel extends ChannelContext { + override async onConnect(): Promise { + connections.push(`connect:${process.pid}`); + } + + async onEvent(payload: unknown): Promise { + return { content: String(payload) }; + } +} + +@App({ id: 'inbox', name: 'Inbox', channels: [InboxChannel] }) +class InboxApp {} + +const alice: DirectAuthContext = { sessionId: 'alice-session', user: { sub: 'alice' } }; +const bob: DirectAuthContext = { sessionId: 'bob-session', user: { sub: 'bob' } }; + +let dir: string; +let dbPath: string; +const workers: Promise[] = []; +const servers: DirectMcpServer[] = []; + +function config() { + return { + info: { name: 'detached-jobs', version: '1.0.0' }, + apps: [DetachedApp], + logging: { level: LogLevel.Off }, + sqlite: { path: dbPath }, + }; +} + +async function open(): Promise { + const server = await FrontMcpInstance.createDirect(config()); + servers.push(server); + return server; +} + +function scopeOf(server: DirectMcpServer): Scope { + return (server as unknown as { scope: Scope }).scope; +} + +function managerOf(server: DirectMcpServer): JobExecutionManager { + const manager = (scopeOf(server) as unknown as { _jobExecutionManager?: JobExecutionManager })._jobExecutionManager; + if (!manager) throw new Error('jobs are not enabled'); + return manager; +} + +function storeOf(server: DirectMcpServer): JobStateStore { + const store = (scopeOf(server) as unknown as { _jobStateStore?: JobStateStore })._jobStateStore; + if (!store) throw new Error('jobs are not enabled'); + return store; +} + +function jobOf(server: DirectMcpServer, name: string) { + const job = scopeOf(server).jobs?.findByName(name); + if (!job) throw new Error(`${name} is not registered`); + return job; +} + +/** A manager on the server's scope whose detached runs start the runner's entry in this process. */ +function starterOn(server: DirectMcpServer): JobExecutionManager { + const scope = scopeOf(server); + return new JobExecutionManager(storeOf(server), scope.logger, undefined, scope, { + findJob: (name) => scope.jobs?.findByName(name), + spawnDetached: (runId) => { + workers.push(runDetachedJob(config(), runId)); + return { pid: process.pid, once: () => undefined }; + }, + }); +} + +async function startDetached(server: DirectMcpServer, name: string, input: Record) { + const starter = starterOn(server); + const handle = await starter.executeJob(jobOf(server, name), input, { + sessionId: 'alice-session', + authInfo: { sessionId: 'alice-session', user: { sub: 'alice' } }, + }); + return { starter, handle }; +} + +async function waitFor(check: () => boolean | Promise, what: string, timeoutMs = 10_000): Promise { + const deadline = Date.now() + timeoutMs; + while (!(await check())) { + if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); + await new Promise((resolve) => setTimeout(resolve, 20)); + } +} + +async function stateOf(server: DirectMcpServer, runId: string): Promise { + return storeOf(server).getRun(runId); +} + +function deadPid(): number { + const exited = spawnSync(process.execPath, ['-e', '']); + return exited.pid; +} + +function orphan(runId: string, scopeId: string, overrides: Partial = {}): JobRunRecord { + return { + runId, + jobId: 'wait_job', + jobName: 'wait_job', + sessionId: 'alice-session', + ownerSub: 'alice', + state: 'running', + input: { tag: runId }, + startedAt: Date.now(), + attempt: 1, + maxAttempts: 1, + logs: [], + background: true, + scopeId, + runner: { kind: 'detached', id: `runner-${runId}`, pid: deadPid(), hostname: os.hostname() }, + ...overrides, + }; +} + +beforeEach(() => { + events.length = 0; + gate = Promise.resolve(); + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sdk-detached-jobs-')); + dbPath = path.join(dir, 'jobs.sqlite'); +}); + +afterEach(async () => { + openGate(); + await Promise.allSettled(workers.splice(0)); + for (const server of servers.splice(0)) await server.dispose().catch(() => undefined); + fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe('a detached run', () => { + it('answers at once, runs in its runner and lands its result in the store after the starting server is gone', async () => { + closeGate(); + const first = await open(); + const { starter, handle } = await startDetached(first, 'count_job', { to: 3 }); + + expect(handle).toEqual({ runId: expect.any(String), state: 'pending' }); + await waitFor(async () => (await stateOf(first, handle.runId))?.progress?.progress === 3, 'the progress to land'); + expect(await stateOf(first, handle.runId)).toEqual( + expect.objectContaining({ + state: 'running', + progress: { progress: 3, total: 3, message: 'step 3' }, + runner: { kind: 'detached', id: expect.any(String), pid: process.pid, hostname: os.hostname() }, + }), + ); + + await starter.dispose(); + await first.dispose(); + openGate(); + expect(await workers[0]).toBe(0); + + const second = await open(); + const status = await second.callTool('get_job_status', { runId: handle.runId }, { authContext: alice }); + expect(status.structuredContent).toEqual(expect.objectContaining({ state: 'completed', result: { counted: 3 } })); + }); + + it('is read and cancelled by a new server instance; its runner sees the cancel and runs the cleanup', async () => { + const first = await open(); + const { starter, handle } = await startDetached(first, 'wait_job', { tag: 'c1' }); + await waitFor(() => events.includes('c1:started'), 'the job to start'); + await starter.dispose(); + await first.dispose(); + + const second = await open(); + const completions: JobCompletionEvent[] = []; + completionEventsOf(scopeOf(second)).jobs.subscribe((event) => completions.push(event)); + const read = await second.readResource(`job://runs/${handle.runId}`, { authContext: bob }); + expect(JSON.parse(String((read.contents[0] as { text: string }).text))).toEqual( + expect.objectContaining({ runId: handle.runId, state: 'running' }), + ); + + const cancelled = await second.callTool('cancel_job', { runId: handle.runId }, { authContext: bob }); + + expect(cancelled.structuredContent).toEqual({ runId: handle.runId, state: 'cancelled' }); + await waitFor(() => events.includes('c1:cleanup:cancel'), 'the runner to run the cleanup'); + expect(await workers[0]).toBe(0); + expect((await stateOf(second, handle.runId))?.state).toBe('cancelled'); + await waitFor(() => completions.some((event) => event.status === 'cancelled'), 'the watching server to publish'); + }); + + it('writes the progress it reports at most every 500 ms, then its last progress with its outcome', async () => { + const updates = jest.spyOn(StorageJobStateStore.prototype, 'updateRun'); + try { + const server = await open(); + const { handle } = await startDetached(server, 'count_job', { to: 200 }); + expect(await workers[0]).toBe(0); + + const progressWrites = updates.mock.calls.filter(([runId, update]) => runId === handle.runId && update.progress); + expect(progressWrites.length).toBeLessThanOrEqual(2); + expect(await stateOf(server, handle.runId)).toEqual( + expect.objectContaining({ state: 'completed', progress: { progress: 200, total: 200, message: 'step 200' } }), + ); + } finally { + updates.mockRestore(); + } + }); + + it('keeps the outcome another process wrote: its attempt ending later does not overwrite it', async () => { + closeGate(); + const server = await open(); + const { handle } = await startDetached(server, 'count_job', { to: 1 }); + await waitFor(async () => (await stateOf(server, handle.runId))?.state === 'running', 'the run to start'); + + await storeOf(server).updateRun(handle.runId, { state: 'interrupted', completedAt: Date.now() }); + openGate(); + + expect(await workers[0]).toBe(0); + expect((await stateOf(server, handle.runId))?.state).toBe('interrupted'); + }); + + it('fails with JOB_RUNNER_BOOT_FAILED, once, when its process cannot start', async () => { + const server = await open(); + const scope = scopeOf(server); + const notify = jest.fn(async (_data: Record) => undefined); + const failing = new EventEmitter(); + const starter = new JobExecutionManager(storeOf(server), scope.logger, notify, scope, { + findJob: (name) => scope.jobs?.findByName(name), + spawnDetached: () => { + setImmediate(() => failing.emit('error', new Error('spawn node ENOENT'))); + return failing; + }, + }); + const handle = await starter.executeJob(jobOf(server, 'count_job'), { to: 1 }, {}); + + await waitFor(async () => (await stateOf(server, handle.runId))?.state === 'failed', 'the run to fail'); + expect((await stateOf(server, handle.runId))?.error).toEqual( + expect.objectContaining({ name: 'JobRunnerBootError', message: expect.stringContaining('spawn node ENOENT') }), + ); + await new Promise((resolve) => setTimeout(resolve, 1_500)); + expect((await stateOf(server, handle.runId))?.error?.message).toContain('spawn node ENOENT'); + expect(notify.mock.calls.filter(([data]) => data['state'] === 'failed')).toHaveLength(1); + await starter.dispose(); + }); + + it('is refused by an in-process server, which has no entry a runner can run again', async () => { + const server = await open(); + + const refusal = server.callTool('execute_job', { name: 'count_job', input: { to: 1 } }, { authContext: alice }); + + await expect(refusal).rejects.toBeInstanceOf(JobRunnerUnavailableError); + await expect(refusal).rejects.toHaveProperty('remedy.say', expect.stringMatching(/entry/)); + }); + + // The first CLI-mode server of a test run imports the CLI-only modules, which takes seconds under coverage + it('is refused by a CLI-mode server, whose command (` job run`) a runner would only run again', async () => { + const cli = await FrontMcpInstance.createForCli(config()); + try { + const scope = cli.getPrimaryScope() as Scope; + const manager = (scope as unknown as { _jobExecutionManager?: JobExecutionManager })._jobExecutionManager; + const job = scope.jobs?.findByName('count_job'); + if (!manager || !job) throw new Error('jobs are not enabled'); + jest.mocked(spawnDetachedRunner).mockClear(); + + await expect(manager.executeJob(job, { to: 1 }, {})).rejects.toBeInstanceOf(JobRunnerUnavailableError); + expect(spawnDetachedRunner).not.toHaveBeenCalled(); + } finally { + await cli.shutdown(); + } + }, 30_000); + + it('stops within its cancel poll when another process ended its run, such as a runner taken for gone', async () => { + const server = await open(); + const { handle } = await startDetached(server, 'wait_job', { tag: 'z1' }); + await waitFor(() => events.includes('z1:started'), 'the job to start'); + + await storeOf(server).updateRun(handle.runId, { state: 'interrupted', completedAt: Date.now() }); + const stopped = await Promise.race([ + workers[0].then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 3_000)), + ]); + if (!stopped) await storeOf(server).requestCancel?.(handle.runId); + + expect(stopped).toBe(true); + expect((await stateOf(server, handle.runId))?.state).toBe('interrupted'); + }); +}); + +describe('a detached job at boot', () => { + afterEach(() => { + delete process.env['FRONTMCP_DEPLOYMENT_MODE']; + resetRuntimeContext(); + }); + + it('fails the start of a serverless deployment, with a remedy, and closes the job store it opened', async () => { + process.env['FRONTMCP_DEPLOYMENT_MODE'] = 'serverless'; + resetRuntimeContext(); + const dispose = jest.spyOn(StorageJobStateStore.prototype, 'dispose'); + try { + const start = FrontMcpInstance.createDirect(config()); + + await expect(start).rejects.toBeInstanceOf(JobRunnerUnavailableError); + await expect(start).rejects.toHaveProperty('code', 'JOB_RUNNER_UNAVAILABLE'); + expect(dispose).toHaveBeenCalled(); + } finally { + dispose.mockRestore(); + } + }); +}); + +describe('the detached runner entry', () => { + it('writes a typed failure to the store when it cannot boot', async () => { + const server = await open(); + const record = orphan('boot-1', scopeOf(server).id, { state: 'pending' }); + await storeOf(server).createRun(record); + + const code = await runDetachedJob(config(), 'boot-1', async () => { + throw new Error('the database is down'); + }); + + expect(code).toBe(1); + expect(await stateOf(server, 'boot-1')).toEqual( + expect.objectContaining({ + state: 'failed', + error: expect.objectContaining({ + name: 'JobRunnerBootError', + message: expect.stringContaining('the database is down'), + }), + }), + ); + }); + + it('leaves an unknown or finished run alone', async () => { + const server = await open(); + await storeOf(server).createRun(orphan('done-1', scopeOf(server).id, { state: 'completed' })); + + expect(await runDetachedJob(config(), 'no-such-run')).toBe(0); + expect(await runDetachedJob(config(), 'done-1')).toBe(0); + expect((await stateOf(server, 'done-1'))?.state).toBe('completed'); + }); + + it('runs its run instead of serving when the entry calls runUnixSocket(), as a daemon does', async () => { + const exit = jest.spyOn(process, 'exit').mockImplementation((() => undefined) as never); + const socketPath = path.join(dir, 'daemon.sock'); + process.env['FRONTMCP_RUN_JOB_ID'] = 'no-such-run'; + let handle: { close: () => Promise } | undefined; + try { + handle = await FrontMcpInstance.runUnixSocket({ ...config(), socketPath }); + + await waitFor(() => exit.mock.calls.length > 0, 'the runner to exit', 3_000); + expect(exit).toHaveBeenCalledWith(0); + expect(fs.existsSync(socketPath)).toBe(false); + } finally { + delete process.env['FRONTMCP_RUN_JOB_ID']; + await handle?.close(); + exit.mockRestore(); + } + }); + + it('takes FRONTMCP_RUN_JOB_ID out of its environment, so no process its job starts boots as a runner', async () => { + process.env['FRONTMCP_RUN_JOB_ID'] = 'no-such-run'; + try { + expect(await runDetachedJob(config(), 'no-such-run')).toBe(0); + expect(process.env['FRONTMCP_RUN_JOB_ID']).toBeUndefined(); + } finally { + delete process.env['FRONTMCP_RUN_JOB_ID']; + } + }); + + it('connects no service channel: the server it was started by already listens', async () => { + connections.length = 0; + const { store } = await createJobStateStore({ sqlite: { path: dbPath } }); + const run = { state: 'pending' as const, jobName: 'count_job', input: { to: 1 }, scopeId: undefined }; + await store.createRun(orphan('quiet-1', '', run)); + await store.dispose(); + + const code = await runDetachedJob( + { ...config(), apps: [DetachedApp, InboxApp], channels: { enabled: true } }, + 'quiet-1', + ); + + expect(code).toBe(0); + expect(connections).toEqual([]); + }); + + it('fails a run whose job the server no longer has', async () => { + const server = await open(); + await storeOf(server).createRun(orphan('gone-1', scopeOf(server).id, { state: 'pending', jobName: 'gone_job' })); + + expect(await runDetachedJob(config(), 'gone-1')).toBe(0); + expect(await stateOf(server, 'gone-1')).toEqual( + expect.objectContaining({ + state: 'failed', + error: expect.objectContaining({ name: 'JobRunnerBootError', message: expect.stringContaining('gone_job') }), + }), + ); + }); +}); + +describe('reconcile on boot', () => { + it('interrupts a run whose runner died and runs its cleanup once, however many servers reconcile it', async () => { + const first = await open(); + await storeOf(first).createRun(orphan('dead-1', scopeOf(first).id)); + + const [second, third] = await Promise.all([open(), open()]); + await Promise.all([managerOf(first).reconcile(), managerOf(second).reconcile(), managerOf(third).reconcile()]); + await waitFor(() => events.includes('dead-1:cleanup:interrupt'), 'the cleanup'); + + expect(events.filter((event) => event === 'dead-1:cleanup:interrupt')).toHaveLength(1); + expect(await stateOf(first, 'dead-1')).toEqual( + expect.objectContaining({ + state: 'interrupted', + error: expect.objectContaining({ + name: 'JobInterruptedError', + message: expect.stringMatching(/runner is gone/), + }), + }), + ); + }); + + it('fails, without a cleanup, a detached run whose runner never took it over', async () => { + const first = await open(); + await storeOf(first).createRun(orphan('never-1', scopeOf(first).id, { state: 'pending' })); + + await managerOf(first).reconcile(); + + expect(await stateOf(first, 'never-1')).toEqual( + expect.objectContaining({ state: 'failed', error: expect.objectContaining({ name: 'JobRunnerBootError' }) }), + ); + expect(events).toEqual([]); + }); + + it("judges another host's runner by its heartbeat, and watches a live detached run until it ends", async () => { + const first = await open(); + const store = storeOf(first); + const scopeId = scopeOf(first).id; + const elsewhere = (id: string) => ({ kind: 'detached' as const, id, pid: 1, hostname: 'another-host' }); + await store.createRun(orphan('remote-gone', scopeId, { runner: elsewhere('lapsed') })); + await store.createRun(orphan('remote-live', scopeId, { runner: elsewhere('beating') })); + await store.touchRunner?.('beating', 60); + + const second = await open(); + const completions: JobCompletionEvent[] = []; + completionEventsOf(scopeOf(second)).jobs.subscribe((event) => completions.push(event)); + await managerOf(second).reconcile(); + + await waitFor(async () => (await stateOf(second, 'remote-gone'))?.state === 'interrupted', 'the lapsed run'); + expect((await stateOf(second, 'remote-live'))?.state).toBe('running'); + await store.updateRun('remote-live', { state: 'completed', result: {}, completedAt: Date.now() }); + await waitFor( + () => completions.some((event) => event.status === 'success' && event.jobName === 'wait_job'), + 'the watching server to publish the outcome', + ); + }); + + it('keeps a runner on this host alive while its pid runs, though its heartbeat lapsed (a busy event loop)', async () => { + const first = await open(); + const busy = { kind: 'detached' as const, id: 'no-heartbeat', pid: process.pid, hostname: os.hostname() }; + await storeOf(first).createRun(orphan('busy-1', scopeOf(first).id, { runner: busy })); + + await managerOf(first).reconcile(); + + expect((await stateOf(first, 'busy-1'))?.state).toBe('running'); + expect(events).toEqual([]); + }); + + it('judges an orphan by its record as it is when it ends it, not as the listing found it', async () => { + const first = await open(); + const store = storeOf(first); + const listed = orphan('moved-1', scopeOf(first).id, { state: 'pending' }); + await store.createRun({ ...listed, state: 'running' }); + jest.spyOn(store, 'listRuns').mockResolvedValueOnce([listed]); + + await managerOf(first).reconcile(); + + expect(await stateOf(first, 'moved-1')).toEqual( + expect.objectContaining({ + state: 'interrupted', + error: expect.objectContaining({ name: 'JobInterruptedError' }), + }), + ); + expect(events).toEqual(['moved-1:cleanup:interrupt']); + }); + + it("ends a run whose runner had this server's pid before this server started, as after a container restart", async () => { + const first = await open(); + const earlier = { kind: 'inline' as const, id: 'old-server', pid: process.pid, hostname: os.hostname() }; + const startedAt = Math.floor(performance.timeOrigin) - 60_000; + await storeOf(first).createRun(orphan('reused-1', scopeOf(first).id, { runner: earlier, startedAt })); + + await managerOf(first).reconcile(); + + expect((await stateOf(first, 'reused-1'))?.state).toBe('interrupted'); + expect(events).toEqual(['reused-1:cleanup:interrupt']); + }); + + it('leaves the inline runs of a live server and the runs of an older release alone', async () => { + const first = await open(); + const store = storeOf(first); + const scopeId = scopeOf(first).id; + await store.createRun( + orphan('inline-live', scopeId, { + runner: { kind: 'inline', id: 'server-2', pid: process.pid, hostname: os.hostname() }, + }), + ); + await store.touchRunner?.('server-2', 60); + await store.createRun(orphan('legacy', scopeId, { runner: undefined })); + + await managerOf(first).reconcile(); + + expect((await stateOf(first, 'inline-live'))?.state).toBe('running'); + expect((await stateOf(first, 'legacy'))?.state).toBe('running'); + expect(events).toEqual([]); + }); +}); diff --git a/libs/sdk/src/job/execution/job-execution.manager.ts b/libs/sdk/src/job/execution/job-execution.manager.ts index e6cf1d109..2f92e9d13 100644 --- a/libs/sdk/src/job/execution/job-execution.manager.ts +++ b/libs/sdk/src/job/execution/job-execution.manager.ts @@ -1,10 +1,10 @@ -import { randomUUID, runRequestExclusive } from '@frontmcp/utils'; +import { getHostname, randomUUID, runRequestExclusive } from '@frontmcp/utils'; import { type ScopeEntry } from '../../common'; import { type JobEntry } from '../../common/entries/job.entry'; import { type WorkflowEntry } from '../../common/entries/workflow.entry'; import { type FrontMcpLogger } from '../../common/interfaces/logger.interface'; -import { type JobPermission } from '../../common/metadata/job.metadata'; +import { type JobInterruptReason, type JobPermission, type JobRunner } from '../../common/metadata/job.metadata'; import { resolvePrincipal } from '../../common/utils/principal.utils'; import { FrontMcpContext } from '../../context'; import { @@ -12,30 +12,37 @@ import { JobInterruptedError, JobNotAuthorizedError, JobRunFinishedError, + JobRunnerBootError, + JobRunnerUnavailableError, JobRunNotCancellableError, JobRunNotFoundError, JobTimeoutError, } from '../../errors'; import { outlasts } from '../../provider/provider.lifecycle'; +import { isAlive } from '../../task/helpers/process-liveness'; import { WorkflowEngine } from '../../workflow/engine/workflow.engine'; import { type JobAdmission } from '../job-admission'; import { abortableDelay, retryDelayMs, runJobAttemptFlow, + runJobCleanup, toJobError, willRetryJobAttempt, type JobAttemptOutcome, type JobAttemptResult, type JobRunRecorder, } from '../job-attempt'; -import { detachedRunContext } from '../job-context-providers'; +import { detachedRunContext, jobContextProviders } from '../job-context-providers'; import { JobPermissionGuard } from '../job-permission.guard'; import { jobRunUri } from '../job-run-access'; import { type JobRegistryInterface } from '../job.registry'; +import { runnerOf, spawnDetachedRunner, type DetachedRunnerProcess } from '../runner/job-runner'; +import { LatestValueFlusher, RunnerHeartbeat } from '../runner/run-flusher'; import { FINISHED_JOB_STATES, type JobExecutionState, + type JobRunnerInfo, type JobRunProgress, type JobRunRecord, type JobStateStore, @@ -45,6 +52,15 @@ import { /** How long an attempt of a job that sets no `timeout` may run. */ export const DEFAULT_JOB_TIMEOUT_MS = 300_000; +/** How often a detached run writes the progress it reported, and reads the store for a cancel. */ +export const DETACHED_FLUSH_MS = 500; + +/** How long a detached runner may take to boot before a missing heartbeat means it is gone. */ +const DETACHED_BOOT_TTL_SECONDS = 60; + +/** How often a server reads a detached run it watches. */ +const DETACHED_WATCH_MS = 1_000; + export interface JobExecutionManagerOptions { /** How long an attempt of a job that sets no `timeout` may run, in ms; 0 for no limit. @default 300000 */ defaultTimeoutMs?: number; @@ -52,6 +68,14 @@ export interface JobExecutionManagerOptions { admission?: JobAdmission; /** How often an active run reads the store for a cancel another process wrote, in ms; unset never reads it. */ cancelPollMs?: number; + /** Where the attempts of a job that sets no `runner` run (`jobs.runner`). @default 'inline' */ + runner?: JobRunner; + /** The job of a run by its name, for a detached run's runner and the cleanup of a run whose runner is gone. */ + findJob?: (name: string) => JobEntry | undefined; + /** Why this server cannot start a detached run, and what to do instead; unset when it can. */ + detachedUnavailable?: { why: string; remedy: string }; + /** Starts the process of a detached run. @default a process running the server's own command again */ + spawnDetached?: (runId: string) => DetachedRunnerProcess; } /** A run this manager is executing: aborting its controller stops the run, and `settled` resolves once it ended. */ @@ -109,6 +133,12 @@ export class JobExecutionManager { private readonly activeRuns = new Map(); /** The last write this process queued for each run, so the next one waits for it. */ private readonly runWrites = new Map>(); + /** This process as the runner of its inline and worker runs, alive in the store while it has any. */ + private readonly heartbeat: RunnerHeartbeat; + /** The detached runs this server watches, by run. */ + private readonly watchers = new Map>(); + /** The progress writes of the detached runs this process runs, at most one per {@link DETACHED_FLUSH_MS}. */ + private readonly progressFlushers = new Map>(); /** * @param scope - The scope the manager serves. A background run without a caller context gets a @@ -127,6 +157,9 @@ export class JobExecutionManager { this.notifyFn = notifyFn; this.scope = scope; this.options = options; + this.heartbeat = new RunnerHeartbeat(stateStore, randomUUID(), (error) => + this.logger.warn(`Could not renew this server's job heartbeat: ${toJobError(error).message}`), + ); } /** @@ -149,6 +182,7 @@ export class JobExecutionManager { const runId = randomUUID(); const retryConfig = job.metadata.retry ?? {}; const maxAttempts = retryConfig.maxAttempts ?? 1; + const runner = runnerOf(job, this.options.runner); const runRecord: JobRunRecord = { runId, @@ -162,31 +196,47 @@ export class JobExecutionManager { attempt: 1, maxAttempts, logs: [], - background: opts.background ?? false, + background: runner === 'detached' || (opts.background ?? false), + scopeId: this.scope?.id, }; - await this.stateStore.createRun(runRecord); + if (runner === 'detached') return this.startDetached(job, runRecord); + + // Alive in the store before the run is, so a server reconciling the store never takes it for an orphan + const release = await this.heartbeat.hold(); + try { + await this.stateStore.createRun({ ...runRecord, runner: this.ownRunner(runner) }); + } catch (error) { + release(); + throw error; + } if (opts.background) { const runOpts = withBackgroundContext(opts, (this.scope ?? job.providers.getActiveScope()).id); // Spawn background execution, as its own request (it outlives this one; a no-op on Node) - runRequestExclusive(() => this.executeJobBackground(job, input, runId, runOpts)).catch(async (err) => { - this.logger.error(`Background job execution failed: ${err}`); - try { - await this.updateState(runId, { - state: 'failed', - error: { message: err?.message ?? String(err), name: err?.name ?? 'Error' }, - completedAt: Date.now(), - }); - } catch (updateErr) { - this.logger.error(`Failed to update run state after error: ${updateErr}`); - } - }); + void runRequestExclusive(() => this.executeJobBackground(job, input, runId, runOpts)) + .catch(async (err) => { + this.logger.error(`Background job execution failed: ${err}`); + try { + await this.updateState(runId, { + state: 'failed', + error: { message: err?.message ?? String(err), name: err?.name ?? 'Error' }, + completedAt: Date.now(), + }); + } catch (updateErr) { + this.logger.error(`Failed to update run state after error: ${updateErr}`); + } + }) + .finally(release); return { runId, state: 'pending' }; } // Inline execution - return this.executeJobInline(job, input, runId, opts); + try { + return await this.executeJobInline(job, input, runId, opts); + } finally { + release(); + } } /** @@ -301,6 +351,9 @@ export class JobExecutionManager { * for them at most that long. */ async dispose(timeoutMs?: number): Promise { + // The detached runs keep running in their own processes + for (const watch of this.watchers.values()) clearInterval(watch); + this.watchers.clear(); const runs = [...this.activeRuns.entries()]; for (const [runId, run] of runs) run.controller.abort(new JobInterruptedError(runId)); const ended = () => Promise.all(runs.map(([, run]) => run.settled)); @@ -311,6 +364,54 @@ export class JobExecutionManager { } } + /** + * Settle the job runs a dead runner left unfinished in the store, and watch the detached runs whose runner is + * alive. A run whose runner is gone (its pid no longer runs on this host, or its heartbeat expired in the store) + * ends `interrupted` and its job's cleanup runs here, or `failed` with `JobRunnerBootError` when its detached + * runner never took it over. One process settles each run, however many reconcile at once. + */ + async reconcile(): Promise { + for (const record of await this.stateStore.listRuns()) { + if (FINISHED_JOB_STATES.has(record.state) || !record.runner) continue; + if (this.scope && record.scopeId !== undefined && record.scopeId !== this.scope.id) continue; + if (await this.runnerGone(record.runner, record.startedAt)) await this.endOrphan(record); + else if (record.runner.kind === 'detached') this.watch(record.runId); + } + } + + /** + * Run a detached run in this process, which becomes its runner: its pid goes on the record, its heartbeat in + * the store, its attempts run as an inline run's do, and its progress is written every 500 ms at most. A cancel + * written to the store is read every 500 ms. An unknown or finished run is left alone. + */ + async runDetached(runId: string): Promise { + const record = await this.stateStore.getRun(runId); + if (!record?.runner || FINISHED_JOB_STATES.has(record.state)) return; + const job = this.options.findJob?.(record.jobName); + if (!job) { + await this.failToBoot(record, `the server has no job named "${record.jobName}"`); + return; + } + const warn = (error: unknown) => this.logger.warn(`Detached run ${runId}: ${toJobError(error).message}`); + const release = await new RunnerHeartbeat(this.stateStore, record.runner.id, warn).hold(); + const flusher = new LatestValueFlusher( + (progress) => this.updateState(runId, { progress }), + DETACHED_FLUSH_MS, + warn, + ); + this.progressFlushers.set(runId, flusher); + try { + await this.updateState(runId, { runner: { ...record.runner, pid: currentPid(), hostname: getHostname() } }); + const scopeId = this.scope?.id ?? job.providers.getActiveScope().id; + const opts = withBackgroundContext({ sessionId: record.sessionId, authInfo: {} }, scopeId); + await this.executeJobInline(job, record.input, runId, opts).catch(() => undefined); + } finally { + flusher.stop(); + this.progressFlushers.delete(runId); + release(); + } + } + /** * List runs with optional filters. */ @@ -323,6 +424,148 @@ export class JobExecutionManager { return this.stateStore.listRuns(opts); } + // ---- Private: detached runs ---- + + /** + * Start a detached run: a heartbeat that lasts while its process boots, the process, then the run record with + * that process as its runner. Answers at once; this server watches the run until it ends. + */ + private async startDetached(job: JobEntry, record: JobRunRecord): Promise { + const unavailable = this.options.detachedUnavailable; + if (unavailable) throw new JobRunnerUnavailableError(job.name, unavailable.why, unavailable.remedy); + const runner: JobRunnerInfo = { kind: 'detached', id: randomUUID(), pid: 0, hostname: getHostname() }; + await this.stateStore.touchRunner?.(runner.id, DETACHED_BOOT_TTL_SECONDS); + const child = (this.options.spawnDetached ?? spawnDetachedRunner)(record.runId); + const created = this.stateStore.createRun({ ...record, runner: { ...runner, pid: child.pid ?? 0 } }); + child.once('error', (error) => { + void created.then( + () => this.failToBoot(record, error.message), + () => undefined, + ); + }); + await created; + this.watch(record.runId); + return { runId: record.runId, state: 'pending' }; + } + + /** Read a detached run every second: tell its subscribers when it changed, and settle it if its runner is gone. */ + private watch(runId: string): void { + if (this.watchers.has(runId)) return; + let revision: number | undefined; + let reading = false; + const timer = setInterval(async () => { + if (reading) return; + reading = true; + try { + const record = await this.stateStore.getRun(runId); + if (record && record.revision !== revision) { + revision = record.revision; + this.scope?.resources.notifyContentChanged(jobRunUri(runId)); + } + if (!record || FINISHED_JOB_STATES.has(record.state)) { + this.unwatch(runId); + if (record) await this.notify({ type: 'job:status', runId, state: record.state, jobName: record.jobName }); + } else if (record.runner && (await this.runnerGone(record.runner, record.startedAt))) { + this.unwatch(runId); + await this.endOrphan(record); + } + } catch (error) { + this.logger.warn(`Could not read detached run ${runId}: ${toJobError(error).message}`); + } finally { + reading = false; + } + }, DETACHED_WATCH_MS); + timer.unref?.(); + this.watchers.set(runId, timer); + } + + private unwatch(runId: string): void { + clearInterval(this.watchers.get(runId)); + this.watchers.delete(runId); + } + + /** + * Whether the runner of a run started at `startedAt` is gone. On this host its pid decides: a runner whose event + * loop is busy misses heartbeats but is alive, and this process's own pid on a run older than this process was an + * earlier process's (a restarted container's). On another host the store's TTL does, and without heartbeats in + * the store it counts as alive. + */ + private async runnerGone(runner: JobRunnerInfo, startedAt: number): Promise { + if (runner.hostname !== undefined && runner.hostname === getHostname()) { + return runner.pid === currentPid() ? startedAt < performance.timeOrigin : !isAlive(runner.pid); + } + if (!this.stateStore.isRunnerAlive) return false; + return !(await this.stateStore.isRunnerAlive(runner.id)); + } + + /** End a run whose runner is gone, once across the processes sharing the store, and run its job's cleanup. */ + private async endOrphan(listed: JobRunRecord): Promise { + const { runId } = listed; + if (this.stateStore.claimRun && !(await this.stateStore.claimRun(runId))) return; + const record = (await this.stateStore.getRun(runId)) ?? listed; + const neverStarted = record.state === 'pending' && record.runner?.kind === 'detached'; + const error = neverStarted + ? new JobRunnerBootError(runId, 'its process exited before it took the run over') + : new JobInterruptedError(runId, 'its runner is gone'); + const state = await this.endRun(record, error); + if (state && record.state === 'running') { + await this.cleanupOrphan(record, state === 'cancelled' ? 'cancel' : 'interrupt'); + } + } + + /** + * End a run that is still unfinished with `error`, stop watching it, and notify; a cancel written first wins. + * Answers the state written, or undefined when the run had already ended. + */ + private async endRun(record: JobRunRecord, error: Error): Promise { + const { runId } = record; + const state = await this.withRunLock(runId, async () => { + const current = await this.stateStore.getRun(runId); + if (!current || FINISHED_JOB_STATES.has(current.state)) return undefined; + return this.writeOutcome(runId, { state: 'failed', attempt: current.attempt, error }); + }); + if (!state) return undefined; + this.unwatch(runId); + await this.notify({ type: 'job:status', runId, state, jobName: record.jobName }); + return state; + } + + /** Run the job's cleanup for a run whose runner is gone, on an instance built here for the run's input. */ + private async cleanupOrphan(record: JobRunRecord, reason: JobInterruptReason): Promise { + const job = this.options.findJob?.(record.jobName); + if (!job?.metadata.cleanup) return; + try { + const scopeId = this.scope?.id ?? job.providers.getActiveScope().id; + const { context } = withBackgroundContext( + { sessionId: record.sessionId, authInfo: {} }, + scopeId, + ); + const contextProviders = context ? await jobContextProviders(job, context) : undefined; + const instance = job.create(job.parseInput(record.input), { + authInfo: {}, + contextProviders, + attempt: record.attempt, + }); + await runJobCleanup(job, instance, reason, this.logger); + } catch (error) { + this.logger.warn(`The ${reason} cleanup of run ${record.runId} could not run: ${toJobError(error).message}`); + } + } + + /** End a detached run whose runner did not start, with `JobRunnerBootError`. */ + private async failToBoot(record: JobRunRecord, cause: string): Promise { + try { + await this.endRun(record, new JobRunnerBootError(record.runId, cause)); + } catch (error) { + this.logger.error(`Could not record that run ${record.runId} did not start: ${toJobError(error).message}`); + } + } + + /** This process as the runner of an inline or worker run. */ + private ownRunner(kind: JobRunner): JobRunnerInfo { + return { kind, id: this.heartbeat.runnerId, pid: currentPid(), hostname: getHostname() }; + } + // ---- Private: inline execution ---- private async executeJobInline( @@ -339,15 +582,16 @@ export class JobExecutionManager { } } - /** Track a run this process executes until `end()`: its abort controller, and its watch for a cancel in the store. */ + /** + * Track a run this process executes until `end()`: its abort controller, and its watch of the store for a cancel + * or for an end another process wrote (one that took this process for gone). + */ private startRun(runId: string): { signal: AbortSignal; end: () => void } { const controller = new AbortController(); let settle: () => void = () => undefined; const settled = new Promise((resolve) => (settle = resolve)); const { cancelPollMs } = this.options; - const watch = cancelPollMs - ? setInterval(() => void this.abortIfCancelled(runId, controller), cancelPollMs) - : undefined; + const watch = cancelPollMs ? setInterval(() => void this.abortIfEnded(runId, controller), cancelPollMs) : undefined; watch?.unref?.(); this.activeRuns.set(runId, { controller, settled }); const end = () => { @@ -358,11 +602,15 @@ export class JobExecutionManager { return { signal: controller.signal, end }; } - private async abortIfCancelled(runId: string, controller: AbortController): Promise { + private async abortIfEnded(runId: string, controller: AbortController): Promise { try { - if (await this.cancelRequested(runId)) controller.abort(new JobCancelledError(runId)); + if (await this.cancelRequested(runId)) { + controller.abort(new JobCancelledError(runId)); + } else if (FINISHED_JOB_STATES.has((await this.stateStore.getRun(runId))?.state ?? 'pending')) { + controller.abort(new JobInterruptedError(runId, 'another process ended it')); + } } catch (error) { - this.logger.warn(`Could not read whether run ${runId} was cancelled: ${toJobError(error).message}`); + this.logger.warn(`Could not read whether run ${runId} was cancelled or ended: ${toJobError(error).message}`); } } @@ -392,7 +640,7 @@ export class JobExecutionManager { runId, maxAttempts, (outcome) => this.recordAttempt(job, runId, outcome), - (progress) => this.updateState(runId, { progress }), + (progress) => this.recordProgress(runId, progress), ); // Each attempt is one run of the `jobs:execute-job` flow, which records how it ended (#700). @@ -437,21 +685,34 @@ export class JobExecutionManager { } } + /** Keep the progress a run reported: at once, or with the next flush of a detached run. */ + private async recordProgress(runId: string, progress: JobRunProgress): Promise { + const flusher = this.progressFlushers.get(runId); + if (flusher) flusher.push(progress); + else await this.updateState(runId, { progress }); + } + /** Write how an attempt ended to the run record, and send the run's status notification. */ private async recordAttempt(job: JobEntry, runId: string, outcome: JobAttemptOutcome): Promise { + await this.progressFlushers.get(runId)?.flush(); const state = await this.withRunLock(runId, () => this.writeOutcome(runId, outcome)); if (!state) return; const attempt = state === 'retrying' ? { attempt: outcome.attempt } : {}; await this.notify({ type: 'job:status', runId, state, jobName: job.name, ...attempt }); } - /** Write how an attempt ended, answering the state written; a cancel written to the store wins over it. */ + /** + * Write how an attempt ended, answering the state written; a cancel written to the store wins over it, and a + * run another process already ended (its runner taken for gone) keeps that end. + */ private async writeOutcome(runId: string, outcome: JobAttemptOutcome): Promise { if (await this.cancelRequested(runId)) { if (outcome.state === 'retrying') return undefined; await this.write(runId, { ...cancelledUpdate(new JobCancelledError(runId)), attempt: outcome.attempt }); return 'cancelled'; } + const current = await this.stateStore.getRun(runId); + if (current && FINISHED_JOB_STATES.has(current.state)) return undefined; switch (outcome.state) { case 'completed': await this.write(runId, { @@ -650,6 +911,11 @@ class RunRecorder implements JobRunRecorder { } } +/** This process's pid; 0 where there is no process. */ +function currentPid(): number { + return typeof process === 'undefined' ? 0 : process.pid; +} + /** What a cancel writes to the run record. */ function cancelledUpdate(reason: JobCancelledError): Partial { return { state: 'cancelled', error: { message: reason.message, name: reason.name }, completedAt: Date.now() }; diff --git a/libs/sdk/src/job/flows/execute-job.flow.ts b/libs/sdk/src/job/flows/execute-job.flow.ts index 7034c5261..3d34775a4 100644 --- a/libs/sdk/src/job/flows/execute-job.flow.ts +++ b/libs/sdk/src/job/flows/execute-job.flow.ts @@ -19,11 +19,11 @@ import { runOnSurface } from '../../context/call-surface'; import { InternalMcpError, JobNotAuthorizedError, JobTimeoutError } from '../../errors'; import { entryClassHooksJoin } from '../../hooks/entry-class-hooks'; import { hooksBoundTo } from '../../hooks/hooks.utils'; -import { outlasts } from '../../provider/provider.lifecycle'; import { type JobAdmissionSlot } from '../job-admission'; import { EXECUTE_JOB_FLOW, interruptReasonOf, + runJobCleanup, toJobError, untilAborted, willRetryJobAttempt, @@ -33,11 +33,9 @@ import { } from '../job-attempt'; import { jobContextProviders } from '../job-context-providers'; import { JobPermissionGuard } from '../job-permission.guard'; +import { runnerOf } from '../runner/job-runner'; import { type JobRunProgress } from '../store/job-state.interface'; -/** How long a job's `cleanup` may take when it sets no `timeoutMs`. */ -const DEFAULT_CLEANUP_TIMEOUT_MS = 3_000; - /** A job's input once its `inputSchema` parsed it. */ type JobInput = ReturnType; /** What a job's `execute()` returns, before its `outputSchema` parses it. */ @@ -241,7 +239,8 @@ export default class ExecuteJobFlow extends FlowBase { } /** - * Run the job's `execute()` on the `'job'` surface. A value passed to `this.respond()` is its result. + * Run the job's `execute()` on the `'job'` surface, in a worker thread of the offload pool for a job on the + * worker runner. A value passed to `this.respond()` is its result. * Past `ctx.timeoutMs` the attempt is aborted with `JobTimeoutError`. An attempt that is aborted ends * at once with the abort's reason, however the job ends; `execute()` itself is not stopped. */ @@ -253,10 +252,20 @@ export default class ExecuteJobFlow extends FlowBase { const timer = timeoutMs ? setTimeout(() => attemptAbort.abort(new JobTimeoutError(job.name, timeoutMs)), timeoutMs) : undefined; + const onWorker = runnerOf(job, this.scope.metadata.jobs?.runner) === 'worker'; let output: unknown; try { output = await untilAborted( - runOnSurface('job', async () => jobContext.execute(parsedInput)), + runOnSurface('job', async () => + onWorker + ? (await import('../../offload/run-job-in-worker.js')).runJobInWorker( + jobContext, + job, + parsedInput, + attemptAbort.signal, + ) + : jobContext.execute(parsedInput), + ), attemptAbort.signal, ); } catch (error) { @@ -286,17 +295,8 @@ export default class ExecuteJobFlow extends FlowBase { @Stage('cleanup') async cleanup() { const { job, jobContext, attemptAbort } = this.state; - const cleanup = job?.metadata.cleanup; - if (!cleanup || !jobContext || !attemptAbort?.signal.aborted) return; - const { run, timeoutMs = DEFAULT_CLEANUP_TIMEOUT_MS } = typeof cleanup === 'function' ? { run: cleanup } : cleanup; - const reason = interruptReasonOf(attemptAbort.signal.reason); - try { - if (await outlasts(() => run(jobContext, reason), timeoutMs)) { - this.logger.warn(`cleanup: the ${reason} cleanup did not finish within ${timeoutMs} ms`); - } - } catch (error) { - this.logger.warn(`cleanup: the ${reason} cleanup failed: ${toJobError(error).message}`); - } + if (!job || !jobContext || !attemptAbort?.signal.aborted) return; + await runJobCleanup(job, jobContext, interruptReasonOf(attemptAbort.signal.reason), this.logger); } /** diff --git a/libs/sdk/src/job/job-attempt.ts b/libs/sdk/src/job/job-attempt.ts index 7f3dc277f..729ba5a22 100644 --- a/libs/sdk/src/job/job-attempt.ts +++ b/libs/sdk/src/job/job-attempt.ts @@ -2,7 +2,8 @@ import type { AuthoritiesContextBuilder } from '@frontmcp/auth'; import { FlowControl, type ScopeEntry } from '../common'; import { type JobEntry } from '../common/entries/job.entry'; -import { type JobInterruptReason, type JobRetryConfig } from '../common/metadata/job.metadata'; +import { type FrontMcpLogger } from '../common/interfaces/logger.interface'; +import { type JobCleanup, type JobInterruptReason, type JobRetryConfig } from '../common/metadata/job.metadata'; import { FrontMcpContextStorage, type FrontMcpContext } from '../context'; import { InvalidOutputError, @@ -11,6 +12,7 @@ import { JobInterruptedError, JobNotAuthorizedError, } from '../errors'; +import { outlasts } from '../provider/provider.lifecycle'; import { type JobAdmission, type JobAdmissionSlot } from './job-admission'; import { type JobRunProgress } from './store/job-state.interface'; @@ -150,6 +152,31 @@ export function willRetryJobAttempt(error: unknown, attempt: number, maxAttempts return !FINAL_FAILURES.some((failure) => cause instanceof failure); } +/** How long a job's `cleanup` may take when it sets no `timeoutMs`. */ +const DEFAULT_CLEANUP_TIMEOUT_MS = 3_000; + +/** + * Run the job's `cleanup` on an attempt's instance, bounded by its own time limit; a throw or an overrun is + * logged and changes nothing else. + */ +export async function runJobCleanup( + job: JobEntry, + jobContext: Parameters[0], + reason: JobInterruptReason, + logger: FrontMcpLogger, +): Promise { + const cleanup = job.metadata.cleanup; + if (!cleanup) return; + const { run, timeoutMs = DEFAULT_CLEANUP_TIMEOUT_MS } = typeof cleanup === 'function' ? { run: cleanup } : cleanup; + try { + if (await outlasts(() => run(jobContext, reason), timeoutMs)) { + logger.warn(`cleanup: the ${reason} cleanup did not finish within ${timeoutMs} ms`); + } + } catch (error) { + logger.warn(`cleanup: the ${reason} cleanup failed: ${toJobError(error).message}`); + } +} + /** What the job's `cleanup` is told about why its attempt stopped, from the reason its signal was aborted with. */ export function interruptReasonOf(reason: unknown): JobInterruptReason { if (reason instanceof JobCancelledError) return 'cancel'; @@ -196,7 +223,7 @@ export function toJobError(error: unknown): Error { return error instanceof Error ? error : new Error(String(error)); } -function contextStorageOf(scope: ScopeEntry): FrontMcpContextStorage | undefined { +export function contextStorageOf(scope: ScopeEntry): FrontMcpContextStorage | undefined { try { return scope.providers.get(FrontMcpContextStorage); } catch { diff --git a/libs/sdk/src/job/job-scope.helper.ts b/libs/sdk/src/job/job-scope.helper.ts index 04a5c6361..9bd8fd49e 100644 --- a/libs/sdk/src/job/job-scope.helper.ts +++ b/libs/sdk/src/job/job-scope.helper.ts @@ -11,6 +11,7 @@ import { type EntryOwnerRef, type RedisOptionsInput } from '../common'; import { type JobType } from '../common/interfaces/job.interface'; import { type FrontMcpLogger } from '../common/interfaces/logger.interface'; import { type WorkflowType } from '../common/interfaces/workflow.interface'; +import { type JobRunner } from '../common/metadata/job.metadata'; import type FlowRegistry from '../flows/flow.registry'; import type ProviderRegistry from '../provider/provider.registry'; import ExecuteWorkflowTool from '../workflow/tools/execute-workflow.tool'; @@ -19,11 +20,12 @@ import ListWorkflowsTool from '../workflow/tools/list-workflows.tool'; import RegisterWorkflowTool from '../workflow/tools/register-workflow.tool'; import RemoveWorkflowTool from '../workflow/tools/remove-workflow.tool'; import WorkflowRegistry, { type WorkflowRegistryInterface } from '../workflow/workflow.registry'; -import { JobExecutionManager } from './execution/job-execution.manager'; +import { DETACHED_FLUSH_MS, JobExecutionManager } from './execution/job-execution.manager'; import ExecuteJobFlow from './flows/execute-job.flow'; import { JobAdmission, type JobAdmissionOptions } from './job-admission'; import JobRegistry, { type JobRegistryInterface } from './job.registry'; import { JobRunResource } from './resources/job-run.resource'; +import { assertRunnersAvailable } from './runner/job-runner'; import { createJobDefinitionStore } from './store/job-definition-store.factory'; import { type JobDefinitionStore } from './store/job-definition.interface'; import { createJobStateStore } from './store/job-state-store.factory'; @@ -56,6 +58,8 @@ export interface JobsConfig { defaultTimeoutMs?: number; /** When job attempts may start: at most `concurrency` at once, each claiming `memoryMb`, waiting up to `maxWaitMs`. */ admission?: JobAdmissionOptions; + /** Where the attempts of a job that sets no `runner` run. @default 'inline' */ + runner?: JobRunner; } /** `jobs.store`: Redis, or a SQLite file the processes on one machine share; memory when neither is set. */ @@ -100,6 +104,10 @@ export interface RegisterJobCapabilitiesArgs { jobsConfig: JobsConfig; logger: FrontMcpLogger; notifyFn?: (data: Record) => Promise; + /** The server was started by an entry a detached runner runs again (`bootstrap()`, `runStdio()`, `runUnixSocket()`). */ + entryServer?: boolean; + /** The scope was booted by a detached runner, which runs one run and reconciles nothing. */ + jobWorker?: boolean; } export interface JobCapabilitiesResult { @@ -117,6 +125,7 @@ export interface JobCapabilitiesResult { */ export async function registerJobCapabilities(args: RegisterJobCapabilitiesArgs): Promise { const { providers, flowRegistry, owner, jobsList, jobProviders, workflowsList, jobsConfig, logger, notifyFn } = args; + const { entryServer = false, jobWorker = false } = args; // Every attempt of every job runs through this hookable flow (#700) await flowRegistry.registryFlows([ExecuteJobFlow]); @@ -155,6 +164,15 @@ export async function registerJobCapabilities(args: RegisterJobCapabilitiesArgs) workflowRegistry.registerDynamic(dynWf); } + // A detached job is refused here, at boot, where its process could not outlive the server + try { + assertRunnersAvailable(jobRegistry.getJobs(), jobsConfig.runner); + } catch (error) { + await stateStore.dispose(); + await definitionStore.dispose(); + throw error; + } + // 4. Create execution manager. Its status notifications also publish finished job and workflow // runs to the scope's completions, which `job-completion` channel sources subscribe to. const scope = providers.getActiveScope(); @@ -171,7 +189,10 @@ export async function registerJobCapabilities(args: RegisterJobCapabilitiesArgs) defaultTimeoutMs: jobsConfig.defaultTimeoutMs, admission: jobsConfig.admission ? new JobAdmission(jobsConfig.admission) : undefined, // A store other processes share can carry a cancel this process must notice - cancelPollMs: stateStoreType === 'memory' ? undefined : CANCEL_POLL_MS, + cancelPollMs: stateStoreType === 'memory' ? undefined : jobWorker ? DETACHED_FLUSH_MS : CANCEL_POLL_MS, + runner: jobsConfig.runner, + findJob: (name) => jobRegistry.findByName(name), + detachedUnavailable: detachedUnavailable(stateStoreType, !entryServer && !jobWorker), }, ); // At shutdown the runs left are interrupted (their cleanup runs) before the stores close @@ -181,6 +202,12 @@ export async function registerJobCapabilities(args: RegisterJobCapabilitiesArgs) await definitionStore.dispose(); }); scope.resources.registerDynamicResource(JobRunResource); + // Runs a dead runner left in a shared store are settled in the background, once the registries exist + if (stateStoreType !== 'memory' && !jobWorker) { + void executionManager + .reconcile() + .catch((error: unknown) => logger.warn(`Could not reconcile the job runs in the store: ${String(error)}`)); + } // 5. Collect management tools. The dynamic-registration pair is omitted // unless the operator opted in — an unregistered tool cannot be called at @@ -219,6 +246,26 @@ export async function registerJobCapabilities(args: RegisterJobCapabilitiesArgs) }; } +/** Why a server cannot start detached runs, and what to do instead; undefined when it can. */ +function detachedUnavailable( + storeType: 'redis' | 'memory' | 'sqlite', + inProcess: boolean, +): { why: string; remedy: string } | undefined { + if (inProcess) { + return { + why: 'this server was not started by an entry a detached process can run again (bootstrap(), runStdio(), runUnixSocket() or @FrontMcp), as an in-process server (create(), createDirect(), connect(), the generated CLI) is not', + remedy: "Start the server from its entry (node dist/main.js, frontmcp dev), or set runner: 'inline' on the job", + }; + } + if (storeType === 'memory') { + return { + why: 'job runs are kept in this process memory, which a detached process cannot read', + remedy: 'Give jobs a store other processes share: jobs.store.sqlite, a top-level sqlite or redis', + }; + } + return undefined; +} + /** * Publish a finished job or workflow run (the status notification of a run that ended) as a job * completion, read from its run record. The event names the session that ran it, if any. diff --git a/libs/sdk/src/job/runner/__tests__/job-runner.spec.ts b/libs/sdk/src/job/runner/__tests__/job-runner.spec.ts new file mode 100644 index 000000000..34575a21c --- /dev/null +++ b/libs/sdk/src/job/runner/__tests__/job-runner.spec.ts @@ -0,0 +1,203 @@ +/** + * Runner selection: a job's own `runner`, else `jobs.runner`, else inline; `detached` is refused at boot where a + * process of its own cannot outlive the server, and for a job whose attempts check the caller's credentials. + */ +import 'reflect-metadata'; + +import { execFileSync } from 'child_process'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; + +import { z } from '@frontmcp/lazy-zod'; + +import { Job, JobContext, type JobPermission, type JobRunner } from '../../../common'; +import { JobRunnerUnavailableError } from '../../../errors'; +import { + assertRunnersAvailable, + detachedJobRunId, + detachedRunnerCommand, + JOB_WORKER_MARKER, + RUN_JOB_ENV_VAR, + runnerOf, + spawnDetachedRunner, +} from '../job-runner'; + +const node = { runtime: 'node', deployment: 'standalone' }; + +function job(name: string, metadata: { runner?: JobRunner; permissions?: JobPermission[] } = {}) { + return { name, metadata }; +} + +describe('runnerOf', () => { + it("is the job's own runner, else the server's, else inline", () => { + expect(runnerOf(job('a', { runner: 'worker' }), 'detached')).toBe('worker'); + expect(runnerOf(job('a'), 'detached')).toBe('detached'); + expect(runnerOf(job('a'))).toBe('inline'); + }); +}); + +describe('@Job({ runner })', () => { + it('takes the three runners', () => { + for (const runner of ['inline', 'worker', 'detached'] as const) { + expect(() => + Job({ name: `ok-${runner}`, inputSchema: {}, outputSchema: {}, runner })( + class extends JobContext { + async execute() { + return {}; + } + }, + ), + ).not.toThrow(); + } + }); + + it('refuses another runner with the ones it takes', () => { + expect(() => + Job({ + name: 'bad', + inputSchema: { n: z.number() }, + outputSchema: {}, + runner: 'thread' as never, + })( + class extends JobContext { + async execute(_input: { n: number }) { + return {}; + } + }, + ), + ).toThrow(/runner must be 'inline', 'worker' or 'detached'/); + }); +}); + +describe('assertRunnersAvailable', () => { + it('lets inline and worker jobs run anywhere', () => { + const jobs = [job('a'), job('b', { runner: 'worker' })]; + expect(() => assertRunnersAvailable(jobs, undefined, { runtime: 'edge', deployment: 'serverless' })).not.toThrow(); + }); + + it('lets a detached job run on a Node server that is not serverless', () => { + expect(() => assertRunnersAvailable([job('a', { runner: 'detached' })], undefined, node)).not.toThrow(); + }); + + it.each([ + [{ runtime: 'node', deployment: 'serverless' }, /serverless/], + [{ runtime: 'edge', deployment: 'standalone' }, /edge runtime/], + [{ runtime: 'browser', deployment: 'browser' }, /browser runtime/], + ])('refuses a detached job on %j, with a remedy', (runtime, why) => { + let refusal: unknown; + try { + assertRunnersAvailable([job('report', { runner: 'detached' })], undefined, runtime); + } catch (error) { + refusal = error; + } + + expect(refusal).toBeInstanceOf(JobRunnerUnavailableError); + expect((refusal as JobRunnerUnavailableError).message).toMatch(why); + expect((refusal as JobRunnerUnavailableError).message).toContain('Job "report"'); + expect((refusal as JobRunnerUnavailableError).remedy?.say).toMatch(/runner: 'inline'/); + expect((refusal as JobRunnerUnavailableError).code).toBe('JOB_RUNNER_UNAVAILABLE'); + }); + + it("refuses jobs.runner: 'detached' for a job that sets no runner, but not for one that sets its own", () => { + const serverless = { runtime: 'node', deployment: 'serverless' }; + expect(() => assertRunnersAvailable([job('a')], 'detached', serverless)).toThrow(JobRunnerUnavailableError); + expect(() => assertRunnersAvailable([job('a', { runner: 'inline' })], 'detached', serverless)).not.toThrow(); + }); + + it('refuses a detached job with execute permissions: its process gets no caller credentials to check', () => { + const guarded = job('guarded', { runner: 'detached', permissions: [{ action: 'execute', roles: ['ops'] }] }); + expect(() => assertRunnersAvailable([guarded], undefined, node)).toThrow(/execute permissions/); + + const readable = job('readable', { runner: 'detached', permissions: [{ action: 'read' }, { action: 'delete' }] }); + expect(() => assertRunnersAvailable([readable], undefined, node)).not.toThrow(); + }); +}); + +describe('the detached runner process', () => { + const { argv, execArgv } = process; + + afterEach(() => { + process.argv = argv; + process.execArgv = execArgv; + delete process.env[RUN_JOB_ENV_VAR]; + }); + + it("runs the server's own command again: node, its flags, its entry and arguments, without an older owner tag", () => { + process.execArgv = ['--import', 'file:///repo/node_modules/tsx/dist/loader.mjs']; + process.argv = [process.execPath, '/repo/dist/main.js', '--stdio', '--frontmcp-owner=frontmcp-task:42']; + + expect(detachedRunnerCommand()).toEqual({ + command: process.execPath, + args: ['--import', 'file:///repo/node_modules/tsx/dist/loader.mjs', '/repo/dist/main.js', '--stdio'], + }); + }); + + it('leaves the inspector flags out: a runner held by --inspect-brk would never start its run', () => { + process.execArgv = [ + '--inspect-brk=9229', + '--inspect', + '--inspect-wait', + '--inspect-port=9230', + '--enable-source-maps', + ]; + process.argv = [process.execPath, '/repo/dist/main.js']; + + expect(detachedRunnerCommand().args).toEqual(['--inspect-port=9230', '--enable-source-maps', '/repo/dist/main.js']); + }); + + it('is refused without an entry to run', () => { + process.argv = [process.execPath]; + expect(() => detachedRunnerCommand()).toThrow(/no entry/); + }); + + it('reads the run it is for from its environment', () => { + expect(detachedJobRunId()).toBeUndefined(); + process.env[RUN_JOB_ENV_VAR] = 'run-1'; + expect(detachedJobRunId()).toBe('run-1'); + }); + + it('is marked for reapOrphans', () => { + expect(JOB_WORKER_MARKER).toBe('frontmcp-job'); + }); + + it('starts in a process group of its own, with the run id, the owner tag and no task id', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sdk-job-runner-')); + const report = path.join(dir, 'report.json'); + const entry = path.join(dir, 'entry.js'); + fs.writeFileSync( + entry, + `require('fs').writeFileSync(${JSON.stringify(report)}, JSON.stringify({ + args: process.argv.slice(2), + run: process.env.${RUN_JOB_ENV_VAR}, + owner: process.env.FRONTMCP_OWNER, + task: process.env.FRONTMCP_RUN_TASK_ID ?? null, + })); setTimeout(() => undefined, 1000);`, + ); + process.execArgv = []; + process.argv = [process.execPath, entry, '--quiet']; + process.env['FRONTMCP_RUN_TASK_ID'] = 'task-1'; + try { + const child = spawnDetachedRunner('run-9'); + const deadline = Date.now() + 10_000; + while (!fs.existsSync(report) && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 20)); + + expect(JSON.parse(fs.readFileSync(report, 'utf8'))).toEqual({ + args: ['--quiet', `--frontmcp-owner=frontmcp-job:${process.pid}`], + run: 'run-9', + owner: `frontmcp-job:${process.pid}`, + task: null, + }); + if (process.platform !== 'win32') { + const group = execFileSync('ps', ['-o', 'pgid=', '-p', String(child.pid)]) + .toString() + .trim(); + expect(Number(group)).toBe(child.pid); + } + process.kill(child.pid ?? 0, 'SIGKILL'); + } finally { + delete process.env['FRONTMCP_RUN_TASK_ID']; + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/libs/sdk/src/job/runner/__tests__/run-flusher.spec.ts b/libs/sdk/src/job/runner/__tests__/run-flusher.spec.ts new file mode 100644 index 000000000..8592f3126 --- /dev/null +++ b/libs/sdk/src/job/runner/__tests__/run-flusher.spec.ts @@ -0,0 +1,165 @@ +/** + * A detached run keeps what it reports in memory and writes the latest of it to the store every 500 ms at most, + * and every runner renews its heartbeat in the store while it holds runs. + */ +import { MemoryJobStateStore } from '../../store/memory-job-state.store'; +import { LatestValueFlusher, RunnerHeartbeat } from '../run-flusher'; + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve: () => void = () => undefined; + const promise = new Promise((done) => (resolve = done)); + return { promise, resolve }; +} + +describe('LatestValueFlusher', () => { + beforeEach(() => jest.useFakeTimers()); + afterEach(() => jest.useRealTimers()); + + it('writes nothing until the first tick, then only the latest value, once per interval', async () => { + const writes: number[] = []; + const flusher = new LatestValueFlusher(async (value) => { + writes.push(value); + }, 500); + + for (let value = 1; value <= 50; value++) flusher.push(value); + expect(writes).toEqual([]); + + await jest.advanceTimersByTimeAsync(499); + expect(writes).toEqual([]); + await jest.advanceTimersByTimeAsync(1); + expect(writes).toEqual([50]); + + await jest.advanceTimersByTimeAsync(2_000); + expect(writes).toEqual([50]); + + flusher.push(51); + flusher.push(52); + await jest.advanceTimersByTimeAsync(500); + expect(writes).toEqual([50, 52]); + flusher.stop(); + }); + + it('starts no write while the last one is still in flight, and writes what came meanwhile after it', async () => { + const writes: number[] = []; + let slow = deferred(); + const flusher = new LatestValueFlusher(async (value) => { + writes.push(value); + await slow.promise; + }, 500); + + flusher.push(1); + await jest.advanceTimersByTimeAsync(500); + flusher.push(2); + flusher.push(3); + await jest.advanceTimersByTimeAsync(2_000); + expect(writes).toEqual([1]); + + const first = slow; + slow = deferred(); + first.resolve(); + await jest.advanceTimersByTimeAsync(500); + expect(writes).toEqual([1, 3]); + slow.resolve(); + flusher.stop(); + }); + + it('writes what is pending on flush(), and nothing after stop()', async () => { + const writes: string[] = []; + const flusher = new LatestValueFlusher(async (value) => { + writes.push(value); + }, 500); + + flusher.push('a'); + await flusher.flush(); + expect(writes).toEqual(['a']); + await flusher.flush(); + expect(writes).toEqual(['a']); + + flusher.push('b'); + flusher.stop(); + await jest.advanceTimersByTimeAsync(5_000); + expect(writes).toEqual(['a']); + }); + + it('keeps the value of a write that failed for the next tick', async () => { + const writes: number[] = []; + let fail = true; + const flusher = new LatestValueFlusher( + async (value) => { + if (fail) throw new Error('store busy'); + writes.push(value); + }, + 500, + () => undefined, + ); + + flusher.push(7); + await jest.advanceTimersByTimeAsync(500); + fail = false; + await jest.advanceTimersByTimeAsync(500); + expect(writes).toEqual([7]); + flusher.stop(); + }); +}); + +describe('RunnerHeartbeat', () => { + beforeEach(() => jest.useFakeTimers()); + afterEach(() => jest.useRealTimers()); + + it('writes the heartbeat before hold() resolves and renews it every second while it is held', async () => { + const store = new MemoryJobStateStore(); + const touch = jest.spyOn(store, 'touchRunner'); + const heartbeat = new RunnerHeartbeat(store, 'runner-1'); + + const release = await heartbeat.hold(); + expect(touch).toHaveBeenCalledTimes(1); + expect(touch).toHaveBeenLastCalledWith('runner-1', 30); + expect(await store.isRunnerAlive('runner-1')).toBe(true); + + await jest.advanceTimersByTimeAsync(3_000); + expect(touch).toHaveBeenCalledTimes(4); + + release(); + release(); + await jest.advanceTimersByTimeAsync(3_000); + expect(touch).toHaveBeenCalledTimes(4); + await jest.advanceTimersByTimeAsync(30_000); + expect(await store.isRunnerAlive('runner-1')).toBe(false); + }); + + it('keeps a runner alive through 10 s in which its event loop could not renew the heartbeat', async () => { + const store = new MemoryJobStateStore(); + const heartbeat = new RunnerHeartbeat(store, 'runner-1'); + const release = await heartbeat.hold(); + + jest.setSystemTime(Date.now() + 10_000); + + expect(await store.isRunnerAlive('runner-1')).toBe(true); + release(); + }); + + it('beats once for every holder, until the last one lets go', async () => { + const store = new MemoryJobStateStore(); + const touch = jest.spyOn(store, 'touchRunner'); + const heartbeat = new RunnerHeartbeat(store, 'runner-1'); + + const first = await heartbeat.hold(); + const second = await heartbeat.hold(); + expect(touch).toHaveBeenCalledTimes(1); + + first(); + await jest.advanceTimersByTimeAsync(1_000); + expect(touch).toHaveBeenCalledTimes(2); + second(); + await jest.advanceTimersByTimeAsync(2_000); + expect(touch).toHaveBeenCalledTimes(2); + }); + + it('does nothing on a store that keeps no heartbeats', async () => { + const heartbeat = new RunnerHeartbeat({}, 'runner-1'); + + const release = await heartbeat.hold(); + release(); + await jest.advanceTimersByTimeAsync(5_000); + }); +}); diff --git a/libs/sdk/src/job/runner/detached-job.worker.ts b/libs/sdk/src/job/runner/detached-job.worker.ts new file mode 100644 index 000000000..ba164b3ce --- /dev/null +++ b/libs/sdk/src/job/runner/detached-job.worker.ts @@ -0,0 +1,110 @@ +import { + frontMcpMetadataSchema, + type FrontMcpConfigInput, + type FrontMcpConfigType, + type ScopeEntry, +} from '../../common'; +import { resolveDefaultSqlitePath } from '../../common/types/options/sqlite/resolve'; +import { JobRunnerBootError } from '../../errors'; +import { JOB_WORKER_FLAG, RUN_JOB_ENV_VAR } from '../../front-mcp/detached-job.flag'; +import { markDirectOnly } from '../../front-mcp/direct-only.flag'; +import { type JobExecutionManager } from '../execution/job-execution.manager'; +import { toJobError } from '../job-attempt'; +import { jobExecutionManagerOf } from '../job-run-access'; +import { resolveJobsStore, type JobsConfig } from '../job-scope.helper'; +import { createJobStateStore } from '../store/job-state-store.factory'; +import { FINISHED_JOB_STATES } from '../store/job-state.interface'; + +/** The server a detached runner boots: its scopes, and its shutdown. */ +export interface DetachedJobHost { + getScopes(): readonly ScopeEntry[]; + shutdown(): Promise; +} + +type ServerConfig = FrontMcpConfigInput | FrontMcpConfigType; + +/** + * The entry of a detached runner: boot the server's config with no transport, run the run through the job + * execution manager of the scope that started it, and shut down. SIGTERM or SIGINT interrupts the run (its + * cleanup runs). A boot that fails is written to the run as `JobRunnerBootError`. Resolves with the exit code. + * The run id leaves the environment first, so a server the job starts does not boot as a runner too. + */ +export async function runDetachedJob( + config: ServerConfig, + runId: string, + boot: (config: ServerConfig) => Promise = bootRunner, +): Promise { + delete process.env[RUN_JOB_ENV_VAR]; + let host: DetachedJobHost; + try { + host = await boot(config); + } catch (error) { + await reportBootFailure(config, runId, error); + return 1; + } + const stop = () => void host.shutdown().catch(() => undefined); + process.once('SIGTERM', stop); + process.once('SIGINT', stop); + // The run's own timers are unref'd, and a job may only await its signal: the process lives until the run ends + const keepAlive = setInterval(() => undefined, 60_000); + try { + await (await managerOf(host, runId))?.runDetached(runId); + return 0; + } finally { + clearInterval(keepAlive); + process.off('SIGTERM', stop); + process.off('SIGINT', stop); + await host.shutdown().catch(() => undefined); + } +} + +async function bootRunner(config: ServerConfig): Promise { + const { FrontMcpInstance } = await import('../../front-mcp/front-mcp.js'); + const parsed = frontMcpMetadataSchema.parse({ ...config, http: undefined, serve: false }); + markDirectOnly(parsed); + (parsed as Record)[JOB_WORKER_FLAG] = true; + const instance = new FrontMcpInstance(parsed); + await instance.ready; + return instance; +} + +/** The job execution manager of the scope that started the run. */ +async function managerOf(host: DetachedJobHost, runId: string): Promise { + for (const scope of host.getScopes()) { + const manager = jobExecutionManagerOf(scope); + const record = await manager?.getStatus(runId); + if (record && (record.scopeId === undefined || record.scopeId === scope.id)) return manager; + } + return undefined; +} + +/** Open the job store the server would and end the run with `JobRunnerBootError`, unless it was cancelled. */ +async function reportBootFailure(config: ServerConfig, runId: string, cause: unknown): Promise { + const sqlite = config.sqlite && { + ...config.sqlite, + path: config.sqlite.path || resolveDefaultSqlitePath({ appName: config.info?.name, cliMode: false }), + }; + const jobs = config.jobs as JobsConfig | undefined; + const store = resolveJobsStore(jobs?.store, { redis: config.redis, sqlite }); + try { + const { store: stateStore } = await createJobStateStore({ + redis: store?.redis, + sqlite: store?.sqlite, + keyPrefix: store?.keyPrefix ?? 'mcp:jobs:', + }); + try { + const record = await stateStore.getRun(runId); + if (!record || FINISHED_JOB_STATES.has(record.state) || (await stateStore.isCancelRequested?.(runId))) return; + const error = new JobRunnerBootError(runId, toJobError(cause).message); + await stateStore.updateRun(runId, { + state: 'failed', + error: { name: error.name, message: error.message }, + completedAt: Date.now(), + }); + } finally { + await stateStore.dispose(); + } + } catch { + // The run stays pending; the next server that reconciles the store ends it once its runner is gone + } +} diff --git a/libs/sdk/src/job/runner/job-runner.ts b/libs/sdk/src/job/runner/job-runner.ts new file mode 100644 index 000000000..8629ee850 --- /dev/null +++ b/libs/sdk/src/job/runner/job-runner.ts @@ -0,0 +1,100 @@ +import { getRuntimeContext, spawnOwned } from '@frontmcp/utils'; + +import { type JobPermission, type JobRunner } from '../../common/metadata/job.metadata'; +import { JobRunnerUnavailableError } from '../../errors'; +import { JOB_WORKER_MARKER, RUN_JOB_ENV_VAR } from '../../front-mcp/detached-job.flag'; +import { RUN_TASK_ENV_VAR } from '../../task/helpers/cli-task-runner'; + +export { detachedJobRunId, JOB_WORKER_MARKER, RUN_JOB_ENV_VAR } from '../../front-mcp/detached-job.flag'; + +const OWNER_ARG_PREFIX = '--frontmcp-owner='; + +/** `--inspect`, `--inspect-brk` and `--inspect-wait`: a runner would take the server's debugger port, or wait for one. */ +const INSPECTOR_FLAG = /^--inspect(-brk|-wait)?(=|$)/; + +const INLINE_REMEDY = "Set runner: 'inline' (or 'worker') on the job, or run the server where it can start processes"; + +interface RunnerJob { + name: string; + metadata: { runner?: JobRunner; permissions?: JobPermission[] }; +} + +/** The job's own runner, else the server's `jobs.runner`, else `'inline'`. */ +export function runnerOf(job: RunnerJob, serverRunner?: JobRunner): JobRunner { + return job.metadata.runner ?? serverRunner ?? 'inline'; +} + +/** + * Refuse, at boot, a detached job where its process cannot outlive the server (an edge or browser runtime, a + * serverless deployment) or where its attempts would check credentials its process never receives. + */ +export function assertRunnersAvailable( + jobs: readonly RunnerJob[], + serverRunner: JobRunner | undefined, + runtime: { runtime: string; deployment: string } = getRuntimeContext(), +): void { + for (const job of jobs) { + if (runnerOf(job, serverRunner) !== 'detached') continue; + if (runtime.runtime !== 'node') { + throw new JobRunnerUnavailableError( + job.name, + `a detached run needs a process of its own, which the ${runtime.runtime} runtime cannot start`, + INLINE_REMEDY, + ); + } + if (runtime.deployment === 'serverless') { + throw new JobRunnerUnavailableError( + job.name, + 'a detached run must outlive the server, which a serverless deployment stops or replaces after each request', + INLINE_REMEDY, + ); + } + if (job.metadata.permissions?.some((rule) => rule.action === 'execute')) { + throw new JobRunnerUnavailableError( + job.name, + 'its execute permissions are checked on every attempt, and a detached process never receives the caller credentials', + "Set runner: 'inline' or 'worker' on the job, or drop its execute permissions", + ); + } + } +} + +/** The server's own command (node, its flags but the inspector's, its entry and arguments), which a runner runs again. */ +export function detachedRunnerCommand(): { command: string; args: string[] } { + const [, entry, ...args] = process.argv; + if (!entry) { + throw new JobRunnerUnavailableError( + 'detached', + 'the server process has no entry a detached runner can run again', + 'Start the server from its entry (node dist/main.js, frontmcp dev) to run jobs detached', + ); + } + const ownArgs = args.filter((arg) => !arg.startsWith(OWNER_ARG_PREFIX)); + const flags = process.execArgv.filter((flag) => !INSPECTOR_FLAG.test(flag)); + return { command: process.execPath, args: [...flags, entry, ...ownArgs] }; +} + +/** The process of a detached run, as the server that starts it sees it. */ +export interface DetachedRunnerProcess { + readonly pid?: number; + once(event: 'error', listener: (error: Error) => void): unknown; +} + +/** + * Start the process of a detached run: the server's own command, detached into a process group of its own (so a + * stop of the server's group spares it), with the server's environment plus the run id and nothing of the caller. + */ +export function spawnDetachedRunner(runId: string): DetachedRunnerProcess { + const { command, args } = detachedRunnerCommand(); + const env: NodeJS.ProcessEnv = { ...process.env, [RUN_JOB_ENV_VAR]: runId }; + delete env[RUN_TASK_ENV_VAR]; + const { child } = spawnOwned(command, args, { + marker: JOB_WORKER_MARKER, + detached: true, + stdio: 'ignore', + windowsHide: true, + env, + }); + child.unref(); + return child; +} diff --git a/libs/sdk/src/job/runner/run-flusher.ts b/libs/sdk/src/job/runner/run-flusher.ts new file mode 100644 index 000000000..43696a652 --- /dev/null +++ b/libs/sdk/src/job/runner/run-flusher.ts @@ -0,0 +1,100 @@ +import { type JobStateStore } from '../store/job-state.interface'; + +/** + * How often a runner renews its heartbeat, and how long the store keeps it without a renewal: long enough that a job + * blocking its runner's event loop is not taken for gone by a server on another host, which would end its run. + */ +export const RUNNER_HEARTBEAT_MS = 1_000; +export const RUNNER_HEARTBEAT_TTL_SECONDS = 30; + +/** + * Keeps the last value pushed and writes it every `everyMs` at most: never two writes at once, nothing + * when nothing new came, and a failed write's value is kept for the next tick. + */ +export class LatestValueFlusher { + private pending?: { value: T }; + private inFlight?: Promise; + private readonly timer: ReturnType; + private stopped = false; + + constructor( + private readonly write: (value: T) => Promise, + everyMs: number, + private readonly onError: (error: unknown) => void = () => undefined, + ) { + this.timer = setInterval(() => void this.tick(), everyMs); + this.timer.unref?.(); + } + + push(value: T): void { + if (!this.stopped) this.pending = { value }; + } + + /** Write what is pending now, after the write in flight. */ + async flush(): Promise { + await this.inFlight; + await this.tick(); + } + + /** Stop the ticks; what is still pending is dropped. */ + stop(): void { + this.stopped = true; + this.pending = undefined; + clearInterval(this.timer); + } + + private async tick(): Promise { + if (this.inFlight || !this.pending) return; + const { value } = this.pending; + this.pending = undefined; + this.inFlight = this.write(value).catch((error: unknown) => { + this.pending ??= { value }; + this.onError(error); + }); + try { + await this.inFlight; + } finally { + this.inFlight = undefined; + } + } +} + +/** + * A runner's heartbeat in the job store: written before the first `hold()` resolves, renewed every second + * while anything holds it, and left to expire by the store's TTL once nothing does. + */ +export class RunnerHeartbeat { + private holders = 0; + private timer?: ReturnType; + + constructor( + private readonly store: Pick, + readonly runnerId: string, + private readonly onError: (error: unknown) => void = () => undefined, + ) {} + + /** Hold the heartbeat until the returned function is called. */ + async hold(): Promise<() => void> { + this.holders++; + if (this.holders === 1 && this.store.touchRunner) { + this.timer = setInterval(() => void this.beat(), RUNNER_HEARTBEAT_MS); + this.timer.unref?.(); + await this.beat(); + } + let released = false; + return () => { + if (released) return; + released = true; + this.holders--; + if (this.holders === 0) clearInterval(this.timer); + }; + } + + private async beat(): Promise { + try { + await this.store.touchRunner?.(this.runnerId, RUNNER_HEARTBEAT_TTL_SECONDS); + } catch (error) { + this.onError(error); + } + } +} diff --git a/libs/sdk/src/job/store/__tests__/job-store.parity.spec.ts b/libs/sdk/src/job/store/__tests__/job-store.parity.spec.ts index 3349e584b..2c27c3cbd 100644 --- a/libs/sdk/src/job/store/__tests__/job-store.parity.spec.ts +++ b/libs/sdk/src/job/store/__tests__/job-store.parity.spec.ts @@ -46,6 +46,15 @@ function run(runId: string, overrides: Partial = {}): JobRunRecord /** The Redis commands the Redis job state store sends, kept in Maps. Its key patterns end in `*`. */ function fakeRedis(): RedisStateStoreLike { const values = new Map(); + const expiries = new Map(); + function live(key: string): string | undefined { + const expiry = expiries.get(key); + if (expiry !== undefined && expiry <= Date.now()) { + values.delete(key); + expiries.delete(key); + } + return values.get(key); + } const sets = new Map>(); function setOf(key: string): Set { let set = sets.get(key); @@ -56,8 +65,15 @@ function fakeRedis(): RedisStateStoreLike { return set; } return { - get: async (key) => values.get(key) ?? null, - set: async (key, value) => values.set(key, value), + get: async (key) => live(key) ?? null, + set: async (key, value, ...args) => { + if (args.includes('NX') && live(key) !== undefined) return null; + const ex = args.indexOf('EX'); + if (ex >= 0) expiries.set(key, Date.now() + Number(args[ex + 1]) * 1000); + else expiries.delete(key); + values.set(key, value); + return 'OK'; + }, del: async (key) => (values.delete(String(key)) ? 1 : 0), sadd: async (key, ...members) => { members.forEach((member) => setOf(key).add(member)); @@ -154,6 +170,24 @@ describe.each(stateBackends)('%s job state store', (_backend, open) => { expect(await store.isCancelRequested?.('r1')).toBe(true); expect(await store.isCancelRequested?.('r2')).toBe(false); }); + + it('keeps a runner alive while its heartbeat lasts, and reads it as gone once the TTL lapses', async () => { + expect(await store.isRunnerAlive?.('runner-1')).toBe(false); + await store.touchRunner?.('runner-1', 1); + + expect(await store.isRunnerAlive?.('runner-1')).toBe(true); + expect(await store.isRunnerAlive?.('runner-2')).toBe(false); + await new Promise((resolve) => setTimeout(resolve, 1_100)); + expect(await store.isRunnerAlive?.('runner-1')).toBe(false); + }); + + it('lets one caller claim a run', async () => { + await store.createRun(run('r1')); + + expect(await store.claimRun?.('r1')).toBe(true); + expect(await store.claimRun?.('r1')).toBe(false); + expect(await store.claimRun?.('r2')).toBe(true); + }); }); describe('sqlite job state store across processes', () => { @@ -162,11 +196,15 @@ describe('sqlite job state store across processes', () => { await first.createRun(run('r1')); await first.updateRun('r1', { state: 'running' }); await first.requestCancel?.('r1'); + await first.touchRunner?.('runner-1', 60); + await first.claimRun?.('r1'); await first.dispose(); const second = (await createJobStateStore({ sqlite: { path: dbPath } })).store; expect(await second.getRun('r1')).toEqual(expect.objectContaining({ state: 'running', revision: 2 })); expect(await second.isCancelRequested?.('r1')).toBe(true); + expect(await second.isRunnerAlive?.('runner-1')).toBe(true); + expect(await second.claimRun?.('r1')).toBe(false); await second.dispose(); }); diff --git a/libs/sdk/src/job/store/job-state.interface.ts b/libs/sdk/src/job/store/job-state.interface.ts index ed7fbda7b..daeb533a3 100644 --- a/libs/sdk/src/job/store/job-state.interface.ts +++ b/libs/sdk/src/job/store/job-state.interface.ts @@ -1,3 +1,5 @@ +import { type JobRunner } from '../../common/metadata/job.metadata'; + export type JobExecutionState = | 'pending' | 'running' @@ -24,6 +26,14 @@ export interface JobRunProgress { message?: string; } +/** The process that runs a run, and the id it keeps its heartbeat under in the store. */ +export interface JobRunnerInfo { + kind: JobRunner; + id: string; + pid: number; + hostname?: string; +} + export interface JobRunRecord { runId: string; jobId: string; @@ -50,6 +60,10 @@ export interface JobRunRecord { revision?: number; /** When the store last wrote the run. */ updatedAt?: number; + /** The process running the run; a run without one was written before runners were recorded. */ + runner?: JobRunnerInfo; + /** The scope whose job registry holds the job. */ + scopeId?: string; } export interface WorkflowRunRecord extends JobRunRecord { @@ -86,5 +100,13 @@ export interface JobStateStore { */ requestCancel?(runId: string): Promise; isCancelRequested?(runId: string): Promise; + /** + * Record that a runner is alive for `ttlSeconds`, expired by the store's own TTL. Without it (and + * `isRunnerAlive`) a runner is judged by its pid on its own host only. + */ + touchRunner?(runnerId: string, ttlSeconds: number): Promise; + isRunnerAlive?(runnerId: string): Promise; + /** True for the one caller that claims an orphaned run, so only one process ends it and runs its cleanup. */ + claimRun?(runId: string): Promise; dispose(): Promise; } diff --git a/libs/sdk/src/job/store/memory-job-state.store.ts b/libs/sdk/src/job/store/memory-job-state.store.ts index c25709b70..b08add435 100644 --- a/libs/sdk/src/job/store/memory-job-state.store.ts +++ b/libs/sdk/src/job/store/memory-job-state.store.ts @@ -12,6 +12,8 @@ import { export class MemoryJobStateStore implements JobStateStore { private readonly runs = new Map(); private readonly cancelRequests = new Set(); + private readonly runnerExpiries = new Map(); + private readonly claims = new Set(); async createRun(record: JobRunRecord | WorkflowRunRecord): Promise { this.runs.set(record.runId, { ...record, revision: 1, updatedAt: Date.now() }); @@ -78,8 +80,24 @@ export class MemoryJobStateStore implements JobStateStore { return this.cancelRequests.has(runId); } + async touchRunner(runnerId: string, ttlSeconds: number): Promise { + this.runnerExpiries.set(runnerId, Date.now() + ttlSeconds * 1000); + } + + async isRunnerAlive(runnerId: string): Promise { + return (this.runnerExpiries.get(runnerId) ?? 0) > Date.now(); + } + + async claimRun(runId: string): Promise { + if (this.claims.has(runId)) return false; + this.claims.add(runId); + return true; + } + async dispose(): Promise { this.runs.clear(); this.cancelRequests.clear(); + this.runnerExpiries.clear(); + this.claims.clear(); } } diff --git a/libs/sdk/src/job/store/redis-job-state.store.ts b/libs/sdk/src/job/store/redis-job-state.store.ts index f0ce17992..069bace4f 100644 --- a/libs/sdk/src/job/store/redis-job-state.store.ts +++ b/libs/sdk/src/job/store/redis-job-state.store.ts @@ -165,6 +165,18 @@ export class RedisJobStateStore implements JobStateStore { return (await this.client.get(this.cancelKey(runId))) !== null; } + async touchRunner(runnerId: string, ttlSeconds: number): Promise { + await this.client.set(`${this.keyPrefix}runner:${runnerId}`, '1', 'EX', ttlSeconds); + } + + async isRunnerAlive(runnerId: string): Promise { + return (await this.client.get(`${this.keyPrefix}runner:${runnerId}`)) !== null; + } + + async claimRun(runId: string): Promise { + return (await this.client.set(`${this.keyPrefix}claim:${runId}`, '1', 'NX', 'EX', this.ttlSeconds)) === 'OK'; + } + async dispose(): Promise { // A client passed in is shared and outlives the store if (!this.ownsClient) return; diff --git a/libs/sdk/src/job/store/storage-job-state.store.ts b/libs/sdk/src/job/store/storage-job-state.store.ts index a2167c9c0..3998c2cd6 100644 --- a/libs/sdk/src/job/store/storage-job-state.store.ts +++ b/libs/sdk/src/job/store/storage-job-state.store.ts @@ -12,7 +12,7 @@ type RunRecord = JobRunRecord | WorkflowRunRecord; /** * A JobStateStore on a `@frontmcp/utils` storage, such as the SQLite storage several processes on one * machine share. Each run is one JSON value under `run:`, and a cancel request is its own key, - * `cancel:`, so no write of the run record can drop it. The store owns the storage and + * `cancel:`, so no write of the run record can drop it; a runner's heartbeat is `runner:`. The store owns the storage and * disconnects it on dispose. */ export class StorageJobStateStore implements JobStateStore { @@ -57,7 +57,9 @@ export class StorageJobStateStore implements JobStateStore { const cutoff = Date.now() - olderThanMs; const stale = (await this.allRuns()).filter((run) => run.startedAt < cutoff); if (stale.length > 0) { - await this.storage.mdelete(stale.flatMap((run) => [runKey(run.runId), cancelKey(run.runId)])); + await this.storage.mdelete( + stale.flatMap((run) => [runKey(run.runId), cancelKey(run.runId), claimKey(run.runId)]), + ); } return stale.length; } @@ -70,6 +72,20 @@ export class StorageJobStateStore implements JobStateStore { return this.storage.exists(cancelKey(runId)); } + async touchRunner(runnerId: string, ttlSeconds: number): Promise { + await this.storage.set(runnerKey(runnerId), '1', { ttlSeconds }); + } + + async isRunnerAlive(runnerId: string): Promise { + return this.storage.exists(runnerKey(runnerId)); + } + + async claimRun(runId: string): Promise { + const claimed = (await this.storage.incr(claimKey(runId))) === 1; + if (claimed) await this.storage.expire(claimKey(runId), this.ttlSeconds); + return claimed; + } + async dispose(): Promise { await this.storage.disconnect(); } @@ -93,3 +109,11 @@ function runKey(runId: string): string { function cancelKey(runId: string): string { return `cancel:${runId}`; } + +function runnerKey(runnerId: string): string { + return `runner:${runnerId}`; +} + +function claimKey(runId: string): string { + return `claim:${runId}`; +} diff --git a/libs/sdk/src/offload/__tests__/offload-pool.spec.ts b/libs/sdk/src/offload/__tests__/offload-pool.spec.ts index 26894863e..379e25d20 100644 --- a/libs/sdk/src/offload/__tests__/offload-pool.spec.ts +++ b/libs/sdk/src/offload/__tests__/offload-pool.spec.ts @@ -71,6 +71,19 @@ globalThis[key].set(__filename + '#report#report://{id}', { }); `; +const JOB_MODULE = ` +const { isMainThread } = require('node:worker_threads'); +const key = Symbol.for('@frontmcp/sdk:worker-jobs'); +globalThis[key] = globalThis[key] || new Map(); +globalThis[key].set(__filename + '#crunch', { + run: async (input, attempt, log, progress) => { + log('info', 'crunching'); + progress({ progress: 1, total: 2, message: 'half way' }); + return { output: { input, attempt, isMainThread }, logs: ['crunched'] }; + }, +}); +`; + function testLogger(): FrontMcpLogger & Record<'verbose' | 'debug' | 'info' | 'warn' | 'error', jest.Mock> { const logger = { child: () => logger, @@ -96,6 +109,7 @@ describe('OffloadPool', () => { await writeFile(pathJoin(dir, 'default-function.mjs'), DEFAULT_FUNCTION_MODULE); await writeFile(pathJoin(dir, 'slow-to-load.mjs'), SLOW_TO_LOAD_MODULE); await writeFile(pathJoin(dir, 'report.cjs'), RESOURCE_MODULE); + await writeFile(pathJoin(dir, 'crunch.cjs'), JOB_MODULE); }); afterAll(async () => { @@ -419,6 +433,31 @@ describe('OffloadPool', () => { expect(logger.info).toHaveBeenCalledWith('unknown level'); }); + it('runs a worker job through the runner its module registered, with its progress and log lines', async () => { + const offload = pool(); + const logger = testLogger(); + const onProgress = jest.fn(); + const file = pathJoin(dir, 'crunch.cjs'); + + const value = await offload.run( + { kind: 'job', file, key: `${file}#crunch`, name: 'crunch', input: { n: 2 }, attempt: 3 }, + { ...call({ label: 'crunch', logger }), onProgress }, + ); + + expect(value).toEqual({ output: { input: { n: 2 }, attempt: 3, isMainThread: false }, logs: ['crunched'] }); + expect(onProgress).toHaveBeenCalledWith({ progress: 1, total: 2, message: 'half way' }); + expect(logger.info).toHaveBeenCalledWith('crunching'); + }); + + it('rejects a worker job its module did not register', async () => { + const offload = pool(); + const file = pathJoin(dir, 'crunch.cjs'); + + await expect( + offload.run({ kind: 'job', file, key: `${file}#other`, name: 'other', input: {}, attempt: 1 }, call()), + ).rejects.toThrow(/registered no worker job "other"/); + }); + it('rejects a worker resource its module did not register', async () => { const offload = pool(); diff --git a/libs/sdk/src/offload/__tests__/worker-jobs.spec.ts b/libs/sdk/src/offload/__tests__/worker-jobs.spec.ts new file mode 100644 index 000000000..7543a90a4 --- /dev/null +++ b/libs/sdk/src/offload/__tests__/worker-jobs.spec.ts @@ -0,0 +1,184 @@ +/** + * `runner: 'worker'`: a job's `execute()` runs in a worker thread of the scope's offload pool, while its + * `jobs:execute-job` flow (permissions, admission, timeout, cleanup, the run record) stays on the main thread. + */ +import 'reflect-metadata'; + +import { z } from '@frontmcp/lazy-zod'; + +import { App, Job, job, JobContext, LogLevel } from '../../common'; +import { type DirectMcpServer } from '../../direct/direct.types'; +import { ProviderNotAvailableError } from '../../errors/provider.errors'; +import { FrontMcpInstance } from '../../front-mcp/front-mcp'; +import { MemoryJobStateStore } from '../../job/store/memory-job-state.store'; +import { workerJobKey, workerJobRunners, type WorkerJobRunner } from '../offload-worker.globals'; +import { OffloadService } from '../offload.service'; +import { workerJobSourceOf } from '../worker-jobs'; + +class Token {} + +@Job({ + name: 'crunch_job', + inputSchema: { n: z.number() }, + outputSchema: { doubled: z.number() }, + runner: 'worker', +}) +class CrunchJob extends JobContext { + async execute({ n }: { n: number }) { + this.log(`crunching ${n}`); + await this.progress(1, 2, 'half way'); + if (n < 0) this.respond({ doubled: 0 }); + if (n === 7) return { doubled: this.get(Token) as never }; + return { doubled: n * 2 }; + } +} + +@Job({ name: 'plain_job', inputSchema: { n: z.number() }, outputSchema: { n: z.number() } }) +class PlainJob extends JobContext { + async execute({ n }: { n: number }) { + return { n }; + } +} + +const HandlerJob = job({ + name: 'handler_job', + inputSchema: { n: z.number() }, + outputSchema: { tripled: z.number() }, + runner: 'worker', +})(({ n }, ctx) => { + ctx.log('tripling'); + return { tripled: n * 3 }; +}); + +@App({ id: 'workers', name: 'Workers', jobs: [CrunchJob, PlainJob, HandlerJob] }) +class WorkersApp {} + +function runnerOf(target: unknown, name: string): WorkerJobRunner { + const source = workerJobSourceOf(target); + expect(source).toEqual({ file: __filename, name, key: workerJobKey(__filename, name) }); + const runner = workerJobRunners().get(source?.key ?? ''); + if (!runner) throw new Error(`${name} registered no worker runner`); + return runner; +} + +describe('a worker job, as its worker thread runs it', () => { + it('runs execute() with its input and attempt, and answers with its output, logs and progress', async () => { + const progress = jest.fn(); + const log = jest.fn(); + + const answer = await runnerOf(CrunchJob, 'crunch_job').run({ n: 4 }, 2, log, progress); + + expect(answer).toEqual({ output: { doubled: 8 }, logs: [expect.stringMatching(/\] crunching 4$/)] }); + expect(progress).toHaveBeenCalledWith({ progress: 1, total: 2, message: 'half way' }); + expect(log).toHaveBeenCalledWith('info', 'crunching 4'); + }); + + it('answers with what respond() sends, and refuses this.get(): a worker has no DI container', async () => { + const runner = runnerOf(CrunchJob, 'crunch_job'); + + await expect(runner.run({ n: -1 }, 1, jest.fn(), jest.fn())).resolves.toMatchObject({ output: { doubled: 0 } }); + await expect(runner.run({ n: 7 }, 1, jest.fn(), jest.fn())).rejects.toBeInstanceOf(ProviderNotAvailableError); + await expect(runner.run({ n: 7 }, 1, jest.fn(), jest.fn())).rejects.toThrow( + /job "crunch_job" runs in a worker thread \(runner: 'worker'\), which has no DI container/, + ); + }); + + it('runs a function-style job through its handler', async () => { + const handler = (HandlerJob as unknown as () => unknown)(); + + await expect(runnerOf(handler, 'handler_job').run({ n: 2 }, 1, jest.fn(), jest.fn())).resolves.toEqual({ + output: { tripled: 6 }, + logs: [expect.stringMatching(/\] tripling$/)], + }); + }); +}); + +describe('a worker job in a server', () => { + const servers: DirectMcpServer[] = []; + + afterEach(async () => { + jest.restoreAllMocks(); + for (const server of servers.splice(0)) await server.dispose(); + }); + + async function serve(runner?: 'worker'): Promise { + const server = await FrontMcpInstance.createDirect({ + info: { name: 'worker-jobs', version: '1.0.0' }, + apps: [WorkersApp], + logging: { level: LogLevel.Off }, + jobs: { enabled: true, ...(runner ? { runner } : {}) }, + }); + servers.push(server); + return server; + } + + it('hands execute() to the pool with the job and its signal, and keeps its result, logs and progress', async () => { + const run = jest.spyOn(OffloadService.prototype, 'run').mockImplementation(async (task, call) => { + call.onProgress?.({ progress: 1, total: 2, message: 'from the worker' }); + return { output: { doubled: 10 }, logs: ['[t] worked'] }; + }); + const server = await serve(); + + const response = await server.callTool('execute_job', { name: 'crunch_job', input: { n: 5 } }); + + expect(response.structuredContent).toEqual( + expect.objectContaining({ state: 'completed', result: { doubled: 10 }, logs: ['[t] worked'] }), + ); + expect(run).toHaveBeenCalledWith( + { + kind: 'job', + file: __filename, + key: workerJobKey(__filename, 'crunch_job'), + name: 'crunch_job', + input: { n: 5 }, + attempt: 1, + }, + expect.objectContaining({ label: 'crunch_job', signal: expect.any(AbortSignal) }), + ); + const runId = (response.structuredContent as { runId: string }).runId; + const read = await server.readResource(`job://runs/${runId}`); + expect(JSON.parse(String((read.contents[0] as { text: string }).text))).toEqual( + expect.objectContaining({ state: 'completed', progress: { progress: 1, total: 2, message: 'from the worker' } }), + ); + }); + + it('writes the latest progress a worker reports, not each report, and the last one before the outcome', async () => { + const updates = jest.spyOn(MemoryJobStateStore.prototype, 'updateRun'); + jest.spyOn(OffloadService.prototype, 'run').mockImplementation(async (_task, call) => { + for (let step = 1; step <= 200; step++) call.onProgress?.({ progress: step, total: 200 }); + return { output: { doubled: 2 }, logs: [] }; + }); + const server = await serve(); + + const response = await server.callTool('execute_job', { name: 'crunch_job', input: { n: 1 } }); + + const runId = (response.structuredContent as { runId: string }).runId; + const progressWrites = updates.mock.calls.filter(([id, update]) => id === runId && update.progress); + expect(progressWrites.length).toBeLessThanOrEqual(2); + const read = await server.readResource(`job://runs/${runId}`); + expect(JSON.parse(String((read.contents[0] as { text: string }).text))).toEqual( + expect.objectContaining({ state: 'completed', progress: { progress: 200, total: 200 } }), + ); + }); + + it("runs a job that sets no runner on the pool under jobs.runner: 'worker', and inline otherwise", async () => { + const run = jest.spyOn(OffloadService.prototype, 'run').mockResolvedValue({ output: { n: 1 }, logs: [] }); + + const inline = await serve(); + await inline.callTool('execute_job', { name: 'plain_job', input: { n: 1 } }); + expect(run).not.toHaveBeenCalled(); + + const pooled = await serve('worker'); + await pooled.callTool('execute_job', { name: 'plain_job', input: { n: 1 } }); + expect(run).toHaveBeenCalledWith(expect.objectContaining({ kind: 'job', name: 'plain_job' }), expect.anything()); + }); + + it("checks the worker's output against the job's outputSchema", async () => { + jest.spyOn(OffloadService.prototype, 'run').mockResolvedValue({ output: { doubled: 'nine' }, logs: [] }); + const server = await serve(); + + await expect(server.callTool('execute_job', { name: 'crunch_job', input: { n: 1 } })).rejects.toThrow( + /output does not match outputSchema/, + ); + }); +}); diff --git a/libs/sdk/src/offload/offload-pool.ts b/libs/sdk/src/offload/offload-pool.ts index bf58cf407..0d583d32d 100644 --- a/libs/sdk/src/offload/offload-pool.ts +++ b/libs/sdk/src/offload/offload-pool.ts @@ -23,7 +23,8 @@ const LOG_LEVELS: readonly WorkerLogLevel[] = ['verbose', 'debug', 'info', 'warn type WorkerTask = | { kind: 'function'; url: string; exportName: string; args: readonly unknown[] } - | { kind: 'resource'; url: string; key: string; name: string; uri: string; params: Record }; + | { kind: 'resource'; url: string; key: string; name: string; uri: string; params: Record } + | { kind: 'job'; url: string; key: string; name: string; input: unknown; attempt: number }; interface SerializedError { name: string; @@ -39,6 +40,7 @@ interface SerializedError { type WorkerMessage = | { type: 'started' } | { type: 'log'; level: string; text: string } + | { type: 'progress'; progress: { progress: number; total?: number; message?: string } } | { type: 'result'; value: unknown } | { type: 'error'; error: SerializedError; notCloneable?: boolean }; @@ -192,6 +194,10 @@ export class OffloadPool { pending.call.logger[level](message.text); return; } + if (message.type === 'progress') { + pending.call.onProgress?.(message.progress); + return; + } pooled.running = undefined; this.release(pooled); if (message.type === 'result') { @@ -268,6 +274,10 @@ export class OffloadPool { } function toWorkerTask(task: OffloadTask): WorkerTask { + if (task.kind === 'job') { + const { file, ...job } = task; + return { ...job, url: pathToFileURL(file).href }; + } if (task.kind === 'resource') { return { kind: 'resource', diff --git a/libs/sdk/src/offload/offload-worker.globals.ts b/libs/sdk/src/offload/offload-worker.globals.ts index 8b6cf5bc6..f85480b61 100644 --- a/libs/sdk/src/offload/offload-worker.globals.ts +++ b/libs/sdk/src/offload/offload-worker.globals.ts @@ -4,6 +4,9 @@ export const OFFLOAD_WORKER_MARKER = Symbol.for('@frontmcp/sdk:offload-worker'); /** Where a worker finds the readers of the `worker: true` resources its imports registered. */ export const WORKER_RESOURCES_KEY = Symbol.for('@frontmcp/sdk:worker-resources'); +/** Where a worker finds the runners of the jobs its imports registered, for `runner: 'worker'`. */ +export const WORKER_JOBS_KEY = Symbol.for('@frontmcp/sdk:worker-jobs'); + export type WorkerLogLevel = 'verbose' | 'debug' | 'info' | 'warn' | 'error'; export type WorkerLog = (level: WorkerLogLevel, text: string) => void; @@ -12,20 +15,55 @@ export interface WorkerResourceReader { read(uri: string, params: Record, log: WorkerLog): Promise; } +/** What a job's `this.progress()` reports from its worker. */ +export interface WorkerJobProgress { + progress: number; + total?: number; + message?: string; +} + +/** What a job's `execute()` answered in its worker, with what it logged with `this.log()`. */ +export interface WorkerJobAnswer { + output: unknown; + logs: string[]; +} + +export interface WorkerJobRunner { + run( + input: unknown, + attempt: number, + log: WorkerLog, + progress: (progress: WorkerJobProgress) => void, + ): Promise; +} + /** True inside a worker thread of the offload pool, where `@FrontMcp` records its metadata but serves nothing. */ export function isOffloadWorker(): boolean { return Reflect.get(globalThis, OFFLOAD_WORKER_MARKER) === true; } export function workerResourceReaders(): Map { - const existing: unknown = Reflect.get(globalThis, WORKER_RESOURCES_KEY); + return globalMap(WORKER_RESOURCES_KEY); +} + +export function workerJobRunners(): Map { + return globalMap(WORKER_JOBS_KEY); +} + +function globalMap(key: symbol): Map { + const existing: unknown = Reflect.get(globalThis, key); if (existing instanceof Map) return existing; - const readers = new Map(); - Reflect.set(globalThis, WORKER_RESOURCES_KEY, readers); - return readers; + const map = new Map(); + Reflect.set(globalThis, key, map); + return map; } /** A worker resource by its module, name and URI (or URI template), so two of one module that share a name differ. */ export function workerResourceKey(file: string, name: string, uri: string): string { return `${file}#${name}#${uri}`; } + +/** A worker job by its module and name. */ +export function workerJobKey(file: string, name: string): string { + return `${file}#${name}`; +} diff --git a/libs/sdk/src/offload/offload-worker.source.ts b/libs/sdk/src/offload/offload-worker.source.ts index 324e11772..a0445a59f 100644 --- a/libs/sdk/src/offload/offload-worker.source.ts +++ b/libs/sdk/src/offload/offload-worker.source.ts @@ -57,9 +57,18 @@ function exportedFunction(namespace, name, url) { return found; } +const log = (level, text) => parentPort.postMessage({ type: 'log', level, text }); + async function run(task) { await loaderReady; const namespace = await import(task.url); + if (task.kind === 'job') { + const runners = globalThis[Symbol.for('@frontmcp/sdk:worker-jobs')]; + const runner = runners && runners.get(task.key); + if (!runner) throw new Error('Importing ' + task.url + ' in a worker registered no worker job "' + task.name + '"'); + parentPort.postMessage({ type: 'started' }); + return runner.run(task.input, task.attempt, log, (progress) => parentPort.postMessage({ type: 'progress', progress })); + } if (task.kind === 'function') { const fn = exportedFunction(namespace, task.exportName, task.url); parentPort.postMessage({ type: 'started' }); @@ -69,7 +78,7 @@ async function run(task) { const reader = readers && readers.get(task.key); if (!reader) throw new Error('Importing ' + task.url + ' in a worker registered no worker resource "' + task.name + '"'); parentPort.postMessage({ type: 'started' }); - return reader.read(task.uri, task.params, (level, text) => parentPort.postMessage({ type: 'log', level, text })); + return reader.read(task.uri, task.params, log); } parentPort.on('message', (task) => { diff --git a/libs/sdk/src/offload/offload.types.ts b/libs/sdk/src/offload/offload.types.ts index 5cda51c70..c83e97f4a 100644 --- a/libs/sdk/src/offload/offload.types.ts +++ b/libs/sdk/src/offload/offload.types.ts @@ -35,7 +35,8 @@ export type OffloadFunction = (...args: never[]) => unknown; /** @internal What the pool runs. */ export type OffloadTask = | { kind: 'function'; moduleRef: OffloadModuleRef; args: readonly unknown[] } - | { kind: 'resource'; file: string; key: string; name: string; uri: string; params: Record }; + | { kind: 'resource'; file: string; key: string; name: string; uri: string; params: Record } + | { kind: 'job'; file: string; key: string; name: string; input: unknown; attempt: number }; /** @internal How the pool runs it. */ export interface OffloadRun { @@ -43,6 +44,8 @@ export interface OffloadRun { label: string; budgetMs?: number; signal?: AbortSignal; - /** Receives the budget warning and the log lines a worker-side resource writes. */ + /** Receives the budget warning and the log lines a worker-side resource or job writes. */ logger: FrontMcpLogger; + /** Receives the progress a worker-side job reports. */ + onProgress?: (progress: { progress: number; total?: number; message?: string }) => void; } diff --git a/libs/sdk/src/offload/run-job-in-worker.ts b/libs/sdk/src/offload/run-job-in-worker.ts new file mode 100644 index 000000000..39c546216 --- /dev/null +++ b/libs/sdk/src/offload/run-job-in-worker.ts @@ -0,0 +1,53 @@ +import { type JobEntry } from '../common/entries/job.entry'; +import { type JobCleanup } from '../common/metadata/job.metadata'; +import { InvalidOffloadCallError } from '../errors/offload.errors'; +import { LatestValueFlusher } from '../job/runner/run-flusher'; +import { type WorkerJobAnswer, type WorkerJobProgress } from './offload-worker.globals'; +import { OffloadService } from './offload.service'; +import { workerJobSourceOf } from './worker-jobs'; + +/** The longest a timer can wait; the job's own `timeout` stops it sooner, through `signal`. */ +const MAX_TIMER_MS = 2_147_483_647; + +/** How often the latest progress a worker reported is kept on the run, at most. */ +const PROGRESS_FLUSH_MS = 500; + +/** + * Run a `runner: 'worker'` job's `execute()` in the scope's offload pool, for the `jobs:execute-job` flow: the + * signal terminates its worker, the latest progress it reported is reported through the attempt's instance every + * 500 ms at most (and before its answer), and its logs join it. + */ +export async function runJobInWorker( + jobContext: Parameters[0], + job: JobEntry, + input: unknown, + signal: AbortSignal, +): Promise { + const name = job.metadata.name; + const source = workerJobSourceOf(job.record.provide); + if (!source?.file) { + throw new InvalidOffloadCallError( + `job "${name}" runs on the worker runner, but the module that declares it was not found on the call stack, so no worker can import it (a bundle that inlines @frontmcp/sdk has no separate module to import)`, + ); + } + const progress = new LatestValueFlusher(async (reported) => { + await jobContext.progress(reported.progress, reported.total, reported.message); + }, PROGRESS_FLUSH_MS); + try { + const answer = (await jobContext.get(OffloadService).run( + { kind: 'job', file: source.file, key: source.key, name: source.name, input, attempt: jobContext.attempt }, + { + label: name, + budgetMs: MAX_TIMER_MS, + signal, + logger: jobContext.scope.logger.child(`job:${name}`), + onProgress: (reported) => progress.push(reported), + }, + )) as WorkerJobAnswer; + await progress.flush(); + jobContext.keepLogs(answer.logs); + return answer.output; + } finally { + progress.stop(); + } +} diff --git a/libs/sdk/src/offload/worker-job.runner.ts b/libs/sdk/src/offload/worker-job.runner.ts new file mode 100644 index 000000000..35740afd0 --- /dev/null +++ b/libs/sdk/src/offload/worker-job.runner.ts @@ -0,0 +1,55 @@ +import { isClass } from '@frontmcp/di'; + +import { FlowControl } from '../common/interfaces/flow.interface'; +import { JobContext, type JobCtorArgs } from '../common/interfaces/job.interface'; +import { type JobMetadata, type ToolInputType, type ToolOutputType } from '../common/metadata'; +import { type WorkerJobAnswer, type WorkerJobProgress, type WorkerLog } from './offload-worker.globals'; +import { workerLogger, WorkerProviders } from './worker-resource.reader'; + +type AnyJobContext = JobContext; +type WorkerJobClass = new (args: JobCtorArgs) => AnyJobContext; +type WorkerJobHandler = (input: unknown, context: AnyJobContext) => unknown; + +/** Build a `runner: 'worker'` job inside its worker, with no DI container, and run its `execute()`. */ +export async function runWorkerJob( + target: object, + metadata: JobMetadata, + input: unknown, + attempt: number, + log: WorkerLog, + progress: (progress: WorkerJobProgress) => void, +): Promise { + const args: JobCtorArgs = { + metadata, + input, + attempt, + providers: new WorkerProviders(`job "${metadata.name}" runs in a worker thread (runner: 'worker')`), + logger: workerLogger(log), + authInfo: {}, + onProgress: async (reported) => progress(reported), + }; + const context = isClass(target) + ? new (target as WorkerJobClass)(args) + : new WorkerFunctionJobContext(target as WorkerJobHandler, args); + try { + return { output: await context.execute(input), logs: [...context.getLogs()] }; + } catch (error) { + if (!(error instanceof FlowControl)) throw error; + if (error.type === 'respond') return { output: error.output, logs: [...context.getLogs()] }; + const original: unknown = Reflect.get(error, 'originalError'); + throw original ?? new Error(`The job ended with ${error.type}`); + } +} + +class WorkerFunctionJobContext extends JobContext { + constructor( + private readonly handler: WorkerJobHandler, + args: JobCtorArgs, + ) { + super(args); + } + + async execute(input: unknown): Promise { + return this.handler(input, this); + } +} diff --git a/libs/sdk/src/offload/worker-jobs.ts b/libs/sdk/src/offload/worker-jobs.ts new file mode 100644 index 000000000..60a8d081f --- /dev/null +++ b/libs/sdk/src/offload/worker-jobs.ts @@ -0,0 +1,31 @@ +import { type JobMetadata } from '../common/metadata/job.metadata'; +import { workerJobKey, workerJobRunners } from './offload-worker.globals'; + +/** The module a worker imports to run a job on the worker runner, the job's name and its key in the worker. */ +export interface WorkerJobSource { + file: string | undefined; + name: string; + key: string; +} + +const sources = new WeakMap(); + +/** + * Record where a job is declared (its class, or a function job's handler), and how a worker that imported that + * module runs it. + */ +export function registerWorkerJob(target: object, metadata: JobMetadata, file: string | undefined): void { + const key = workerJobKey(file ?? '', metadata.name); + sources.set(target, { file, name: metadata.name, key }); + if (!file) return; + workerJobRunners().set(key, { + run: async (input, attempt, log, progress) => { + const { runWorkerJob } = await import('./worker-job.runner.js'); + return runWorkerJob(target, metadata, input, attempt, log, progress); + }, + }); +} + +export function workerJobSourceOf(target: unknown): WorkerJobSource | undefined { + return typeof target === 'function' ? sources.get(target) : undefined; +} diff --git a/libs/sdk/src/offload/worker-resource.reader.ts b/libs/sdk/src/offload/worker-resource.reader.ts index 926449e44..59e0c2f07 100644 --- a/libs/sdk/src/offload/worker-resource.reader.ts +++ b/libs/sdk/src/offload/worker-resource.reader.ts @@ -26,7 +26,7 @@ export async function readWorkerResource( metadata, uri, params, - providers: new WorkerProviders(metadata.name), + providers: new WorkerProviders(`resource "${metadata.name}" reads in a worker thread (worker: true)`), logger: workerLogger(log), authInfo: {}, }; @@ -55,9 +55,9 @@ class WorkerFunctionResourceContext extends ResourceContext { } } -/** A worker has no DI container: whatever a read asks of it names the resource and why. */ -class WorkerProviders implements ProviderRegistryInterface { - constructor(private readonly resourceName: string) {} +/** A worker has no DI container: whatever is asked of it names the entry and why. */ +export class WorkerProviders implements ProviderRegistryInterface { + constructor(private readonly where: string) {} get(token: Token): T { throw this.unavailable(tokenName(token)); @@ -80,14 +80,11 @@ class WorkerProviders implements ProviderRegistryInterface { } private unavailable(what: string): ProviderNotAvailableError { - return new ProviderNotAvailableError( - what, - `resource "${this.resourceName}" reads in a worker thread (worker: true), which has no DI container, scope or request context`, - ); + return new ProviderNotAvailableError(what, `${this.where}, which has no DI container, scope or request context`); } } -function workerLogger(log: WorkerLog): FrontMcpLogger { +export function workerLogger(log: WorkerLog): FrontMcpLogger { const at = (level: WorkerLogLevel) => (...args: unknown[]) => diff --git a/libs/sdk/src/scope/scope.instance.ts b/libs/sdk/src/scope/scope.instance.ts index 3ea2761ec..df6fba0a7 100644 --- a/libs/sdk/src/scope/scope.instance.ts +++ b/libs/sdk/src/scope/scope.instance.ts @@ -47,6 +47,7 @@ import { AuthConfigurationError, FlowExitedWithoutOutputError } from '../errors' import { UnenforcedMetadataError } from '../errors/plugin.errors'; import FlowRegistry from '../flows/flow.registry'; import { assertAuthoritiesRules } from '../front-mcp/authorities-rules.check'; +import { isEntryServer, isJobWorker } from '../front-mcp/detached-job.flag'; import { isDirectOnly } from '../front-mcp/direct-only.flag'; import type { HaManager } from '../ha'; import type { HealthService } from '../health'; @@ -888,6 +889,8 @@ export class Scope extends ScopeEntry { jobsConfig: effectiveConfig, logger: this.logger, notifyFn, + entryServer: isEntryServer(this.metadata), + jobWorker: isJobWorker(this.metadata), }); this._scopeJobs = result.jobRegistry as JobRegistry; @@ -1054,7 +1057,8 @@ export class Scope extends ScopeEntry { this._channelTeardown = channelResult.teardown; // `dispose()` (what `create()` returns calls it) disconnects service channels, as `shutdown()` does. this.onDispose(() => this.teardownChannels()); - await channelResult.connectServices(); + // A detached job runner runs one run: the server that started it already listens to the services + if (!isJobWorker(this.metadata)) await channelResult.connectServices(); } // Initialize health service (after all registries and stores are ready) diff --git a/libs/skills/catalog/frontmcp-development/SKILL.md b/libs/skills/catalog/frontmcp-development/SKILL.md index 09b148175..0670f7fe0 100644 --- a/libs/skills/catalog/frontmcp-development/SKILL.md +++ b/libs/skills/catalog/frontmcp-development/SKILL.md @@ -69,7 +69,7 @@ This is a router skill. The "steps" here are how to choose the right reference ( | Create a reusable conversation template or system prompt | `create-prompt` | Prompt entries with arguments and multi-turn message sequences | | Build an autonomous AI loop that orchestrates tools | `create-agent` | Agent entries with LLM config, inner tools, and swarm handoff | | Register shared services or configuration via DI | `create-provider` | Dependency injection tokens, `onBootstrap` / `onReady` / `onShutdown` hooks, factory providers, `createContainer()` | -| Run a background task with progress and retries | `create-job` | Job entries with attempt tracking, retry config, timeouts, cancel, cleanup, admission, and progress on `job://runs/{runId}` | +| Run a background task with progress and retries | `create-job` | Job entries with attempt tracking, retry config, timeouts, cancel, cleanup, admission, runners, and progress on `job://runs/{runId}` | | Chain multiple jobs into a sequential pipeline | `create-workflow` | Workflow entries that compose jobs with data passing | | Run recurring work in one process of the machine or project | `create-duty` | `@Duty({ every, scope, leaseMs })` providers: a lease one process holds, renewed while held, taken over when the holder dies | | Serve pages, APIs or live SSE views for people beside the MCP endpoint | `create-http-route` | `@HttpRoute` classes with the app's providers, people auth (`http.auth.people: 'loopback-token'`), same-origin check, SSE from a `resourceStore()` | @@ -182,6 +182,7 @@ Each reference has matching examples under [`examples//`](./examples/ | [`job-with-permissions`](./examples/create-job/job-with-permissions.md) | Advanced | A data export job with declarative permission controls, plus a function-style job for simple tasks. | | [`job-with-retry`](./examples/create-job/job-with-retry.md) | Intermediate | A job that syncs data from an external API with automatic retry, exponential backoff, and batch progress tracking. | | [`durable-build-job`](./examples/create-job/durable-build-job.md) | Advanced | A build job on a SQLite store with a timeout, a cleanup, admission limits, cancel and a subscribed run handle. | +| [`detached-export-job`](./examples/create-job/detached-export-job.md) | Advanced | An export job that runs in a process of its own, survives a server restart and is cancelled from any server. | ### `create-plugin-hooks` diff --git a/libs/skills/catalog/frontmcp-development/examples/create-job/detached-export-job.md b/libs/skills/catalog/frontmcp-development/examples/create-job/detached-export-job.md new file mode 100644 index 000000000..850decb3f --- /dev/null +++ b/libs/skills/catalog/frontmcp-development/examples/create-job/detached-export-job.md @@ -0,0 +1,127 @@ +--- +name: detached-export-job +reference: create-job +level: advanced +description: 'An export job that runs in a process of its own, survives a server restart and is cancelled from any server.' +tags: [development, job, detached, worker, sqlite, cancel] +features: + - "`runner: 'detached'`: each run in a process of its own that outlives the server that started it" + - 'A store the runner and every server share (the top-level `sqlite`), which the runs live in' + - 'Reading and cancelling a run from a server started after it, and a `cleanup` the runner runs on a cancel' + - 'Runs whose runner died settled as `interrupted` when the next server boots, with their cleanup' + - "`runner: 'worker'` for a CPU-heavy job that runs in a worker thread instead" +--- + +# Detached Export Job + +An export job that runs in a process of its own, survives a server restart and is cancelled from any server. + +A nightly export takes an hour. It must not stop when the server is redeployed, restarted by a client, or crashes: +the job runs detached, keeps its run in the server's SQLite file, and any server on that file reads, follows or +cancels it. + +## Code + +```typescript +// src/jobs/export.job.ts +import { Job, JobContext, z, type JobCleanup } from '@frontmcp/sdk'; + +declare function exportTable( + table: string, + options: { signal?: AbortSignal; onBatch: (done: number, total: number) => void }, +): Promise; +declare function dropPartialExport(table: string): Promise; + +const dropPartial: JobCleanup = async (job) => { + await dropPartialExport((job.input as { table: string }).table); +}; + +@Job({ + name: 'export-table', + inputSchema: { table: z.string() }, + outputSchema: { rows: z.number() }, + runner: 'detached', + timeout: 0, + // Read and cancel rules: a server started later answers callers other than the one who started the run + permissions: [ + { action: 'read', roles: ['ops'] }, + { action: 'delete', roles: ['ops'] }, + ], + cleanup: { run: dropPartial, timeoutMs: 30_000 }, +}) +export class ExportTableJob extends JobContext { + async execute({ table }: { table: string }) { + const rows = await exportTable(table, { + signal: this.signal, // a cancel from any server aborts it + onBatch: (done, total) => void this.progress(done, total, `${done} of ${total} batches`), + }); + return { rows }; + } +} +``` + +```typescript +// src/jobs/checksum.job.ts +import { createHash } from 'node:crypto'; + +import { Job, JobContext, z } from '@frontmcp/sdk'; + +// CPU-heavy and provider-free: it runs in a worker thread of the server, not on its event loop +@Job({ + name: 'checksum', + inputSchema: { text: z.string() }, + outputSchema: { sha256: z.string() }, + runner: 'worker', +}) +export class ChecksumJob extends JobContext { + async execute({ text }: { text: string }) { + return { sha256: createHash('sha256').update(text).digest('hex') }; + } +} +``` + +```typescript +// src/main.ts +import { App, FrontMcp } from '@frontmcp/sdk'; + +import { ChecksumJob } from './jobs/checksum.job'; +import { ExportTableJob } from './jobs/export.job'; + +@App({ name: 'exports', jobs: [ExportTableJob, ChecksumJob] }) +class ExportsApp {} + +@FrontMcp({ + info: { name: 'exports', version: '1.0.0' }, + apps: [ExportsApp], + // The runners read and write their runs here, as every server on this machine does + sqlite: { path: '~/.exports/state.sqlite' }, +}) +export default class ExportsServer {} +``` + +```typescript +// A client: start an export, then (after the server restarted) follow and cancel it +const started = await client.callTool({ + name: 'execute_job', + arguments: { name: 'export-table', input: { table: 'orders' } }, +}); +const { runId, uri } = started.structuredContent as { runId: string; uri: string }; // state: 'pending', at once + +// ...the server is redeployed; the export keeps running in its own process... + +await client.subscribeResource({ uri }); // the new server reads the run every second and notifies on changes +await client.callTool({ name: 'cancel_job', arguments: { runId } }); // the runner aborts and runs dropPartial +``` + +## What This Demonstrates + +- `runner: 'detached'`: each run in a process of its own that outlives the server that started it +- A store the runner and every server share (the top-level `sqlite`), which the runs live in +- Reading and cancelling a run from a server started after it, and a `cleanup` the runner runs on a cancel +- Runs whose runner died settled as `interrupted` when the next server boots, with their cleanup +- `runner: 'worker'` for a CPU-heavy job that runs in a worker thread instead + +## Related + +- See `create-job` for the runners, what a detached runner receives, and where it is refused +- See `offload-cpu-work` for the worker pool's rules diff --git a/libs/skills/catalog/frontmcp-development/references/create-job.md b/libs/skills/catalog/frontmcp-development/references/create-job.md index 2659c6018..b2f3000ec 100644 --- a/libs/skills/catalog/frontmcp-development/references/create-job.md +++ b/libs/skills/catalog/frontmcp-development/references/create-job.md @@ -1,6 +1,6 @@ --- name: create-job -description: Create long-running background jobs with retry policies, timeouts, cancellation, cleanup, progress tracking, and permissions +description: Create long-running background jobs with retry policies, timeouts, cancellation, cleanup, progress tracking, permissions, and runners (inline, worker thread, or a detached process that outlives the server) --- # Creating Jobs @@ -35,18 +35,19 @@ Create a class extending `JobContext` and implement the `execute(input: ### JobMetadata Fields -| Field | Type | Required | Default | Description | -| -------------- | ----------------------------------- | -------- | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| `name` | `string` | Yes | -- | Unique job name | -| `inputSchema` | `ZodRawShape` | Yes | -- | Zod raw shape for input validation | -| `outputSchema` | `ZodRawShape \| ZodType` | Yes | -- | Zod schema for output validation | -| `description` | `string` | No | -- | Human-readable description | -| `timeout` | `number` | No | `jobs.defaultTimeoutMs` (`300000`) | How long one attempt may run, in ms; the run ends as `timed_out` when its last attempt did. `0` = no limit | -| `cleanup` | `JobCleanup \| { run, timeoutMs? }` | No | -- | Runs when an attempt is interrupted, times out or is cancelled (see [Timeouts, Cancel and Cleanup](#timeouts-cancel-and-cleanup)) | -| `retry` | `RetryPolicy` | No | -- | Retry configuration (see below) | -| `tags` | `string[]` | No | -- | Categorization tags | -| `labels` | `Record` | No | -- | Key-value labels for filtering | -| `permissions` | `JobPermission[]` | No | -- | Array of permission rules. Multiple rules may target the same action; all matching rules must pass | +| Field | Type | Required | Default | Description | +| -------------- | ------------------------------------ | -------- | ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| `name` | `string` | Yes | -- | Unique job name | +| `inputSchema` | `ZodRawShape` | Yes | -- | Zod raw shape for input validation | +| `outputSchema` | `ZodRawShape \| ZodType` | Yes | -- | Zod schema for output validation | +| `description` | `string` | No | -- | Human-readable description | +| `timeout` | `number` | No | `jobs.defaultTimeoutMs` (`300000`) | How long one attempt may run, in ms; the run ends as `timed_out` when its last attempt did. `0` = no limit | +| `cleanup` | `JobCleanup \| { run, timeoutMs? }` | No | -- | Runs when an attempt is interrupted, times out or is cancelled (see [Timeouts, Cancel and Cleanup](#timeouts-cancel-and-cleanup)) | +| `runner` | `'inline' \| 'worker' \| 'detached'` | No | `jobs.runner` (`'inline'`) | Where the attempts run (see [Runners](#runners)) | +| `retry` | `RetryPolicy` | No | -- | Retry configuration (see below) | +| `tags` | `string[]` | No | -- | Categorization tags | +| `labels` | `Record` | No | -- | Key-value labels for filtering | +| `permissions` | `JobPermission[]` | No | -- | Array of permission rules. Multiple rules may target the same action; all matching rules must pass | ### Basic Example @@ -324,6 +325,62 @@ class CiServer {} `execute_job` answers with `{ runId, state, uri }`; a background run answers at once with `state: 'pending'` and `uri: 'job://runs/'`. Reading that resource returns the run as JSON (`runId`, `jobName`, `state`, `attempt`, `maxAttempts`, `startedAt`, `completedAt`, `updatedAt`, `revision`, `progress`, `result`, `error`, `logs`); `resources/subscribe` to it, and every write this process makes to the run (state changes and `this.progress()` reports) sends `notifications/resources/updated`. The same owner and `read` rule as `get_job_status` decide who may read it; a run the caller may not read reads as not found. Run states: `pending`, `running`, `retrying`, then `completed`, `failed`, `cancelled`, `timed_out` or `interrupted`. Each write bumps the record's `revision` and stamps `updatedAt`. +## Runners + +`@Job({ runner })`, or `@FrontMcp({ jobs: { runner } })` for the jobs that set none, picks where a job's attempts run. Any other value fails the decorator (`runner must be 'inline', 'worker' or 'detached'`). + +| Runner | Where `execute()` runs | Use for | +| ---------- | ------------------------------------------------------- | ------------------------------------------------------------------------ | +| `inline` | The server's event loop (the default) | Jobs that wait on I/O and use providers | +| `worker` | A worker thread of the server's offload pool | CPU-heavy jobs that would block the server | +| `detached` | A process of its own that outlives the server's process | Work that must finish after a restart, a deploy or a crash of the server | + +**`worker`.** The `jobs:execute-job` flow (permissions, admission, timeout, `cleanup`, the run record) stays on the main thread; only `execute()` moves to a worker of the offload pool (see `offload-cpu-work`), which imports the module that declares the job. The same rules as `this.offload()` apply: the worker has no DI container, scope or request context (`this.get()` and `this.scope` throw `ProviderNotAvailableError`), the worker's environment holds only `offload.env`, and the input and output cross by structured clone. `this.log()` lines reach the run as they do inline, and the latest `this.progress()` is kept on the run every 500 ms at most (and before the outcome), however often the worker reports. A timeout, a cancel or a shutdown terminates the worker, so `execute()` really stops; `this.signal` is not available inside the worker, and `cleanup` runs on the main thread. A bundle that inlines `@frontmcp/sdk` (SEA, `mcpb`, `lambda`, `vercel`) has no module to import and refuses the call with `InvalidOffloadCallError`. + +**`detached`.** `execute_job` answers at once with `{ runId, state: 'pending', uri }`, whatever `background` says, and the server starts a process for the run with `spawnOwned`: the server's own command again (`process.execPath`, its `execArgv` without `--inspect`, `--inspect-brk` or `--inspect-wait`, its entry and arguments) in a process group of its own, so stopping the server's group spares it. That process boots the server's config with no transport, no `onReady()` and no service channel connection, runs the run through the same `jobs:execute-job` flow, then exits. The run lives in the job store: + +- The runner writes its pid and keeps a heartbeat key in the store (renewed every second, expiring after 30 s by the store's own TTL, so a job that blocks its runner's event loop for less than that is not taken for gone on another host). The progress it reports is written every 500 ms at most; state changes are written at once. +- The run survives the server: a server killed or restarted mid-run leaves it running, and its result lands in the store. Any server on that store reads it (`get_job_status`, `job://runs/{runId}`) and cancels it with `cancel_job`; the runner reads the cancel within 500 ms, aborts `this.signal` and runs `cleanup` itself. A server watches the detached runs it knows of every second, so its `job://runs/{runId}` subscribers and `job-completion` channels hear about them. +- **Reconcile on boot.** A server that boots reads the store for runs left `pending`, `running` or `retrying`. A run whose runner is gone (on this host, its pid no longer runs, so a runner whose event loop is busy is never taken for gone, or it is the booting server's own pid on a run older than that server, as after a container restart; on another host, its heartbeat expired) ends `interrupted` (`JOB_INTERRUPTED`, `its runner is gone`) and its job's `cleanup` runs in that server with reason `'interrupt'`; one process settles each run, however many boot at once. A detached run whose runner never took it over ends `failed` with `JOB_RUNNER_BOOT_FAILED`, as does one whose runner could not boot (it writes the cause itself). A run whose runner is alive is watched until it ends. Inline runs of another live server are left alone. A run another process ended this way keeps that end, however its attempt finishes later, and a runner still alive (a server on another host took it for gone while its event loop was blocked for more than 30 s) aborts its attempt at its next read of the store (every 500 ms; a second for a server's inline run) and runs its `cleanup`. +- **What the runner gets.** The server's environment (it boots the same providers), its argv, `FRONTMCP_RUN_JOB_ID` and the owner tag; the run's input from the store. Nothing of the caller: no access token, headers or session, so `this.auth` is empty in a detached attempt. A job with `execute` permissions cannot run detached, since its attempts would check credentials the runner never receives. +- **Requirements.** A Node server started from its entry (`node dist/main.js`, `frontmcp dev`, a stdio server, a socket daemon) whose entry declares the server with `@FrontMcp` or calls `FrontMcpInstance.bootstrap()`, `runStdio()` or `runUnixSocket()`, and a store other processes share (`jobs.store.sqlite`, a top-level `sqlite`, or Redis). The runner runs that entry again, so code the entry runs before it starts the server runs in the runner too. At boot, a detached job on an edge or browser runtime, on a serverless deployment, or with `execute` permissions fails startup with `JobRunnerUnavailableError` (`JOB_RUNNER_UNAVAILABLE`, with a remedy). A server created any other way (`create()`, `createDirect()`, `connect()`, the generated CLI, `createForGraph()`, a handler) or a memory store refuses the run itself with the same code. +- `jobs.admission` bounds the attempts of one process, and each detached run is a process of its own: admission does not bound how many detached runs run at once. +- Runners carry the `spawnOwned` marker `JOB_WORKER_MARKER` (`'frontmcp-job'`). They are never reaped automatically, since outliving their server is the point; `reapOrphans(JOB_WORKER_MARKER)` from `@frontmcp/utils` stops the ones whose server is gone. SIGTERM interrupts a runner's run, which runs its `cleanup` and ends `interrupted`. + +```typescript +import { FrontMcp, Job, JobContext, z } from '@frontmcp/sdk'; + +declare function exportBucket( + bucket: string, + options: { signal?: AbortSignal; onFile: (count: number) => void }, +): Promise; +declare function removePartialUpload(bucket: string): Promise; +declare class ExportsApp {} + +@Job({ + name: 'export-archive', + inputSchema: { bucket: z.string() }, + outputSchema: { files: z.number() }, + runner: 'detached', + timeout: 0, + permissions: [{ action: 'read' }, { action: 'delete' }], + cleanup: async (job) => removePartialUpload((job.input as { bucket: string }).bucket), +}) +class ExportArchiveJob extends JobContext { + async execute({ bucket }: { bucket: string }) { + const files = await exportBucket(bucket, { signal: this.signal, onFile: (n) => this.progress(n) }); + return { files }; + } +} + +@FrontMcp({ + info: { name: 'exports', version: '1.0.0' }, + apps: [ExportsApp], + sqlite: { path: '~/.exports/state.sqlite' }, // the runners read and write runs here +}) +class ExportsServer {} +``` + ## Permissions Control who can interact with jobs using the `permissions` field. **`permissions` is an array** of rules; each rule grants access to a single `action` and lists the roles, scopes, and/or custom predicate required for that action. @@ -739,6 +796,8 @@ class DataServer {} | Watching a run | `resources/subscribe` to the `uri` `execute_job` returned | Polling `get_job_status` in a tight loop | Every write to the run sends `notifications/resources/updated` to subscribers | | Permissions | `permissions: [{ action: 'execute', roles: [...] }, ...]` (array, singular `action`) | `permissions: { actions: [...], roles, predicate }` (does not parse) | The schema requires an array of `{ action, roles, scopes, custom }` rules; `actions` plural and top-level `predicate` are not accepted | | Job hooks | `@JobHook.Will('execute')` on a plugin, or on the `@Job` class itself | Hooking `tools:call-tool` for `execute_job` | `jobs:execute-job` runs for every attempt, background run and workflow step; the `execute_job` tool's flow sees only the call | +| CPU-heavy job | `runner: 'worker'` | A tight loop in an inline `execute()` | An inline job blocks every other call of the server, `ping` included | +| Outliving restart | `runner: 'detached'` with a shared store | A background inline run for work that must survive a deploy | An inline run ends `interrupted` when its server stops; a detached run keeps going in its own process | ## Verification Checklist @@ -749,6 +808,7 @@ class DataServer {} - [ ] `retry` policy is configured if the job may fail transiently - [ ] `timeout` is set appropriately for the expected execution duration (`0` only for work that must never be cut off) - [ ] A job that starts processes or writes temp files has a `cleanup`, and passes `this.signal` on +- [ ] A `runner: 'worker'` job uses no providers; a `runner: 'detached'` job has no `execute` permissions and a shared store - [ ] Job is registered in `jobs` array of `@App` ### Runtime @@ -763,16 +823,20 @@ class DataServer {} ## Troubleshooting -| Problem | Cause | Solution | -| ------------------------------------------------------- | ------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | -| Job not activated | `@App` doesn't declare `jobs: [...]` AND `@FrontMcp({ jobs: { enabled: true } })` isn't set | Add the job class to `@App({ jobs: [...] })` (auto-enables) OR set `@FrontMcp({ jobs: { enabled: true, store: { ... } } })` | -| Job fails without retrying | No `retry` policy configured | Add `retry: { maxAttempts: 3, backoffMs: 2000 }` to `@Job` options | -| Progress not visible | Not calling `this.progress()` during execution | Add `this.progress(pct, total, message)` calls at each stage | -| Job times out unexpectedly | Default 5-minute timeout too short (it applies to `execute_job` since 1.11) | Set `timeout` in `@Job` (e.g., `600000`), `jobs.defaultTimeoutMs`, or `0` for no limit | -| `JOB_ADMISSION_TIMEOUT` | `jobs.admission` had no slot (or not enough free memory) within `maxWaitMs` | Retry later, run in the background, or raise `concurrency` / `maxWaitMs` | -| `JOB_RUN_FINISHED` | `cancel_job` on a run that already ended | Read its outcome with `get_job_status` | -| Run stays `running` after a cancel from another process | That process has no shared store, so it never sees the cancel | Use a shared `jobs.store` (`sqlite` on one machine, `redis` across machines) | -| Permission denied error | User lacks required roles or scopes | Verify user has one of the `roles` and all `scopes` defined in `permissions` | +| Problem | Cause | Solution | +| ------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | +| Job not activated | `@App` doesn't declare `jobs: [...]` AND `@FrontMcp({ jobs: { enabled: true } })` isn't set | Add the job class to `@App({ jobs: [...] })` (auto-enables) OR set `@FrontMcp({ jobs: { enabled: true, store: { ... } } })` | +| Job fails without retrying | No `retry` policy configured | Add `retry: { maxAttempts: 3, backoffMs: 2000 }` to `@Job` options | +| Progress not visible | Not calling `this.progress()` during execution | Add `this.progress(pct, total, message)` calls at each stage | +| Job times out unexpectedly | Default 5-minute timeout too short (it applies to `execute_job` since 1.11) | Set `timeout` in `@Job` (e.g., `600000`), `jobs.defaultTimeoutMs`, or `0` for no limit | +| `JOB_ADMISSION_TIMEOUT` | `jobs.admission` had no slot (or not enough free memory) within `maxWaitMs` | Retry later, run in the background, or raise `concurrency` / `maxWaitMs` | +| `JOB_RUN_FINISHED` | `cancel_job` on a run that already ended | Read its outcome with `get_job_status` | +| Run stays `running` after a cancel from another process | That process has no shared store, so it never sees the cancel | Use a shared `jobs.store` (`sqlite` on one machine, `redis` across machines) | +| Permission denied error | User lacks required roles or scopes | Verify user has one of the `roles` and all `scopes` defined in `permissions` | +| `JOB_RUNNER_UNAVAILABLE` at startup | A detached job on an edge, browser or serverless runtime, or with `execute` permissions | Set `runner: 'inline'` (or `'worker'`) on the job, or drop its `execute` permissions | +| `JOB_RUNNER_UNAVAILABLE` from `execute_job` | A detached job on a server not started from its entry (in process, the generated CLI) or a memory store | Start the server from its entry, and give jobs a shared store (`sqlite` or `redis`) | +| `JOB_RUNNER_BOOT_FAILED` | The detached runner could not boot the server, or exited before it took the run over | Check that the entry starts the server; the error's message names the cause | +| Detached run `interrupted`: `its runner is gone` | Its process died (killed, out of memory, machine restart) and a server reconciled the store | Its `cleanup` ran in that server; run the job again | ## Examples @@ -782,6 +846,7 @@ class DataServer {} | [`job-with-permissions`](../examples/create-job/job-with-permissions.md) | Advanced | A data export job with declarative permission controls, plus a function-style job for simple tasks. | | [`job-with-retry`](../examples/create-job/job-with-retry.md) | Intermediate | A job that syncs data from an external API with automatic retry, exponential backoff, and batch progress tracking. | | [`durable-build-job`](../examples/create-job/durable-build-job.md) | Advanced | A build job on a SQLite store with a timeout, a cleanup, admission limits, cancel and a subscribed run handle. | +| [`detached-export-job`](../examples/create-job/detached-export-job.md) | Advanced | An export job that runs in a process of its own, survives a server restart and is cancelled from any server. | > See all examples in [`examples/create-job/`](../examples/create-job/) diff --git a/libs/skills/catalog/frontmcp-development/references/create-workflow.md b/libs/skills/catalog/frontmcp-development/references/create-workflow.md index a7dea35ba..d2e97ca5a 100644 --- a/libs/skills/catalog/frontmcp-development/references/create-workflow.md +++ b/libs/skills/catalog/frontmcp-development/references/create-workflow.md @@ -59,7 +59,7 @@ Create a class decorated with `@Workflow`. The decorator requires `name` and `st | `timeout` | `number` | No | Per-step timeout in milliseconds (overrides workflow timeout for this step) | | `retry` | `RetryPolicy` | No | Per-step retry policy (overrides the job's retry policy for this step) | -When a step's `timeout` passes, the step fails with `WorkflowJobTimeoutError`. An attempt whose job has not started never starts it. A job already in `execute()` is not stopped: its `this.signal` (`ctx.signal` in a `job()` function) is aborted with the timeout as its reason, so pass it to `fetch` and other cancellable work, or check it. Its attempt's `jobs:execute-job` flow ends as failed at once: `Did('execute')` and `Did('validateOutput')` don't run, the job's `cleanup` runs with `'timeout'`, and the finalize hooks see the `WorkflowJobTimeoutError` as `state.flowError`. Up to 1.10 the flow waited for `execute()` to return first; up to 1.9.4 the flow of such an attempt ran on as a success, so audit and metrics hooks recorded a success for a step the workflow reported as failed. Each step attempt also waits for a slot under `jobs.admission` (see `create-job`). `cancel_job` cancels job runs only: a workflow run is refused with `JOB_RUN_NOT_CANCELLABLE`. +When a step's `timeout` passes, the step fails with `WorkflowJobTimeoutError`. An attempt whose job has not started never starts it. A job already in `execute()` is not stopped: its `this.signal` (`ctx.signal` in a `job()` function) is aborted with the timeout as its reason, so pass it to `fetch` and other cancellable work, or check it. Its attempt's `jobs:execute-job` flow ends as failed at once: `Did('execute')` and `Did('validateOutput')` don't run, the job's `cleanup` runs with `'timeout'`, and the finalize hooks see the `WorkflowJobTimeoutError` as `state.flowError`. Up to 1.10 the flow waited for `execute()` to return first; up to 1.9.4 the flow of such an attempt ran on as a success, so audit and metrics hooks recorded a success for a step the workflow reported as failed. Each step attempt also waits for a slot under `jobs.admission` (see `create-job`). `cancel_job` cancels job runs only: a workflow run is refused with `JOB_RUN_NOT_CANCELLABLE`. A step whose job sets `runner: 'worker'` runs its `execute()` on the offload pool; a job on the `'detached'` runner runs inline as a step, in the workflow's process, since the workflow run owns its steps. ### Basic Example diff --git a/libs/skills/catalog/frontmcp-development/references/offload-cpu-work.md b/libs/skills/catalog/frontmcp-development/references/offload-cpu-work.md index 25db57b79..4b4d591f3 100644 --- a/libs/skills/catalog/frontmcp-development/references/offload-cpu-work.md +++ b/libs/skills/catalog/frontmcp-development/references/offload-cpu-work.md @@ -13,8 +13,10 @@ such work in a pool of `worker_threads` instead: an exported function of another module in a worker and resolves with its result. - `@Resource({ worker: true })` (and `@ResourceTemplate`, `resource()`, `resourceTemplate()`) runs the resource's whole read in a worker. +- `@Job({ runner: 'worker' })` (or `jobs.runner: 'worker'`) runs a job's whole `execute()` in a worker, under the same + rules as a worker resource (see `create-job`). -Both are Node only. The pool starts its first worker on the first call that needs one, and shuts down with the server. +All three are Node only. The pool starts its first worker on the first call that needs one, and shuts down with the server. ## When to Use This Skill diff --git a/libs/skills/catalog/frontmcp-production-readiness/references/common-checklist.md b/libs/skills/catalog/frontmcp-production-readiness/references/common-checklist.md index ed5795ff2..00d5c8fcb 100644 --- a/libs/skills/catalog/frontmcp-production-readiness/references/common-checklist.md +++ b/libs/skills/catalog/frontmcp-production-readiness/references/common-checklist.md @@ -146,6 +146,8 @@ These checks apply to ALL deployment targets. Run them first, then proceed to yo - [ ] Jobs keep their runs in a shared store for production: `jobs.store.redis` across machines, `jobs.store.sqlite` on one machine, or the top-level `redis` / `sqlite` they fall back to - [ ] Long jobs set `timeout` (or `jobs.defaultTimeoutMs`) and a `cleanup` for what they start; `jobs.admission` bounds heavy jobs (`concurrency`, `memoryMb`) +- [ ] CPU-heavy jobs use `runner: 'worker'`; work that must finish across a restart uses `runner: 'detached'` on a shared store, from a server started from its entry (not on serverless or edge) +- [ ] Detached runners are allowed to outlive the server (a container's restart stops them with it); `reapOrphans(JOB_WORKER_MARKER)` stops the ones left behind by a server that is gone for good - [ ] Job retry config has reasonable `maxAttempts` and `maxBackoffMs` - [ ] Workflow timeout is set to prevent runaway workflows - [ ] Job execution times are monitored (long-running jobs need alerting) diff --git a/libs/skills/catalog/skills-manifest.json b/libs/skills/catalog/skills-manifest.json index cf287e136..94b56c41f 100644 --- a/libs/skills/catalog/skills-manifest.json +++ b/libs/skills/catalog/skills-manifest.json @@ -1399,11 +1399,6 @@ } ] }, - { - "name": "build-manifest", - "description": "The frontmcp.manifest.json that frontmcp build writes for every target, its lint, and the instructionsBudget size limits", - "examples": [] - }, { "name": "build-for-sdk", "description": "Build a FrontMCP server as an embeddable library with create() and connect() APIs", @@ -1443,6 +1438,11 @@ } ] }, + { + "name": "build-manifest", + "description": "The frontmcp.manifest.json that frontmcp build writes for every target, its lint, and the instructionsBudget size limits", + "examples": [] + }, { "name": "deploy-manifest-yaml", "description": "The frontmcp.deploy.yaml v1 schema — declarative manifest the GitHub Action consumes on every push to build, sign, and hot-reload the Cloudflare Worker", @@ -1853,7 +1853,7 @@ }, { "name": "create-job", - "description": "Create long-running background jobs with retry policies, timeouts, cancellation, cleanup, progress tracking, and permissions", + "description": "Create long-running background jobs with retry policies, timeouts, cancellation, cleanup, progress tracking, permissions, and runners (inline, worker thread, or a detached process that outlives the server)", "examples": [ { "name": "basic-report-job", @@ -1866,6 +1866,19 @@ "Using `this.log()` for persistent, queryable log entries" ] }, + { + "name": "detached-export-job", + "description": "An export job that runs in a process of its own, survives a server restart and is cancelled from any server.", + "level": "advanced", + "tags": ["development", "job", "detached", "worker", "sqlite", "cancel"], + "features": [ + "`runner: 'detached'`: each run in a process of its own that outlives the server that started it", + "A store the runner and every server share (the top-level `sqlite`), which the runs live in", + "Reading and cancelling a run from a server started after it, and a `cleanup` the runner runs on a cancel", + "Runs whose runner died settled as `interrupted` when the next server boots, with their cleanup", + "`runner: 'worker'` for a CPU-heavy job that runs in a worker thread instead" + ] + }, { "name": "durable-build-job", "description": "A build job on a SQLite store with a timeout, a cleanup, admission limits, cancel and a subscribed run handle.",