From 901f9e17f42a5ad9764dea30adf009c389449966 Mon Sep 17 00:00:00 2001 From: David Antoon Date: Mon, 2 Nov 2026 08:39:10 +0200 Subject: [PATCH 1/6] feat(storage): a local SQLite store several processes can share - SqliteStorageAdapter: conditional set, delete, incr, deleteIfEquals and expireIfEquals each run in one BEGIN IMMEDIATE transaction, so they are atomic across processes; expire and expireIfEquals validate the TTL and treat an expired key as missing, like the memory and Redis adapters. DistributedSemaphore now works on SQLite. - One connection per database file per process, shared by every store on the file, with a queue that runs the adapter's operations in order and waits for another process's lock on a timer instead of blocking the event loop. - createStorage({ type: 'sqlite', sqlite: { path } }) loads @frontmcp/storage-sqlite on demand; an invalid sqlite block throws StorageConfigError whatever the fallback. - SDK: sqliteStorageProvider() and SqliteStorageToken give providers and tools a SQLite-backed storage, disconnected at shutdown. - New e2e app demo-e2e-local-runtime: ten processes contend on one file. --- .../e2e/dts-consumer.e2e.spec.ts | 5 + .../fixture/sqlite-storage-must-compile.ts | 43 ++++ .../fixture/tsconfig.json | 1 + .../e2e/helpers/workers.ts | 83 +++++++ .../e2e/sqlite-contention.e2e.spec.ts | 78 ++++++ .../e2e/sqlite-storage-provider.e2e.spec.ts | 43 ++++ .../fixture/container-worker.ts | 29 +++ .../fixture/lock-holder.ts | 22 ++ .../fixture/storage-worker.ts | 72 ++++++ .../fixture/worker-protocol.ts | 50 ++++ .../demo-e2e-local-runtime/jest.e2e.config.ts | 30 +++ apps/e2e/demo-e2e-local-runtime/project.json | 32 +++ .../demo-e2e-local-runtime/tsconfig.app.json | 10 + .../demo-e2e-local-runtime/tsconfig.e2e.json | 8 + apps/e2e/demo-e2e-local-runtime/tsconfig.json | 15 ++ libs/guard/src/concurrency/README.md | 2 +- libs/guard/src/errors/errors.ts | 2 +- libs/sdk/src/index.ts | 2 + .../__tests__/sqlite-storage.provider.spec.ts | 80 ++++++ .../src/storage/sqlite-storage.provider.ts | 31 +++ .../create-tool/references/throttling.md | 2 +- .../references/configure-throttle.md | 16 +- libs/skills/catalog/frontmcp-setup/SKILL.md | 3 +- .../setup-sqlite/basic-sqlite-setup.md | 3 +- .../setup-sqlite/encrypted-sqlite-storage.md | 1 + .../setup-sqlite/multi-process-storage.md | 81 ++++++ .../setup-sqlite/unix-socket-daemon.md | 1 + .../frontmcp-setup/references/setup-sqlite.md | 103 ++++++-- libs/skills/catalog/skills-manifest.json | 14 +- libs/storage-sqlite/jest.config.ts | 1 + .../src/__tests__/sqlite-connection.spec.ts | 235 ++++++++++++++++++ .../src/__tests__/sqlite-semaphore.spec.ts | 67 +++++ .../__tests__/sqlite-storage.adapter.spec.ts | 139 +++++++++++ libs/storage-sqlite/src/index.ts | 3 +- libs/storage-sqlite/src/open-database.ts | 3 +- libs/storage-sqlite/src/sqlite-connection.ts | 124 +++++++++ libs/storage-sqlite/src/sqlite-event.store.ts | 14 +- libs/storage-sqlite/src/sqlite-kv.store.ts | 28 ++- .../src/sqlite-storage.adapter.ts | 93 ++++--- libs/storage-sqlite/src/sqlite-task.store.ts | 10 +- libs/utils/jest.config.ts | 6 + libs/utils/src/index.ts | 1 + .../src/storage/__tests__/factory.spec.ts | 85 +++++++ libs/utils/src/storage/adapters/base.ts | 4 +- libs/utils/src/storage/factory.ts | 55 +++- libs/utils/src/storage/index.ts | 1 + libs/utils/src/storage/types.ts | 25 +- 47 files changed, 1667 insertions(+), 89 deletions(-) create mode 100644 apps/e2e/demo-e2e-dts-consumer/fixture/sqlite-storage-must-compile.ts create mode 100644 apps/e2e/demo-e2e-local-runtime/e2e/helpers/workers.ts create mode 100644 apps/e2e/demo-e2e-local-runtime/e2e/sqlite-contention.e2e.spec.ts create mode 100644 apps/e2e/demo-e2e-local-runtime/e2e/sqlite-storage-provider.e2e.spec.ts create mode 100644 apps/e2e/demo-e2e-local-runtime/fixture/container-worker.ts create mode 100644 apps/e2e/demo-e2e-local-runtime/fixture/lock-holder.ts create mode 100644 apps/e2e/demo-e2e-local-runtime/fixture/storage-worker.ts create mode 100644 apps/e2e/demo-e2e-local-runtime/fixture/worker-protocol.ts create mode 100644 apps/e2e/demo-e2e-local-runtime/jest.e2e.config.ts create mode 100644 apps/e2e/demo-e2e-local-runtime/project.json create mode 100644 apps/e2e/demo-e2e-local-runtime/tsconfig.app.json create mode 100644 apps/e2e/demo-e2e-local-runtime/tsconfig.e2e.json create mode 100644 apps/e2e/demo-e2e-local-runtime/tsconfig.json create mode 100644 libs/sdk/src/storage/__tests__/sqlite-storage.provider.spec.ts create mode 100644 libs/sdk/src/storage/sqlite-storage.provider.ts create mode 100644 libs/skills/catalog/frontmcp-setup/examples/setup-sqlite/multi-process-storage.md create mode 100644 libs/storage-sqlite/src/__tests__/sqlite-connection.spec.ts create mode 100644 libs/storage-sqlite/src/__tests__/sqlite-semaphore.spec.ts create mode 100644 libs/storage-sqlite/src/sqlite-connection.ts diff --git a/apps/e2e/demo-e2e-dts-consumer/e2e/dts-consumer.e2e.spec.ts b/apps/e2e/demo-e2e-dts-consumer/e2e/dts-consumer.e2e.spec.ts index 585e3250e..9b617259b 100644 --- a/apps/e2e/demo-e2e-dts-consumer/e2e/dts-consumer.e2e.spec.ts +++ b/apps/e2e/demo-e2e-dts-consumer/e2e/dts-consumer.e2e.spec.ts @@ -26,6 +26,7 @@ function toFixtureDiagnostic(diagnostic: ts.Diagnostic): FixtureDiagnostic { const fixtureFiles = [ 'provider-lifecycle-must-compile.ts', + 'sqlite-storage-must-compile.ts', 'tool-must-fail.ts', 'prompts-must-compile.ts', 'hooks-must-compile.ts', @@ -91,6 +92,10 @@ describe('@frontmcp/sdk declarations in a strict consumer project', () => { expect(describeEach(inFixtureFile('provider-lifecycle-must-compile.ts'))).toEqual([]); }); + it('types sqliteStorageProvider() and the storage SqliteStorageToken resolves to', () => { + expect(describeEach(inFixtureFile('sqlite-storage-must-compile.ts'))).toEqual([]); + }); + it('accepts the documented @Prompt forms', () => { expect(describeEach(inFixtureFile('prompts-must-compile.ts'))).toEqual([]); }); diff --git a/apps/e2e/demo-e2e-dts-consumer/fixture/sqlite-storage-must-compile.ts b/apps/e2e/demo-e2e-dts-consumer/fixture/sqlite-storage-must-compile.ts new file mode 100644 index 000000000..cf5996bb9 --- /dev/null +++ b/apps/e2e/demo-e2e-dts-consumer/fixture/sqlite-storage-must-compile.ts @@ -0,0 +1,43 @@ +import 'reflect-metadata'; + +import { + App, + createContainer, + FrontMcp, + sqliteStorageProvider, + SqliteStorageToken, + type SqliteStorageProviderOptions, +} from '@frontmcp/sdk'; + +const options: SqliteStorageProviderOptions = { + path: '/var/lib/counter/state.sqlite', + walMode: true, + busyTimeoutMs: 10_000, + prefix: 'counter:', +}; + +@App({ name: 'counter' }) +class CounterApp {} + +@FrontMcp({ + info: { name: 'sqlite-storage', version: '1.0.0' }, + apps: [CounterApp], + providers: [sqliteStorageProvider(options)], +}) +export class SqliteStorageServer {} + +// @ts-expect-error the database file's path is required +sqliteStorageProvider({ walMode: true }); + +// @ts-expect-error walMode is a boolean +sqliteStorageProvider({ path: '/var/lib/counter/state.sqlite', walMode: 'yes' }); + +export async function countVisit(): Promise { + const container = await createContainer({ providers: [sqliteStorageProvider(options)] }); + try { + const visits: number = await container.get(SqliteStorageToken).incr('visits'); + return visits; + } finally { + await container.dispose(); + } +} diff --git a/apps/e2e/demo-e2e-dts-consumer/fixture/tsconfig.json b/apps/e2e/demo-e2e-dts-consumer/fixture/tsconfig.json index 200062d44..623d6dfc3 100644 --- a/apps/e2e/demo-e2e-dts-consumer/fixture/tsconfig.json +++ b/apps/e2e/demo-e2e-dts-consumer/fixture/tsconfig.json @@ -12,6 +12,7 @@ }, "files": [ "provider-lifecycle-must-compile.ts", + "sqlite-storage-must-compile.ts", "tool-must-fail.ts", "prompts-must-compile.ts", "hooks-must-compile.ts", diff --git a/apps/e2e/demo-e2e-local-runtime/e2e/helpers/workers.ts b/apps/e2e/demo-e2e-local-runtime/e2e/helpers/workers.ts new file mode 100644 index 000000000..59d896c57 --- /dev/null +++ b/apps/e2e/demo-e2e-local-runtime/e2e/helpers/workers.ts @@ -0,0 +1,83 @@ +/** + * Start fixture scripts as child processes that follow `fixture/worker-protocol.ts`, and drive them + * from a spec: wait for a line a worker prints, send it a line, read the result it reports. + */ +import { spawn, type ChildProcess } from 'node:child_process'; +import { join } from 'node:path'; + +const projectRoot = join(__dirname, '..', '..'); + +export interface Worker { + readonly child: ChildProcess; + /** Resolves when the worker prints `line`; rejects if it exits first. */ + waitFor(line: string): Promise; + send(line: string): void; + /** What the worker reported; rejects when it exits with a code other than 0. */ + readonly result: Promise; +} + +/** Run `fixture/