diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5e4e46..d9b8600 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -125,13 +125,13 @@ jobs: with: go-version: '1.27.x' cache: true - cache-dependency-path: tools/golden/go.sum + cache-dependency-path: packages/sqlite-js-stream/tools/golden/go.sum - name: Generate Go LTX goldens - run: cd tools/golden && go run . + run: cd packages/sqlite-js-stream/tools/golden && go run . - name: Verify goldens are up to date - run: git diff --exit-code -- testdata/go + run: git diff --exit-code -- packages/sqlite-js-stream/testdata/go - name: Setup pnpm uses: pnpm/action-setup@v6 @@ -146,4 +146,4 @@ jobs: run: pnpm install --frozen-lockfile - name: Decode Go LTX files with the TS decoder - run: node --test test/ltx/go-interop.test.ts + run: node --test packages/sqlite-js-stream/test/ltx/go-interop.test.ts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 05e54bc..0a716c3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -68,7 +68,7 @@ jobs: VERSION_INPUT: ${{ inputs.version }} run: | node scripts/bump-version.mjs "$VERSION_INPUT" "$GITHUB_ACTOR" - version=$(node -p "require('./package.json').version") + version=$(node -p "require('./packages/sqlite-js-stream/package.json').version") printf 'version=%s\n' "$version" >> "$GITHUB_OUTPUT" - name: Push changes diff --git a/README.md b/README.md index 47e5b53..b330afa 100644 --- a/README.md +++ b/README.md @@ -1,790 +1,39 @@ # sqlite-js-stream -A **Node.js port of [Litestream](https://github.com/benbjohnson/litestream)** — SQLite streaming -replication and disaster recovery — written in **TypeScript** and built on the built-in `node:sqlite`. +A **Node.js port of [Litestream](https://github.com/benbjohnson/litestream)** — +SQLite streaming replication and disaster recovery — written in **TypeScript** +and built on the built-in `node:sqlite`. -SQLite is the most widely deployed database on the planet but it only lives on a single machine. -If that machine dies, your database is gone. This library replicates a SQLite database's **WAL -(Write-Ahead Log)** file into immutable, chronological **LTX (Log Transaction)** files on a remote -replica (S3, a local directory, etc.), and can **restore** a database from those files. +This repository is a [pnpm](https://pnpm.io) workspace. The packages live under +`packages/`: ---- +| Package | Description | +| --- | --- | +| [`@platformatic/sqlite-js-stream`](packages/sqlite-js-stream) | The core library: SQLite streaming replication, LTX files, sync/restore, paged restores. | +| [`@platformatic/sqlite-js-stream-s3`](packages/backend-s3) | Amazon S3 upload strategy (replica client). | +| [`@platformatic/sqlite-js-stream-pg`](packages/backend-pg) | PostgreSQL upload strategy (replica client). | +| [`@platformatic/sqlite-js-stream-paged-vfs`](packages/sqlite-js-stream-paged-vfs) | A native SQLite VFS that reads database pages on demand. | ## Requirements - **Node.js `>= 24`** — required for the built-in `node:sqlite`. -- Runtime dependency: `lz4js` (pure-JS LZ4 **block** decoder), matching the LTX format. - Compression is a byte-identical TypeScript port of Go's `pierrec/lz4/v4` Fast block compressor. +- [pnpm](https://pnpm.io) to install and run the workspace. -The **published** package ships compiled JavaScript plus TypeScript declarations -(`dist/*.js` + `dist/*.d.ts`), built at publish time via `pnpm build`. - ---- - -## Installation - -This repository is a **pnpm workspace**. The core library lives at the root and -upload strategies (replica backends) live under `packages/`: +## Development ```bash pnpm install # link the workspace -pnpm build # transpile src -> dist (.js + .d.ts), used by addons/tests -``` - -The core package is published as **`@platformatic/sqlite-js-stream`** and the S3 -and PostgreSQL addons as **`@platformatic/sqlite-js-stream-s3`** and -**`@platformatic/sqlite-js-stream-pg`**: - -```bash -pnpm add @platformatic/sqlite-js-stream -pnpm add @platformatic/sqlite-js-stream-s3 # optional addon -pnpm add @platformatic/sqlite-js-stream-pg # optional addon +pnpm build # transpile the core package src -> dist +pnpm typecheck:all # typecheck every package +pnpm lint # ESLint + neostandard, with a type-stripping guard +pnpm test:all # run the core + addon test suites +pnpm test:paged-vfs # build & test the native paged VFS ``` -Importing the addon registers the `s3` URL scheme with -`newReplicaClientFromURL('s3://bucket/prefix')`. - -**`@platformatic/sqlite-js-stream-paged-vfs`** is a SQLite VFS for `node:sqlite` -which reads the pages of a database on demand, so that an application can query -a large database while it is still being restored. It is a native library, with -prebuilt binaries for macOS and Linux. `replica.restorePaged()` uses it when it is -installed (see [Restore page by page](#restore-page-by-page)). See its -[README](packages/sqlite-js-stream-paged-vfs/README.md). - -```bash -pnpm add @platformatic/sqlite-js-stream-paged-vfs -``` - ---- - -## Quick start - -```ts -import { Replica, FileReplicaClient } from '@platformatic/sqlite-js-stream'; - -// 1. Point the replica at your database path and a replica destination. -// The library reads the database + `-wal` directly and is agnostic to -// whichever SQLite driver your application uses. -const client = new FileReplicaClient('/data/replica'); -const replica = new Replica('/data/app.db', client); - -// 2. Sync builds a local LTX file (into -litestream/ltx/0/) and uploads it. -await replica.sync(); -``` - -The database is written to by your **own** SQLite connection (using `node:sqlite`, `better-sqlite3`, -or anything else). After more writes, just `sync()` again: - -```ts -// ... your app writes rows with whatever driver you use ... -await replica.sync(); - -const status = await replica.syncStatus(); -console.log(status.inSync, status.localTXID, status.remoteTXID); // true 3n 3n -``` - -### Take a full snapshot - -Normal `sync()` writes **incremental** level-0 files. For a complete, point-in-time backup, take a -**level-9 snapshot** (a full database copy as an LTX file with `MinTXID = 1`): - -```ts -await replica.snapshot(); -``` - -### Restore a database - -Restore reconstructs a SQLite database by compacting a level-9 snapshot with the subsequent -level-0 files: - -```ts -await replica.restore({ outputPath: '/data/restored.db' }); -``` - -Restore to a specific transaction ID, or to a point-in-time (the latest backup created at/before -the given time). Pass exactly one of `txID` or `timestamp`: - -```ts -await replica.restore({ outputPath: '/data/app-at-txid.db', txID: 42n }); -await replica.restore({ outputPath: '/data/app-at-time.db', timestamp: new Date('2025-06-01T12:00:00Z') }); -``` - -Restoring to `txID`/`timestamp` is at **backup boundaries** (the end of an LTX file), and a target -with no matching backup throws `NoMatchingBackupError`. A "latest" restore over a non-contiguous -file set throws `RestoreGapError` rather than silently returning a partial database. - -### Continuously restore (follow) - -`follow()` restores the latest backup, then keeps the database up to date as new backups are -replicated — maintaining a read-only replica (litestream's `restore -f`): - -```ts -const ac = new AbortController(); -await replica.follow({ outputPath: '/data/replica.db', intervalMs: 1000, signal: ac.signal }); -// later: ac.abort() to stop -``` - -The applied TXID is recorded in a `-txid` sidecar (default `-txid`) so a later `follow()` -resumes instead of re-restoring. The output database should only be opened **read-only**. - -### Restore the local database if it is missing - -```ts -await replica.ensureExists(); -``` - -Restoring a replica's own database (`outputPath` equal to its path, as -`ensureExists()` does) makes it the writer from the restored position: the local -LTX cache of any previous database at that path is discarded, and the next -`sync()` uploads the following TXID. Any `-wal` and `-shm` files left at the -output path are removed before the restored database is moved into place. -`sync()` throws `ErrReplicaAhead` (`ERR_REPLICA_AHEAD`) rather than skipping -uploads when the replica already has a higher TXID than the local database. - -The restore also writes the first WAL of the restored database, under salts it -records, so that the next `sync()` uploads only the changes made since the -restore instead of a snapshot of the whole database. That holds while the WAL is -not reset: a checkpoint that restarts or truncates it, or the last connection -closing (which deletes it), makes the next `sync()` fall back to a snapshot. -With `wal_autocheckpoint` disabled and checkpoints run through -`replica.checkpoint()` (see below), syncs stay incremental from the restore on. - -### Restore page by page - -A full restore downloads the whole database before it can be opened. With -`@platformatic/sqlite-js-stream-paged-vfs` installed, `restorePaged()` restores -the replica's own database page by page instead: it resolves once the database -can be opened, and queries read the pages they need on demand while the others -are downloaded in the background. - -```ts -import { databaseUri } from '@platformatic/sqlite-js-stream-paged-vfs'; - -const result = await replica.restorePaged({ minSize: 64 * 1024 * 1024 }); -// In a worker thread, not this one: -const db = new DatabaseSync(databaseUri('/data/app.db')); - -await result.filled; // every page is local -``` - -- The database is created with every page missing. A query that reads a missing - page waits while it is read from the replica, with the next pages of the same - LTX file (`readAhead`, 32 by default). Page 1 and the schema are read before - `restorePaged()` resolves. -- Open the database with the URI from `databaseUri()`, from **another thread** - than the one that called `restorePaged()`: that thread serves the pages, so a - query from it would wait for itself (such reads fail instead). Every - connection that writes the database must use the VFS. -- Replication goes on meanwhile. `sync()` and `snapshot()` read the pages they - need first, and retention keeps the LTX files the restore still reads. -- `result.filled` resolves once every page is local. `replica.close()`, or - restoring the database again, stops the restore: the database stays - incomplete. So does a crash. An incomplete database has a `localPos()` of 0, - and `sync()`/`snapshot()` throw `ErrIncompleteDatabase` until it is restored - again (`ensureExists()` does). -- It restores fully, like `restore()`, when the LTX files to download add up - to less than `minSize` bytes, or when the paged VFS cannot be used - (`result.unavailable` holds the reason: the package is not installed, has no - binary for the platform, or Node.js runs with `--permission`). - `result.paged` tells which. -- `onMiss(pgno, pages, durationMs)` is called for each page a query waited for, - for metrics. - -### Manage many databases - -```ts -import { Store } from '@platformatic/sqlite-js-stream'; - -const store = new Store(); -await store.openDB('/data/a.db', new FileReplicaClient('/data/replica-a')); -await store.openDB('/data/b.db', new FileReplicaClient('/data/replica-b')); - -await store.syncAll(); // local sync + upload for every database -await store.snapshotAll(); // level-9 snapshots for every database - -store.close(); -``` - -### Replica URL - -Replica clients are created from URLs via the factory registry: - -```ts -import { newReplicaClientFromURL, Replica } from '@platformatic/sqlite-js-stream'; - -const client = newReplicaClientFromURL('file:///data/replica'); -const replica = new Replica('/data/app.db', client); -``` - -The `file` scheme is built into the core. Additional schemes (S3, etc.) are -registered by importing the matching addon (see below). - -### Custom read connection - -During sync and snapshot, a replica pins the WAL with a short read-only -`node:sqlite` transaction. To use another driver for that connection, e.g. for -libsql-encrypted databases that `node:sqlite` cannot open, pass `openReader`: - -```js -import Database from 'libsql'; - -const replica = new Replica('/data/app.db', client, undefined, { - openReader: (path) => new Database(path, { readonly: true, encryptionKey: process.env.DB_KEY }), -}); -``` - -Pages are copied as-is, so the replica stays encrypted and a restored database -opens with the same key. - -### S3 upload strategy - -The `@platformatic/sqlite-js-stream-s3` addon provides an S3 replica client. Import it -once to register the `s3://` scheme, then create clients from URLs or directly: - -```ts -import { Replica, newReplicaClientFromURL } from '@platformatic/sqlite-js-stream'; -import { S3ReplicaClient } from '@platformatic/sqlite-js-stream-s3'; - -// Via a URL (bucket from the host, prefix from the path). -const client = newReplicaClientFromURL('s3://my-bucket/backup?region=us-east-1'); - -// Or directly, with explicit AWS config. -const s3 = new S3ReplicaClient({ - bucket: 'my-bucket', - prefix: 'backup', - region: 'us-east-1', - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, -}); - -const replica = new Replica('/data/app.db', client); -await replica.run({ intervalMs: 1000 }); -``` - -Objects are stored as `{prefix}/ltx/{level}/{minTXID}-{maxTXID}.ltx` — the same -layout as the file backend, so a replica can move between strategies. Creation -timestamps are preserved as the `ltx-timestamp` object metadata. - -> **Testing without AWS:** `S3ReplicaClient` accepts an injected `client` -> implementing `send(command)`. Pass an in-memory fake (see -> `packages/backend-s3/test/helpers/memory-s3.ts`) to exercise the addon without -> network access. - -### PostgreSQL upload strategy - -The `@platformatic/sqlite-js-stream-pg` addon provides a PostgreSQL replica client. -Import it once to register the `postgres://` and `postgresql://` schemes, then -create clients from URLs or directly: - -```ts -import { Replica, newReplicaClientFromURL } from '@platformatic/sqlite-js-stream'; -import { PgReplicaClient, PgLeaser } from '@platformatic/sqlite-js-stream-pg'; - -// Via a URL: connection details from the URL, replica prefix from `?prefix=`. -const client = newReplicaClientFromURL( - 'postgres://user:pass@host:5432/mydb?prefix=app/db1', -); - -// Or directly, with an explicit connection string. -const pg = new PgReplicaClient({ - url: 'postgres://user:pass@host:5432/mydb', - prefix: 'app/db1', -}); - -const replica = new Replica('/data/app.db', client); -await replica.run({ intervalMs: 1000 }); -``` - -Files are stored as one row in `ltx_files` plus one row per 1 MiB chunk in -`ltx_chunks`, scoped by a `prefix` column, in a configurable schema (`?schema=`). -Every file is written atomically (row + chunks in a single transaction), so a -reader never sees a partially-written file. `created_at` preserves the LTX -creation timestamp for point-in-time restore and retention. - -> **Fenced writes:** see [Fenced writes](#fenced-writes). - -> **Distributed leasing:** `PgLeaser` acquires/renews/releases a lease on a -> `ltx_leases` row using a compare-and-swap `version` column, with the same -> semantics as `S3Leaser` (`LeaseExistsError`, `ErrLeaseNotHeld`). Pass it to a -> `Replica` via the `leaser` option so only one host writes at a time. - -### Fenced writes - -When the application elects the writer of a database itself (with its own lease, -or a `Leaser`), a former writer that was paused (GC, a stopped VM, a network -partition) can wake up after losing the lease and keep uploading. The S3 and -PostgreSQL clients can reject those uploads: give every writer a *generation* -that only grows (for example a fencing token from your lease), and follow this -order when a writer takes over a database: - -```ts -import { Replica } from '@platformatic/sqlite-js-stream'; -import { S3ReplicaClient } from '@platformatic/sqlite-js-stream-s3'; - -const client = new S3ReplicaClient({ bucket: 'my-bucket', prefix: 'app/db1' }); - -// 1. Register: fails with ErrGenerationSuperseded if a higher generation is -// registered. From now on, older writers' uploads and deletions fail. -await client.registerGeneration(token); - -// 2. Restore (or keep the local database if it is at the replica's latest TXID). -const replica = new Replica('/data/app.db', client); -await replica.restore({ outputPath: '/data/app.db' }); - -// 3. Write and replicate. -await replica.sync(); -``` - -`registerGeneration(n)` sets the client's generation (the `generation` option -does the same without registering). From then on: - -- An upload, compaction or retention deletion fails with - `ErrGenerationSuperseded` (`ERR_GENERATION_SUPERSEDED`) once a higher - generation is registered. The `Replica` then stops writing: every later - `sync()`, `snapshot()`, compaction and retention call fails with the same - error (also in `replica.superseded`), and `run()` stops and rejects with it - instead of retrying. -- An LTX file of a given level and TXID range is written only once. - -**PostgreSQL** checks the generation in the transaction that writes or deletes -the files (`SELECT … FOR SHARE` on the `ltx_generations` row), so registering -waits for the uploads in flight and the following ones fail. - -**S3** can't check one object and write another in one request, so fencing is a -protocol of conditional writes on a `generation.json` object next to the `ltx/` -directory: - -- `registerGeneration(n)` raises the generation with `If-Match`/`If-None-Match`, - then reads the replica's latest TXID `T` and records the *fence* `(n, T)` in - the same object. It does both, so there is no separate step after the restore. -- Fenced uploads use `If-None-Match: *` and store their generation in the - `ltx-generation` metadata. After each upload, the client reads - `generation.json` and fails if a higher generation is registered. An upload - that resolves was therefore made before any newer writer registered, and - every newer writer's restore sees it. -- An upload of a superseded writer can still land if it was in flight. Listings - ignore a file whose generation is lower than a fence's and whose `maxTXID` is - above that fence's `T`. If such a file holds a key that the new writer needs, - the new writer replaces it (`If-Match` on its ETag). -- Deletions check `generation.json` right before `DeleteObjects`. A deletion - sent just before a new generation registers still applies; this is the one - race the protocol leaves open, and it needs the old writer to pause between - the check and the request. -- Once a fence exists, listing a file that ends above the lowest fence costs - one `HeadObject` request to read its generation (cached per key and ETag; a - writer knows its own files). Replicas that never registered a generation pay - one `GetObject` of `generation.json` per listing (a 404). -- `deleteAll()` keeps `generation.json`, so a superseded writer stays fenced. - -The file backend does not implement fenced writes. - -### Deleting every replica under a prefix - -`deleteAll()` deletes the files of one replica and keeps its generation, so a -superseded writer stays fenced. To delete a whole group of replicas, for -example every database of a tenant, each client has a static `deletePrefix()`. -It deletes the replica at the prefix and every replica under `prefix/` (`a` -covers `a` and `a/b`, not `ab`), with their generations and leases, and resolves -with the sorted prefixes (paths for the file backend) of the deleted replicas: - -```ts -import { FileReplicaClient } from '@platformatic/sqlite-js-stream'; -import { PgReplicaClient } from '@platformatic/sqlite-js-stream-pg'; -import { S3ReplicaClient } from '@platformatic/sqlite-js-stream-s3'; - -await FileReplicaClient.deletePrefix({ path: '/backups/tenant-1' }); -await PgReplicaClient.deletePrefix({ sql, prefix: 'tenant-1', tablePrefix: 'ltx' }); -await S3ReplicaClient.deletePrefix({ bucket: 'my-bucket', prefix: 'tenant-1', client }); -// => ['tenant-1/db1', 'tenant-1/db2'] -``` - -The options take the same connection settings as the client's constructor (`url` -or `sql`, `schema` and `tablePrefix` for PostgreSQL; `client` or `region`, -`endpoint` and credentials for S3). An empty prefix is rejected. - -With a `generation` option, the deletion is fenced like the deletions of a fenced -client: it fails with `ErrGenerationSuperseded` if a replica under the prefix has -a higher generation. - -- **PostgreSQL** deletes everything in one transaction that first locks the - generation rows (`FOR UPDATE`): it waits for the fenced uploads and deletions - in flight, the ones that start meanwhile wait for it, and a superseded - deletion deletes nothing. Pass a transaction as `sql` (the argument of a - `sql.begin()` callback) to run the deletion in a savepoint of it, so it commits - or rolls back with the caller's own changes. -- **S3** is best effort: the generation of each replica is checked before - deleting its first object and again before the generation objects, which are - deleted last, so former writers stay fenced while the files are deleted. A - generation registered between a check and a `DeleteObjects` request does not - stop the deletion, and a superseded deletion keeps what it already deleted. -- **The file backend** has no generations: the directory is removed. - -Once the generations are deleted, nothing fences the former writers of these -replicas: a writer still running can upload again and create its replica anew. -Stop the writers first. - -### Continuous sync - -For ongoing replication, `replica.run()` syncs immediately and then repeats on an -interval until the supplied `AbortSignal` is aborted. Sync errors are logged and the -loop keeps going, so a transient failure doesn't stop replication: - -```ts -import { Replica, FileReplicaClient } from '@platformatic/sqlite-js-stream'; - -const replica = new Replica('/data/app.db', new FileReplicaClient('/data/replica')); -const ac = new AbortController(); - -// Run continuously (sync now, then every 1s); resolves when aborted. -const runP = replica.run({ intervalMs: 1000, signal: ac.signal }); - -// ... later, to stop: -ac.abort(); -await runP; -``` - -If you want to keep replicating in the background without awaiting, call `run()` and -`catch` its promise (it resolves on abort): - -```ts -replica.run({ intervalMs: 1000, signal: ac.signal }).catch((err) => { - console.error('continuous sync stopped', err); -}); -``` - -`Store` also exposes `syncAll()`/`snapshotAll()` for one-shot operations; a continuous -loop can be driven by calling `replica.run()` per database, or via your own scheduler. - -### Checkpoints without full snapshots - -When the WAL is reset by a checkpoint, the next `sync()` cannot resume from its previous position -in the WAL, so it uploads a full snapshot of the database. An application that decides when -checkpoints run can avoid that with `replica.checkpoint()`: it syncs, checkpoints the WAL, and -records that the next WAL continues from the synced position, so the next `sync()` uploads only the -changes. - -```ts -import { DatabaseSync } from 'node:sqlite'; - -const db = new DatabaseSync('/data/app.db'); -db.exec('PRAGMA journal_mode=WAL'); -db.exec('PRAGMA wal_autocheckpoint=0'); // checkpoints run only through replica.checkpoint() - -// ... write with db, and call replica.sync() as usual ... - -// Instead of db.exec('PRAGMA wal_checkpoint(TRUNCATE)'): -const incremental = await replica.checkpoint(db, 'TRUNCATE'); -``` - -`checkpoint(conn, mode = 'TRUNCATE')` takes the connection the application writes with (`mode` is -`'RESTART'` or `'TRUNCATE'`): - -- Pass the **writing connection**, with no open transaction. SQLite gives the next WAL a predictable - salt only when the connection that starts it is the one that reset the WAL; that is how the next - `sync()` recognizes the WAL that directly followed the checkpoint. -- Disable `wal_autocheckpoint` on every connection and never run `PRAGMA wal_checkpoint` yourself: - any other reset is detected and the next `sync()` falls back to a full snapshot (correct, but - not incremental). -- Do not call it concurrently with `sync()` or `run()`. -- The next WAL must be started by the same connection: a different connection that writes first - picks random salts, and the next `sync()` falls back to a snapshot. - -When replication runs on a different thread (or process) than the writing connection, which -cannot cross threads, split the call in two. `replica.prepareCheckpoint()` syncs and returns a -plain JSON token; send it to the writing thread and pass it to `runCheckpoint()` with the -connection: - -```ts -// Replication thread: -const token = await replica.prepareCheckpoint(); // { path, txid, salt1, salt2, frames } -port.postMessage({ type: 'checkpoint', token }); - -// Writing thread (owns `db`), between transactions: -import { runCheckpoint } from '@platformatic/sqlite-js-stream'; -const incremental = runCheckpoint(db, token, 'TRUNCATE'); // synchronous -``` - -`runCheckpoint()` writes the record into `-litestream/` next to the database, and the -replica's next `sync()` reads it from disk, so the two sides share nothing else. No `sync()` of -the database may run between `prepareCheckpoint()` and `runCheckpoint()`, and the writing thread -must not commit between receiving the token and calling `runCheckpoint()`; a commit makes the -checkpoint return `false`, which is safe but not incremental. `replica.checkpoint(conn, mode)` is -`prepareCheckpoint()` followed by `runCheckpoint()`. - -It returns `true` when the WAL was fully checkpointed right after the sync. It returns `false` when -a commit landed between the sync and the checkpoint, or when a reader kept the checkpoint from -completing: the next `sync()` then resumes in the old WAL if it was not reset, or uploads a -snapshot if it was. Restoring across checkpoints works as usual: the incremental files form a -contiguous TXID chain. - -### Replication from a worker thread - -Run `replica.run()` in a **worker thread** so continuous replication doesn't compete with -your application's main thread. The worker opens its own **read-only** connection (via the -library) and reads the database + `-wal` files, while your app keeps writing from the main -thread on its own connection. Only the **path**s need to cross threads. - -`replica-worker.ts`: - -```ts -import { parentPort } from 'node:worker_threads'; -import { Replica, FileReplicaClient } from '@platformatic/sqlite-js-stream'; - -parentPort.on('message', async (msg) => { - if (msg?.type !== 'start') return; - const { dbPath, clientPath, intervalMs = 1000 } = msg; - - const replica = new Replica(dbPath, new FileReplicaClient(clientPath)); - const ac = new AbortController(); - const runP = replica.run({ intervalMs, signal: ac.signal }); - - parentPort.postMessage({ type: 'started' }); - parentPort.on('message', async (m) => { - if (m?.type === 'stop') { - ac.abort(); - await runP; - parentPort.postMessage({ type: 'done' }); - } - }); -}); -``` - -Main thread: - -```ts -import { Worker } from 'node:worker_threads'; - -const worker = new Worker(new URL('./replica-worker.ts', import.meta.url)); -const started = new Promise((resolve) => worker.on('message', (m) => m?.type === 'started' && resolve())); -const done = new Promise((resolve) => worker.on('message', (m) => m?.type === 'done' && resolve())); - -worker.postMessage({ type: 'start', dbPath: '/data/app.db', clientPath: '/data/replica', intervalMs: 1000 }); -await started; - -// Your app keeps using node:sqlite (or better-sqlite3) from the main thread. -// ... write rows ... - -// Shut the replication worker down gracefully. -worker.postMessage({ type: 'stop' }); -await done; -await worker.terminate(); -``` - -> **Important:** the worker and your app each use their **own** connection to the same database -> file. Never hand the worker the same `DatabaseSync` handle your app writes with — a single -> connection holds one transaction at a time, so sharing a writable handle would merge the read -> snapshot with your writes (see *Concurrency & snapshot consistency* above). - ---- - -## How it works - -Litestream (and this port) never touches the database through the SQLite WAL/VFS as part of -replication. Instead it: - -1. **Reads the `-wal` file directly** (parsing frames, verifying the WAL salt + checksum) and the - **database file directly** via `fs`. -2. **Builds an LTX file** per sync — a sequence of LZ4-compressed, checksummed pages that capture - the changed pages, tagged with a sequential **transaction ID**. -3. **Uploads** those immutable LTX files to a replica. -4. **Restores** by compacting a snapshot with the stream of level-0 files and writing a valid - SQLite file. - -See [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) for the full design and -[`docs/LTX_FORMAT.md`](docs/LTX_FORMAT.md) for the on-disk format. - -### Concurrency & snapshot consistency - -To keep snapshots consistent, `sync()`/`snapshot()` open a **separate, read-only** SQLite -connection to the same database and hold a short-lived read transaction while they read the -`-wal` + database files. SQLite will not checkpoint/truncate the WAL past an active reader, and the -pinned reader's `PRAGMA page_count` reports the commit including uncheckpointed WAL pages. - -Your application is free to use its own SQLite driver and connections (`node:sqlite`, -`better-sqlite3`, …) independently — the library reads only the files and never touches them. - -> **Do not point the library at your application's writable connection.** -> -> A single SQLite connection holds **one transaction at a time**. If the library were to use the -> same `DatabaseSync` handle as your writer: -> - its read snapshot would **not be isolated** from concurrent app writes (they'd merge into one -> transaction, and the library could read *uncommitted* rows or commit the app's writes), and -> - if your app had a transaction open, the library could not `BEGIN` its snapshot pin at all. -> -> The library deliberately opens its own read-only reader instead. The only API that takes your -> writable connection is the checkpoint (`replica.checkpoint()` / `runCheckpoint()`), which runs -> `PRAGMA wal_checkpoint` on it and never reads through it. Keep the library's reader and your app's writer on separate -> connections to the same database file. - -### Large databases & streaming - -The replication path is **streaming** end-to-end, so peak memory is bounded by one page buffer -(plus the small LTX page index) rather than the database size: - -- `sync()`/`snapshot()` read database **pages one at a time from a file handle** and stream the - encoded LTX straight to a temp file (never buffering the whole DB/WAL). -- `restore()` streams each LTX file's pages into the output database, overlaying incremental files - on the snapshot base — it never materializes a full merged file or the whole DB in memory. -- The S3 addon uploads objects larger than **5 MiB** with the **multipart** API, so a large snapshot - is never buffered on the upload side either. - -### Benchmarks - -Reproducible, configurable benchmarks live in [`bench/bench-sync-restore.ts`](bench/bench-sync-restore.ts) -and measure `sync()`/`restore()` throughput plus peak process RSS: - -```bash -pnpm bench -# another size / batch: -ROWS=200000 BATCH=1000 pnpm bench -# demonstrate the streaming memory bound (peak RSS vs DB size): -SCALE=4 pnpm bench -``` - -Conditions are disclosed in the output (rows, batch/transactions, source WAL bytes). Peak RSS is -process-wide (it includes the Node baseline), so the meaningful claim is that it stays **flat** as the -database grows — see the `SCALE` mode, which shows WAL bytes and peak RSS at two sizes. - -### Compaction & retention - -Over time `sync()` writes one level-0 LTX file per transaction. To keep the replica bounded and -restore fast, compact those into higher levels and prune data that has been superseded: - -```ts -// Merge pending level-0 files into a level-1 file (and 1→2→3 up to maxCompactionLevel). -await replica.compact({ dstLevel: 1 }); - -// Time-based retention: delete snapshots older than 24h (always keeping the -// newest), cascade the boundary down through the other levels, and delete -// level-0 files that were compacted into level-1 once they age past 5m. -const result = await replica.enforceRetention({ - snapshotRetentionMs: 24 * 60 * 60 * 1000, - l0RetentionMs: 5 * 60 * 1000, -}); -``` - -`Replica.run()` can drive all of this on their own schedules, and `Store` orchestrates it across -every replica. - -```ts -import { CompactionLevels } from '@platformatic/sqlite-js-stream'; - -// Each level is compacted at most once per wall-clock bucket (L1 every 30s, -// L2 every 5m, L3 every 1h) — mirrors litestream's per-level pacing. -const levels = new CompactionLevels([ - { level: 0, intervalMs: 0 }, - { level: 1, intervalMs: 30_000 }, - { level: 2, intervalMs: 300_000 }, - { level: 3, intervalMs: 3_600_000 }, -]); - -await store.run({ - intervalMs: 1000, - compactionLevels: levels, - snapshotIntervalMs: 24 * 60 * 60 * 1000, - snapshotRetentionMs: 24 * 60 * 60 * 1000, - l0RetentionMs: 5 * 60 * 1000, - signal, -}); -``` - -Compaction is **streaming** (bounded memory) and only deletes a lower-level file after the higher -level is written *and* uploaded, so a mid-compaction failure never loses data. Retention never deletes -files needed to restore any point within the retention window, and the live-head level-0 file is -always retained. - -See [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md#compaction--retention) for the full model. - -### Distributed leasing - -When several replicas point at the same storage, only one may write at a time. Give the replica a -`Leaser` (a `lock.json` lease) and `run()` will make exactly one replica the active **primary**; -the others become idle **followers** that poll until the lease is free. Storage writes are gated on -lease ownership, so a secondary can never split the brain. - -```ts -import { FileLeaser } from '@platformatic/sqlite-js-stream'; - -replica.leaser = new FileLeaser('/data/replica'); // becomes primary if the lease is free -await replica.run({ intervalMs: 1000, signal }); - -// Query which mode we are in: -if (replica.isPrimary()) { - // we are writing -} -``` - -The file backend uses an exclusive-create lease with a content-hash token; the S3 addon -(`import { S3Leaser } from '@platformatic/sqlite-js-stream-s3'`) uses S3 `If-None-Match`/`If-Match` -conditional writes so racy acquirers cannot both win. - -### ⚠️ This is a backup tool, not a distributed database - -The lease coordinates **which host writes to storage**. It does **not** merge, compare, or reconcile -the actual data in your databases. Be very clear about what you're pointing at the same prefix. - -**Multiple hosts are only safe if they hold the *same* database content** (e.g. a DR/standby host that -was restored from the same backup). The lease then prevents two hosts from both writing, and gives -you automatic failover when the primary stops. - -**Do NOT point hosts that hold *different* data at the same prefix.** SQLite is a -single-writer, single-file database. If host A has `row id=1 → "X"` and host B has -`row id=1 → "Y"` and they share one prefix, here is what happens: - -| Host A local DB | Host B local DB | What lands in S3 | -|----------------------|---------------------|------------------| -| `row id=1 → "X"` | `row id=1 → "Y"` | `"X"` (only if **A** is the primary) | - -- Whichever host is the **primary** gets backed up. The other host's value is **never uploaded**, is - not merged, and is effectively **lost** from the backup's perspective. -- When you restore, you get exactly one host's state. There is **no conflict resolution**. `"Y"` is - simply gone. - -**Choose the right setup:** - -| If you need… | Do this | -|---------------------------------------------------------|----------| -| Each host owns a **different** database | Give each its own prefix, e.g. `app/${tenantId}` — separate backup per DB, no conflict. | -| Multiple app instances sharing one consistent dataset | Don't use SQLite + this replication. Use a client-server DB (Postgres/MySQL) or a real shared-file lock + single writer. | -| Same data on every host (failover / DR) | Keep exactly one writer. On failover, **restore the standby from S3 first**, then let it take the lease — don't let a stale local file start writing. | - ---- - -## Deferred features - -Not implemented — listed here so the roadmap is clear. None of these are required for the library -core to replicate, snapshot, compact, lease, and restore a SQLite database. - -- **CLI + YAML config** — this is a *library*. The `litestream replicate/restore/ltx/reset/snapshots` - CLI and a `litestream.yml`-style config are out of scope for now. -- **Metrics/telemetry** — no Prometheus/OpenTelemetry instrumentation yet. -- **VFS real-time mirroring** — hooking into SQLite's VFS to mirror writes live is not implemented. -- **IPC control server** — litestream's Unix-socket IPC server is not implemented. - ---- - -## Documentation - -- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) — system design (DB, Replica, ReplicaClient, - LTX, WAL, backends). -- [`docs/LTX_FORMAT.md`](docs/LTX_FORMAT.md) — the LTX binary format. -- [`test/`](test) — TDD test suite (`npm test`). - ---- - -## Credits - -This project is a **port of [Litestream](https://github.com/benbjohnson/litestream)** -by [Ben Johnson](https://github.com/benbjohnson) (Apache-2.0). The LTX file format and much of the -design come from the [`superfly/ltx`](https://github.com/superfly/ltx) library and the litestream -documentation (`docs/ARCHITECTURE.md`, `docs/SQLITE_INTERNALS.md`, `docs/LTX_FORMAT.md`, -`docs/PATTERNS.md`), which are adapted here with attribution. - -The SQLite **WAL** and **LTX** formats are replicated faithfully so that this library can read LTX -files produced by Litestream and vice-versa. - ---- +See the [`@platformatic/sqlite-js-stream` README](packages/sqlite-js-stream/README.md) +for usage, and the architecture notes under +[`packages/sqlite-js-stream/docs`](packages/sqlite-js-stream/docs). ## License -Apache License 2.0. See [`LICENSE`](LICENSE) and [`NOTICE`](NOTICE). +[Apache-2.0](LICENSE). diff --git a/package.json b/package.json index e99bdad..0dc0b01 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,8 @@ { - "name": "@platformatic/sqlite-js-stream", + "name": "sqlite-js-stream", "version": "0.5.0", - "description": "Node.js port of Litestream: SQLite streaming replication & disaster recovery via LTX files", + "description": "Monorepo for @platformatic/sqlite-js-stream and its addons", + "private": true, "type": "module", "packageManager": "pnpm@12.16.1", "engines": { @@ -17,56 +18,20 @@ "bugs": { "url": "https://github.com/platformatic/sqlite-js-stream/issues" }, - "main": "./dist/index.js", - "types": "./dist/index.d.ts", - "exports": { - ".": { - "types": "./dist/index.d.ts", - "import": "./dist/index.js", - "default": "./dist/index.js" - }, - "./package.json": "./package.json" - }, - "files": [ - "dist", - "src", - "README.md", - "LICENSE", - "NOTICE", - "CHANGELOG.md" - ], - "publishConfig": { - "access": "public", - "registry": "https://registry.npmjs.org" - }, "scripts": { - "test": "node --test 'test/**/*.test.ts'", + "build": "pnpm --filter @platformatic/sqlite-js-stream build", "pretypecheck": "pnpm build", - "typecheck": "tsc -p . --noEmit", + "typecheck": "tsc -p tsconfig.json --noEmit", + "typecheck:all": "pnpm typecheck", "lint": "eslint .", "lint:fix": "eslint . --fix", - "build": "tsc -p tsconfig.build.json", - "prepack": "pnpm build", - "golden:go": "cd tools/golden && go run .", "pretest:all": "pnpm build", - "test:all": "pnpm test && pnpm -r --filter '!@platformatic/sqlite-js-stream-paged-vfs' test", - "test:paged-vfs": "pnpm --filter @platformatic/sqlite-js-stream-paged-vfs build:native && pnpm --filter @platformatic/sqlite-js-stream-paged-vfs build && pnpm --filter @platformatic/sqlite-js-stream-paged-vfs test && node --test test/integration/paged-restore.test.ts", - "typecheck:all": "pnpm typecheck", - "bench": "node bench/bench-sync-restore.ts" - }, - "dependencies": { - "lz4js": "^0.2.0" - }, - "peerDependencies": { - "@platformatic/sqlite-js-stream-paged-vfs": "workspace:^" - }, - "peerDependenciesMeta": { - "@platformatic/sqlite-js-stream-paged-vfs": { - "optional": true - } + "test:all": "pnpm -r --filter '!@platformatic/sqlite-js-stream-paged-vfs' test", + "test:paged-vfs": "pnpm --filter @platformatic/sqlite-js-stream-paged-vfs build:native && pnpm --filter @platformatic/sqlite-js-stream-paged-vfs build && pnpm --filter @platformatic/sqlite-js-stream-paged-vfs test && node --test packages/sqlite-js-stream/test/integration/paged-restore.test.ts", + "bench": "pnpm --filter @platformatic/sqlite-js-stream bench", + "golden:go": "cd packages/sqlite-js-stream/tools/golden && go run ." }, "devDependencies": { - "@platformatic/sqlite-js-stream-paged-vfs": "workspace:*", "@types/node": "^26.0.0", "eslint": "^9.39.5", "neostandard": "^0.14.0", @@ -78,6 +43,7 @@ "replication", "backup", "ltx", - "disaster-recovery" + "disaster-recovery", + "monorepo" ] } diff --git a/packages/backend-pg/test/delete-prefix.test.ts b/packages/backend-pg/test/delete-prefix.test.ts index e3fcad9..4bad062 100644 --- a/packages/backend-pg/test/delete-prefix.test.ts +++ b/packages/backend-pg/test/delete-prefix.test.ts @@ -3,7 +3,7 @@ import assert from 'node:assert/strict'; import { marshalHeader, type Header } from '@platformatic/sqlite-js-stream'; -import { deletePrefixSuite } from '../../../test/helpers/delete-prefix-suite.ts'; +import { deletePrefixSuite } from '../../sqlite-js-stream/test/helpers/delete-prefix-suite.ts'; import { PgLeaser, PgReplicaClient } from '../src/index.ts'; import { connectForTest, pgUrl } from './helpers/db.ts'; diff --git a/packages/backend-pg/test/fencing.test.ts b/packages/backend-pg/test/fencing.test.ts index 2832c3e..8f65ff7 100644 --- a/packages/backend-pg/test/fencing.test.ts +++ b/packages/backend-pg/test/fencing.test.ts @@ -1,4 +1,4 @@ -import { fencingSuite } from '../../../test/helpers/fencing-suite.ts'; +import { fencingSuite } from '../../sqlite-js-stream/test/helpers/fencing-suite.ts'; import { PgReplicaClient } from '../src/index.ts'; import { connectForTest } from './helpers/db.ts'; diff --git a/packages/backend-s3/test/delete-prefix.test.ts b/packages/backend-s3/test/delete-prefix.test.ts index e62c535..2e7a320 100644 --- a/packages/backend-s3/test/delete-prefix.test.ts +++ b/packages/backend-s3/test/delete-prefix.test.ts @@ -4,7 +4,7 @@ import { CreateBucketCommand, ListObjectsV2Command, PutObjectCommand, S3Client } import { marshalHeader } from '@platformatic/sqlite-js-stream'; -import { deletePrefixSuite, type DeletePrefixBackend } from '../../../test/helpers/delete-prefix-suite.ts'; +import { deletePrefixSuite, type DeletePrefixBackend } from '../../sqlite-js-stream/test/helpers/delete-prefix-suite.ts'; import { S3Leaser, S3ReplicaClient, type S3ClientLike } from '../src/index.ts'; import { MemoryS3 } from './helpers/memory-s3.ts'; diff --git a/packages/backend-s3/test/fencing.test.ts b/packages/backend-s3/test/fencing.test.ts index 3062c11..8ea6a0f 100644 --- a/packages/backend-s3/test/fencing.test.ts +++ b/packages/backend-s3/test/fencing.test.ts @@ -17,7 +17,7 @@ import { type Header, } from '@platformatic/sqlite-js-stream'; -import { fencingSuite } from '../../../test/helpers/fencing-suite.ts'; +import { fencingSuite } from '../../sqlite-js-stream/test/helpers/fencing-suite.ts'; import { S3ReplicaClient, isFenced } from '../src/index.ts'; import { MemoryS3 } from './helpers/memory-s3.ts'; diff --git a/CHANGELOG.md b/packages/sqlite-js-stream/CHANGELOG.md similarity index 100% rename from CHANGELOG.md rename to packages/sqlite-js-stream/CHANGELOG.md diff --git a/packages/sqlite-js-stream/LICENSE b/packages/sqlite-js-stream/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/packages/sqlite-js-stream/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/packages/sqlite-js-stream/NOTICE similarity index 100% rename from NOTICE rename to packages/sqlite-js-stream/NOTICE diff --git a/packages/sqlite-js-stream/README.md b/packages/sqlite-js-stream/README.md new file mode 100644 index 0000000..3e95846 --- /dev/null +++ b/packages/sqlite-js-stream/README.md @@ -0,0 +1,791 @@ +# sqlite-js-stream + +A **Node.js port of [Litestream](https://github.com/benbjohnson/litestream)** — SQLite streaming +replication and disaster recovery — written in **TypeScript** and built on the built-in `node:sqlite`. + +SQLite is the most widely deployed database on the planet but it only lives on a single machine. +If that machine dies, your database is gone. This library replicates a SQLite database's **WAL +(Write-Ahead Log)** file into immutable, chronological **LTX (Log Transaction)** files on a remote +replica (S3, a local directory, etc.), and can **restore** a database from those files. + +--- + +## Requirements + +- **Node.js `>= 24`** — required for the built-in `node:sqlite`. +- Runtime dependency: `lz4js` (pure-JS LZ4 **block** decoder), matching the LTX format. + Compression is a byte-identical TypeScript port of Go's `pierrec/lz4/v4` Fast block compressor. + +The **published** package ships compiled JavaScript plus TypeScript declarations +(`dist/*.js` + `dist/*.d.ts`), built at publish time via `pnpm build`. + +--- + +## Installation + +This repository is a **pnpm workspace**. The core library lives in +`packages/sqlite-js-stream` and the upload strategies (replica backends) live +under `packages/`: + +```bash +pnpm install # link the workspace +pnpm build # transpile src -> dist (.js + .d.ts), used by addons/tests +``` + +The core package is published as **`@platformatic/sqlite-js-stream`** and the S3 +and PostgreSQL addons as **`@platformatic/sqlite-js-stream-s3`** and +**`@platformatic/sqlite-js-stream-pg`**: + +```bash +pnpm add @platformatic/sqlite-js-stream +pnpm add @platformatic/sqlite-js-stream-s3 # optional addon +pnpm add @platformatic/sqlite-js-stream-pg # optional addon +``` + +Importing the addon registers the `s3` URL scheme with +`newReplicaClientFromURL('s3://bucket/prefix')`. + +**`@platformatic/sqlite-js-stream-paged-vfs`** is a SQLite VFS for `node:sqlite` +which reads the pages of a database on demand, so that an application can query +a large database while it is still being restored. It is a native library, with +prebuilt binaries for macOS and Linux. `replica.restorePaged()` uses it when it is +installed (see [Restore page by page](#restore-page-by-page)). See its +[README](../sqlite-js-stream-paged-vfs/README.md). + +```bash +pnpm add @platformatic/sqlite-js-stream-paged-vfs +``` + +--- + +## Quick start + +```ts +import { Replica, FileReplicaClient } from '@platformatic/sqlite-js-stream'; + +// 1. Point the replica at your database path and a replica destination. +// The library reads the database + `-wal` directly and is agnostic to +// whichever SQLite driver your application uses. +const client = new FileReplicaClient('/data/replica'); +const replica = new Replica('/data/app.db', client); + +// 2. Sync builds a local LTX file (into -litestream/ltx/0/) and uploads it. +await replica.sync(); +``` + +The database is written to by your **own** SQLite connection (using `node:sqlite`, `better-sqlite3`, +or anything else). After more writes, just `sync()` again: + +```ts +// ... your app writes rows with whatever driver you use ... +await replica.sync(); + +const status = await replica.syncStatus(); +console.log(status.inSync, status.localTXID, status.remoteTXID); // true 3n 3n +``` + +### Take a full snapshot + +Normal `sync()` writes **incremental** level-0 files. For a complete, point-in-time backup, take a +**level-9 snapshot** (a full database copy as an LTX file with `MinTXID = 1`): + +```ts +await replica.snapshot(); +``` + +### Restore a database + +Restore reconstructs a SQLite database by compacting a level-9 snapshot with the subsequent +level-0 files: + +```ts +await replica.restore({ outputPath: '/data/restored.db' }); +``` + +Restore to a specific transaction ID, or to a point-in-time (the latest backup created at/before +the given time). Pass exactly one of `txID` or `timestamp`: + +```ts +await replica.restore({ outputPath: '/data/app-at-txid.db', txID: 42n }); +await replica.restore({ outputPath: '/data/app-at-time.db', timestamp: new Date('2025-06-01T12:00:00Z') }); +``` + +Restoring to `txID`/`timestamp` is at **backup boundaries** (the end of an LTX file), and a target +with no matching backup throws `NoMatchingBackupError`. A "latest" restore over a non-contiguous +file set throws `RestoreGapError` rather than silently returning a partial database. + +### Continuously restore (follow) + +`follow()` restores the latest backup, then keeps the database up to date as new backups are +replicated — maintaining a read-only replica (litestream's `restore -f`): + +```ts +const ac = new AbortController(); +await replica.follow({ outputPath: '/data/replica.db', intervalMs: 1000, signal: ac.signal }); +// later: ac.abort() to stop +``` + +The applied TXID is recorded in a `-txid` sidecar (default `-txid`) so a later `follow()` +resumes instead of re-restoring. The output database should only be opened **read-only**. + +### Restore the local database if it is missing + +```ts +await replica.ensureExists(); +``` + +Restoring a replica's own database (`outputPath` equal to its path, as +`ensureExists()` does) makes it the writer from the restored position: the local +LTX cache of any previous database at that path is discarded, and the next +`sync()` uploads the following TXID. Any `-wal` and `-shm` files left at the +output path are removed before the restored database is moved into place. +`sync()` throws `ErrReplicaAhead` (`ERR_REPLICA_AHEAD`) rather than skipping +uploads when the replica already has a higher TXID than the local database. + +The restore also writes the first WAL of the restored database, under salts it +records, so that the next `sync()` uploads only the changes made since the +restore instead of a snapshot of the whole database. That holds while the WAL is +not reset: a checkpoint that restarts or truncates it, or the last connection +closing (which deletes it), makes the next `sync()` fall back to a snapshot. +With `wal_autocheckpoint` disabled and checkpoints run through +`replica.checkpoint()` (see below), syncs stay incremental from the restore on. + +### Restore page by page + +A full restore downloads the whole database before it can be opened. With +`@platformatic/sqlite-js-stream-paged-vfs` installed, `restorePaged()` restores +the replica's own database page by page instead: it resolves once the database +can be opened, and queries read the pages they need on demand while the others +are downloaded in the background. + +```ts +import { databaseUri } from '@platformatic/sqlite-js-stream-paged-vfs'; + +const result = await replica.restorePaged({ minSize: 64 * 1024 * 1024 }); +// In a worker thread, not this one: +const db = new DatabaseSync(databaseUri('/data/app.db')); + +await result.filled; // every page is local +``` + +- The database is created with every page missing. A query that reads a missing + page waits while it is read from the replica, with the next pages of the same + LTX file (`readAhead`, 32 by default). Page 1 and the schema are read before + `restorePaged()` resolves. +- Open the database with the URI from `databaseUri()`, from **another thread** + than the one that called `restorePaged()`: that thread serves the pages, so a + query from it would wait for itself (such reads fail instead). Every + connection that writes the database must use the VFS. +- Replication goes on meanwhile. `sync()` and `snapshot()` read the pages they + need first, and retention keeps the LTX files the restore still reads. +- `result.filled` resolves once every page is local. `replica.close()`, or + restoring the database again, stops the restore: the database stays + incomplete. So does a crash. An incomplete database has a `localPos()` of 0, + and `sync()`/`snapshot()` throw `ErrIncompleteDatabase` until it is restored + again (`ensureExists()` does). +- It restores fully, like `restore()`, when the LTX files to download add up + to less than `minSize` bytes, or when the paged VFS cannot be used + (`result.unavailable` holds the reason: the package is not installed, has no + binary for the platform, or Node.js runs with `--permission`). + `result.paged` tells which. +- `onMiss(pgno, pages, durationMs)` is called for each page a query waited for, + for metrics. + +### Manage many databases + +```ts +import { Store } from '@platformatic/sqlite-js-stream'; + +const store = new Store(); +await store.openDB('/data/a.db', new FileReplicaClient('/data/replica-a')); +await store.openDB('/data/b.db', new FileReplicaClient('/data/replica-b')); + +await store.syncAll(); // local sync + upload for every database +await store.snapshotAll(); // level-9 snapshots for every database + +store.close(); +``` + +### Replica URL + +Replica clients are created from URLs via the factory registry: + +```ts +import { newReplicaClientFromURL, Replica } from '@platformatic/sqlite-js-stream'; + +const client = newReplicaClientFromURL('file:///data/replica'); +const replica = new Replica('/data/app.db', client); +``` + +The `file` scheme is built into the core. Additional schemes (S3, etc.) are +registered by importing the matching addon (see below). + +### Custom read connection + +During sync and snapshot, a replica pins the WAL with a short read-only +`node:sqlite` transaction. To use another driver for that connection, e.g. for +libsql-encrypted databases that `node:sqlite` cannot open, pass `openReader`: + +```js +import Database from 'libsql'; + +const replica = new Replica('/data/app.db', client, undefined, { + openReader: (path) => new Database(path, { readonly: true, encryptionKey: process.env.DB_KEY }), +}); +``` + +Pages are copied as-is, so the replica stays encrypted and a restored database +opens with the same key. + +### S3 upload strategy + +The `@platformatic/sqlite-js-stream-s3` addon provides an S3 replica client. Import it +once to register the `s3://` scheme, then create clients from URLs or directly: + +```ts +import { Replica, newReplicaClientFromURL } from '@platformatic/sqlite-js-stream'; +import { S3ReplicaClient } from '@platformatic/sqlite-js-stream-s3'; + +// Via a URL (bucket from the host, prefix from the path). +const client = newReplicaClientFromURL('s3://my-bucket/backup?region=us-east-1'); + +// Or directly, with explicit AWS config. +const s3 = new S3ReplicaClient({ + bucket: 'my-bucket', + prefix: 'backup', + region: 'us-east-1', + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, +}); + +const replica = new Replica('/data/app.db', client); +await replica.run({ intervalMs: 1000 }); +``` + +Objects are stored as `{prefix}/ltx/{level}/{minTXID}-{maxTXID}.ltx` — the same +layout as the file backend, so a replica can move between strategies. Creation +timestamps are preserved as the `ltx-timestamp` object metadata. + +> **Testing without AWS:** `S3ReplicaClient` accepts an injected `client` +> implementing `send(command)`. Pass an in-memory fake (see +> `../backend-s3/test/helpers/memory-s3.ts`) to exercise the addon without +> network access. + +### PostgreSQL upload strategy + +The `@platformatic/sqlite-js-stream-pg` addon provides a PostgreSQL replica client. +Import it once to register the `postgres://` and `postgresql://` schemes, then +create clients from URLs or directly: + +```ts +import { Replica, newReplicaClientFromURL } from '@platformatic/sqlite-js-stream'; +import { PgReplicaClient, PgLeaser } from '@platformatic/sqlite-js-stream-pg'; + +// Via a URL: connection details from the URL, replica prefix from `?prefix=`. +const client = newReplicaClientFromURL( + 'postgres://user:pass@host:5432/mydb?prefix=app/db1', +); + +// Or directly, with an explicit connection string. +const pg = new PgReplicaClient({ + url: 'postgres://user:pass@host:5432/mydb', + prefix: 'app/db1', +}); + +const replica = new Replica('/data/app.db', client); +await replica.run({ intervalMs: 1000 }); +``` + +Files are stored as one row in `ltx_files` plus one row per 1 MiB chunk in +`ltx_chunks`, scoped by a `prefix` column, in a configurable schema (`?schema=`). +Every file is written atomically (row + chunks in a single transaction), so a +reader never sees a partially-written file. `created_at` preserves the LTX +creation timestamp for point-in-time restore and retention. + +> **Fenced writes:** see [Fenced writes](#fenced-writes). + +> **Distributed leasing:** `PgLeaser` acquires/renews/releases a lease on a +> `ltx_leases` row using a compare-and-swap `version` column, with the same +> semantics as `S3Leaser` (`LeaseExistsError`, `ErrLeaseNotHeld`). Pass it to a +> `Replica` via the `leaser` option so only one host writes at a time. + +### Fenced writes + +When the application elects the writer of a database itself (with its own lease, +or a `Leaser`), a former writer that was paused (GC, a stopped VM, a network +partition) can wake up after losing the lease and keep uploading. The S3 and +PostgreSQL clients can reject those uploads: give every writer a *generation* +that only grows (for example a fencing token from your lease), and follow this +order when a writer takes over a database: + +```ts +import { Replica } from '@platformatic/sqlite-js-stream'; +import { S3ReplicaClient } from '@platformatic/sqlite-js-stream-s3'; + +const client = new S3ReplicaClient({ bucket: 'my-bucket', prefix: 'app/db1' }); + +// 1. Register: fails with ErrGenerationSuperseded if a higher generation is +// registered. From now on, older writers' uploads and deletions fail. +await client.registerGeneration(token); + +// 2. Restore (or keep the local database if it is at the replica's latest TXID). +const replica = new Replica('/data/app.db', client); +await replica.restore({ outputPath: '/data/app.db' }); + +// 3. Write and replicate. +await replica.sync(); +``` + +`registerGeneration(n)` sets the client's generation (the `generation` option +does the same without registering). From then on: + +- An upload, compaction or retention deletion fails with + `ErrGenerationSuperseded` (`ERR_GENERATION_SUPERSEDED`) once a higher + generation is registered. The `Replica` then stops writing: every later + `sync()`, `snapshot()`, compaction and retention call fails with the same + error (also in `replica.superseded`), and `run()` stops and rejects with it + instead of retrying. +- An LTX file of a given level and TXID range is written only once. + +**PostgreSQL** checks the generation in the transaction that writes or deletes +the files (`SELECT … FOR SHARE` on the `ltx_generations` row), so registering +waits for the uploads in flight and the following ones fail. + +**S3** can't check one object and write another in one request, so fencing is a +protocol of conditional writes on a `generation.json` object next to the `ltx/` +directory: + +- `registerGeneration(n)` raises the generation with `If-Match`/`If-None-Match`, + then reads the replica's latest TXID `T` and records the *fence* `(n, T)` in + the same object. It does both, so there is no separate step after the restore. +- Fenced uploads use `If-None-Match: *` and store their generation in the + `ltx-generation` metadata. After each upload, the client reads + `generation.json` and fails if a higher generation is registered. An upload + that resolves was therefore made before any newer writer registered, and + every newer writer's restore sees it. +- An upload of a superseded writer can still land if it was in flight. Listings + ignore a file whose generation is lower than a fence's and whose `maxTXID` is + above that fence's `T`. If such a file holds a key that the new writer needs, + the new writer replaces it (`If-Match` on its ETag). +- Deletions check `generation.json` right before `DeleteObjects`. A deletion + sent just before a new generation registers still applies; this is the one + race the protocol leaves open, and it needs the old writer to pause between + the check and the request. +- Once a fence exists, listing a file that ends above the lowest fence costs + one `HeadObject` request to read its generation (cached per key and ETag; a + writer knows its own files). Replicas that never registered a generation pay + one `GetObject` of `generation.json` per listing (a 404). +- `deleteAll()` keeps `generation.json`, so a superseded writer stays fenced. + +The file backend does not implement fenced writes. + +### Deleting every replica under a prefix + +`deleteAll()` deletes the files of one replica and keeps its generation, so a +superseded writer stays fenced. To delete a whole group of replicas, for +example every database of a tenant, each client has a static `deletePrefix()`. +It deletes the replica at the prefix and every replica under `prefix/` (`a` +covers `a` and `a/b`, not `ab`), with their generations and leases, and resolves +with the sorted prefixes (paths for the file backend) of the deleted replicas: + +```ts +import { FileReplicaClient } from '@platformatic/sqlite-js-stream'; +import { PgReplicaClient } from '@platformatic/sqlite-js-stream-pg'; +import { S3ReplicaClient } from '@platformatic/sqlite-js-stream-s3'; + +await FileReplicaClient.deletePrefix({ path: '/backups/tenant-1' }); +await PgReplicaClient.deletePrefix({ sql, prefix: 'tenant-1', tablePrefix: 'ltx' }); +await S3ReplicaClient.deletePrefix({ bucket: 'my-bucket', prefix: 'tenant-1', client }); +// => ['tenant-1/db1', 'tenant-1/db2'] +``` + +The options take the same connection settings as the client's constructor (`url` +or `sql`, `schema` and `tablePrefix` for PostgreSQL; `client` or `region`, +`endpoint` and credentials for S3). An empty prefix is rejected. + +With a `generation` option, the deletion is fenced like the deletions of a fenced +client: it fails with `ErrGenerationSuperseded` if a replica under the prefix has +a higher generation. + +- **PostgreSQL** deletes everything in one transaction that first locks the + generation rows (`FOR UPDATE`): it waits for the fenced uploads and deletions + in flight, the ones that start meanwhile wait for it, and a superseded + deletion deletes nothing. Pass a transaction as `sql` (the argument of a + `sql.begin()` callback) to run the deletion in a savepoint of it, so it commits + or rolls back with the caller's own changes. +- **S3** is best effort: the generation of each replica is checked before + deleting its first object and again before the generation objects, which are + deleted last, so former writers stay fenced while the files are deleted. A + generation registered between a check and a `DeleteObjects` request does not + stop the deletion, and a superseded deletion keeps what it already deleted. +- **The file backend** has no generations: the directory is removed. + +Once the generations are deleted, nothing fences the former writers of these +replicas: a writer still running can upload again and create its replica anew. +Stop the writers first. + +### Continuous sync + +For ongoing replication, `replica.run()` syncs immediately and then repeats on an +interval until the supplied `AbortSignal` is aborted. Sync errors are logged and the +loop keeps going, so a transient failure doesn't stop replication: + +```ts +import { Replica, FileReplicaClient } from '@platformatic/sqlite-js-stream'; + +const replica = new Replica('/data/app.db', new FileReplicaClient('/data/replica')); +const ac = new AbortController(); + +// Run continuously (sync now, then every 1s); resolves when aborted. +const runP = replica.run({ intervalMs: 1000, signal: ac.signal }); + +// ... later, to stop: +ac.abort(); +await runP; +``` + +If you want to keep replicating in the background without awaiting, call `run()` and +`catch` its promise (it resolves on abort): + +```ts +replica.run({ intervalMs: 1000, signal: ac.signal }).catch((err) => { + console.error('continuous sync stopped', err); +}); +``` + +`Store` also exposes `syncAll()`/`snapshotAll()` for one-shot operations; a continuous +loop can be driven by calling `replica.run()` per database, or via your own scheduler. + +### Checkpoints without full snapshots + +When the WAL is reset by a checkpoint, the next `sync()` cannot resume from its previous position +in the WAL, so it uploads a full snapshot of the database. An application that decides when +checkpoints run can avoid that with `replica.checkpoint()`: it syncs, checkpoints the WAL, and +records that the next WAL continues from the synced position, so the next `sync()` uploads only the +changes. + +```ts +import { DatabaseSync } from 'node:sqlite'; + +const db = new DatabaseSync('/data/app.db'); +db.exec('PRAGMA journal_mode=WAL'); +db.exec('PRAGMA wal_autocheckpoint=0'); // checkpoints run only through replica.checkpoint() + +// ... write with db, and call replica.sync() as usual ... + +// Instead of db.exec('PRAGMA wal_checkpoint(TRUNCATE)'): +const incremental = await replica.checkpoint(db, 'TRUNCATE'); +``` + +`checkpoint(conn, mode = 'TRUNCATE')` takes the connection the application writes with (`mode` is +`'RESTART'` or `'TRUNCATE'`): + +- Pass the **writing connection**, with no open transaction. SQLite gives the next WAL a predictable + salt only when the connection that starts it is the one that reset the WAL; that is how the next + `sync()` recognizes the WAL that directly followed the checkpoint. +- Disable `wal_autocheckpoint` on every connection and never run `PRAGMA wal_checkpoint` yourself: + any other reset is detected and the next `sync()` falls back to a full snapshot (correct, but + not incremental). +- Do not call it concurrently with `sync()` or `run()`. +- The next WAL must be started by the same connection: a different connection that writes first + picks random salts, and the next `sync()` falls back to a snapshot. + +When replication runs on a different thread (or process) than the writing connection, which +cannot cross threads, split the call in two. `replica.prepareCheckpoint()` syncs and returns a +plain JSON token; send it to the writing thread and pass it to `runCheckpoint()` with the +connection: + +```ts +// Replication thread: +const token = await replica.prepareCheckpoint(); // { path, txid, salt1, salt2, frames } +port.postMessage({ type: 'checkpoint', token }); + +// Writing thread (owns `db`), between transactions: +import { runCheckpoint } from '@platformatic/sqlite-js-stream'; +const incremental = runCheckpoint(db, token, 'TRUNCATE'); // synchronous +``` + +`runCheckpoint()` writes the record into `-litestream/` next to the database, and the +replica's next `sync()` reads it from disk, so the two sides share nothing else. No `sync()` of +the database may run between `prepareCheckpoint()` and `runCheckpoint()`, and the writing thread +must not commit between receiving the token and calling `runCheckpoint()`; a commit makes the +checkpoint return `false`, which is safe but not incremental. `replica.checkpoint(conn, mode)` is +`prepareCheckpoint()` followed by `runCheckpoint()`. + +It returns `true` when the WAL was fully checkpointed right after the sync. It returns `false` when +a commit landed between the sync and the checkpoint, or when a reader kept the checkpoint from +completing: the next `sync()` then resumes in the old WAL if it was not reset, or uploads a +snapshot if it was. Restoring across checkpoints works as usual: the incremental files form a +contiguous TXID chain. + +### Replication from a worker thread + +Run `replica.run()` in a **worker thread** so continuous replication doesn't compete with +your application's main thread. The worker opens its own **read-only** connection (via the +library) and reads the database + `-wal` files, while your app keeps writing from the main +thread on its own connection. Only the **path**s need to cross threads. + +`replica-worker.ts`: + +```ts +import { parentPort } from 'node:worker_threads'; +import { Replica, FileReplicaClient } from '@platformatic/sqlite-js-stream'; + +parentPort.on('message', async (msg) => { + if (msg?.type !== 'start') return; + const { dbPath, clientPath, intervalMs = 1000 } = msg; + + const replica = new Replica(dbPath, new FileReplicaClient(clientPath)); + const ac = new AbortController(); + const runP = replica.run({ intervalMs, signal: ac.signal }); + + parentPort.postMessage({ type: 'started' }); + parentPort.on('message', async (m) => { + if (m?.type === 'stop') { + ac.abort(); + await runP; + parentPort.postMessage({ type: 'done' }); + } + }); +}); +``` + +Main thread: + +```ts +import { Worker } from 'node:worker_threads'; + +const worker = new Worker(new URL('./replica-worker.ts', import.meta.url)); +const started = new Promise((resolve) => worker.on('message', (m) => m?.type === 'started' && resolve())); +const done = new Promise((resolve) => worker.on('message', (m) => m?.type === 'done' && resolve())); + +worker.postMessage({ type: 'start', dbPath: '/data/app.db', clientPath: '/data/replica', intervalMs: 1000 }); +await started; + +// Your app keeps using node:sqlite (or better-sqlite3) from the main thread. +// ... write rows ... + +// Shut the replication worker down gracefully. +worker.postMessage({ type: 'stop' }); +await done; +await worker.terminate(); +``` + +> **Important:** the worker and your app each use their **own** connection to the same database +> file. Never hand the worker the same `DatabaseSync` handle your app writes with — a single +> connection holds one transaction at a time, so sharing a writable handle would merge the read +> snapshot with your writes (see *Concurrency & snapshot consistency* above). + +--- + +## How it works + +Litestream (and this port) never touches the database through the SQLite WAL/VFS as part of +replication. Instead it: + +1. **Reads the `-wal` file directly** (parsing frames, verifying the WAL salt + checksum) and the + **database file directly** via `fs`. +2. **Builds an LTX file** per sync — a sequence of LZ4-compressed, checksummed pages that capture + the changed pages, tagged with a sequential **transaction ID**. +3. **Uploads** those immutable LTX files to a replica. +4. **Restores** by compacting a snapshot with the stream of level-0 files and writing a valid + SQLite file. + +See [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) for the full design and +[`docs/LTX_FORMAT.md`](docs/LTX_FORMAT.md) for the on-disk format. + +### Concurrency & snapshot consistency + +To keep snapshots consistent, `sync()`/`snapshot()` open a **separate, read-only** SQLite +connection to the same database and hold a short-lived read transaction while they read the +`-wal` + database files. SQLite will not checkpoint/truncate the WAL past an active reader, and the +pinned reader's `PRAGMA page_count` reports the commit including uncheckpointed WAL pages. + +Your application is free to use its own SQLite driver and connections (`node:sqlite`, +`better-sqlite3`, …) independently — the library reads only the files and never touches them. + +> **Do not point the library at your application's writable connection.** +> +> A single SQLite connection holds **one transaction at a time**. If the library were to use the +> same `DatabaseSync` handle as your writer: +> - its read snapshot would **not be isolated** from concurrent app writes (they'd merge into one +> transaction, and the library could read *uncommitted* rows or commit the app's writes), and +> - if your app had a transaction open, the library could not `BEGIN` its snapshot pin at all. +> +> The library deliberately opens its own read-only reader instead. The only API that takes your +> writable connection is the checkpoint (`replica.checkpoint()` / `runCheckpoint()`), which runs +> `PRAGMA wal_checkpoint` on it and never reads through it. Keep the library's reader and your app's writer on separate +> connections to the same database file. + +### Large databases & streaming + +The replication path is **streaming** end-to-end, so peak memory is bounded by one page buffer +(plus the small LTX page index) rather than the database size: + +- `sync()`/`snapshot()` read database **pages one at a time from a file handle** and stream the + encoded LTX straight to a temp file (never buffering the whole DB/WAL). +- `restore()` streams each LTX file's pages into the output database, overlaying incremental files + on the snapshot base — it never materializes a full merged file or the whole DB in memory. +- The S3 addon uploads objects larger than **5 MiB** with the **multipart** API, so a large snapshot + is never buffered on the upload side either. + +### Benchmarks + +Reproducible, configurable benchmarks live in [`bench/bench-sync-restore.ts`](bench/bench-sync-restore.ts) +and measure `sync()`/`restore()` throughput plus peak process RSS: + +```bash +pnpm bench +# another size / batch: +ROWS=200000 BATCH=1000 pnpm bench +# demonstrate the streaming memory bound (peak RSS vs DB size): +SCALE=4 pnpm bench +``` + +Conditions are disclosed in the output (rows, batch/transactions, source WAL bytes). Peak RSS is +process-wide (it includes the Node baseline), so the meaningful claim is that it stays **flat** as the +database grows — see the `SCALE` mode, which shows WAL bytes and peak RSS at two sizes. + +### Compaction & retention + +Over time `sync()` writes one level-0 LTX file per transaction. To keep the replica bounded and +restore fast, compact those into higher levels and prune data that has been superseded: + +```ts +// Merge pending level-0 files into a level-1 file (and 1→2→3 up to maxCompactionLevel). +await replica.compact({ dstLevel: 1 }); + +// Time-based retention: delete snapshots older than 24h (always keeping the +// newest), cascade the boundary down through the other levels, and delete +// level-0 files that were compacted into level-1 once they age past 5m. +const result = await replica.enforceRetention({ + snapshotRetentionMs: 24 * 60 * 60 * 1000, + l0RetentionMs: 5 * 60 * 1000, +}); +``` + +`Replica.run()` can drive all of this on their own schedules, and `Store` orchestrates it across +every replica. + +```ts +import { CompactionLevels } from '@platformatic/sqlite-js-stream'; + +// Each level is compacted at most once per wall-clock bucket (L1 every 30s, +// L2 every 5m, L3 every 1h) — mirrors litestream's per-level pacing. +const levels = new CompactionLevels([ + { level: 0, intervalMs: 0 }, + { level: 1, intervalMs: 30_000 }, + { level: 2, intervalMs: 300_000 }, + { level: 3, intervalMs: 3_600_000 }, +]); + +await store.run({ + intervalMs: 1000, + compactionLevels: levels, + snapshotIntervalMs: 24 * 60 * 60 * 1000, + snapshotRetentionMs: 24 * 60 * 60 * 1000, + l0RetentionMs: 5 * 60 * 1000, + signal, +}); +``` + +Compaction is **streaming** (bounded memory) and only deletes a lower-level file after the higher +level is written *and* uploaded, so a mid-compaction failure never loses data. Retention never deletes +files needed to restore any point within the retention window, and the live-head level-0 file is +always retained. + +See [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md#compaction--retention) for the full model. + +### Distributed leasing + +When several replicas point at the same storage, only one may write at a time. Give the replica a +`Leaser` (a `lock.json` lease) and `run()` will make exactly one replica the active **primary**; +the others become idle **followers** that poll until the lease is free. Storage writes are gated on +lease ownership, so a secondary can never split the brain. + +```ts +import { FileLeaser } from '@platformatic/sqlite-js-stream'; + +replica.leaser = new FileLeaser('/data/replica'); // becomes primary if the lease is free +await replica.run({ intervalMs: 1000, signal }); + +// Query which mode we are in: +if (replica.isPrimary()) { + // we are writing +} +``` + +The file backend uses an exclusive-create lease with a content-hash token; the S3 addon +(`import { S3Leaser } from '@platformatic/sqlite-js-stream-s3'`) uses S3 `If-None-Match`/`If-Match` +conditional writes so racy acquirers cannot both win. + +### ⚠️ This is a backup tool, not a distributed database + +The lease coordinates **which host writes to storage**. It does **not** merge, compare, or reconcile +the actual data in your databases. Be very clear about what you're pointing at the same prefix. + +**Multiple hosts are only safe if they hold the *same* database content** (e.g. a DR/standby host that +was restored from the same backup). The lease then prevents two hosts from both writing, and gives +you automatic failover when the primary stops. + +**Do NOT point hosts that hold *different* data at the same prefix.** SQLite is a +single-writer, single-file database. If host A has `row id=1 → "X"` and host B has +`row id=1 → "Y"` and they share one prefix, here is what happens: + +| Host A local DB | Host B local DB | What lands in S3 | +|----------------------|---------------------|------------------| +| `row id=1 → "X"` | `row id=1 → "Y"` | `"X"` (only if **A** is the primary) | + +- Whichever host is the **primary** gets backed up. The other host's value is **never uploaded**, is + not merged, and is effectively **lost** from the backup's perspective. +- When you restore, you get exactly one host's state. There is **no conflict resolution**. `"Y"` is + simply gone. + +**Choose the right setup:** + +| If you need… | Do this | +|---------------------------------------------------------|----------| +| Each host owns a **different** database | Give each its own prefix, e.g. `app/${tenantId}` — separate backup per DB, no conflict. | +| Multiple app instances sharing one consistent dataset | Don't use SQLite + this replication. Use a client-server DB (Postgres/MySQL) or a real shared-file lock + single writer. | +| Same data on every host (failover / DR) | Keep exactly one writer. On failover, **restore the standby from S3 first**, then let it take the lease — don't let a stale local file start writing. | + +--- + +## Deferred features + +Not implemented — listed here so the roadmap is clear. None of these are required for the library +core to replicate, snapshot, compact, lease, and restore a SQLite database. + +- **CLI + YAML config** — this is a *library*. The `litestream replicate/restore/ltx/reset/snapshots` + CLI and a `litestream.yml`-style config are out of scope for now. +- **Metrics/telemetry** — no Prometheus/OpenTelemetry instrumentation yet. +- **VFS real-time mirroring** — hooking into SQLite's VFS to mirror writes live is not implemented. +- **IPC control server** — litestream's Unix-socket IPC server is not implemented. + +--- + +## Documentation + +- [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) — system design (DB, Replica, ReplicaClient, + LTX, WAL, backends). +- [`docs/LTX_FORMAT.md`](docs/LTX_FORMAT.md) — the LTX binary format. +- [`test/`](test) — TDD test suite (`npm test`). + +--- + +## Credits + +This project is a **port of [Litestream](https://github.com/benbjohnson/litestream)** +by [Ben Johnson](https://github.com/benbjohnson) (Apache-2.0). The LTX file format and much of the +design come from the [`superfly/ltx`](https://github.com/superfly/ltx) library and the litestream +documentation (`docs/ARCHITECTURE.md`, `docs/SQLITE_INTERNALS.md`, `docs/LTX_FORMAT.md`, +`docs/PATTERNS.md`), which are adapted here with attribution. + +The SQLite **WAL** and **LTX** formats are replicated faithfully so that this library can read LTX +files produced by Litestream and vice-versa. + +--- + +## License + +Apache License 2.0. See [`LICENSE`](LICENSE) and [`NOTICE`](NOTICE). diff --git a/bench/bench-sync-restore.ts b/packages/sqlite-js-stream/bench/bench-sync-restore.ts similarity index 98% rename from bench/bench-sync-restore.ts rename to packages/sqlite-js-stream/bench/bench-sync-restore.ts index e7f7b9a..8c03c48 100644 --- a/bench/bench-sync-restore.ts +++ b/packages/sqlite-js-stream/bench/bench-sync-restore.ts @@ -56,7 +56,7 @@ function makeSourceDB(path: string, rows: number, batch: number): DatabaseSync { } /** Measures wall ms + peak RSS (sampled) around an async operation. */ -async function measure(fn: () => Promise): Promise<{ ms: number; peakRssMB: number }> { +async function measure(fn: () => Promise): Promise<{ ms: number; peakRssMB: number }> { let peak = process.memoryUsage().rss; const sampler = setInterval(() => { const rss = process.memoryUsage().rss; diff --git a/docs/ARCHITECTURE.md b/packages/sqlite-js-stream/docs/ARCHITECTURE.md similarity index 100% rename from docs/ARCHITECTURE.md rename to packages/sqlite-js-stream/docs/ARCHITECTURE.md diff --git a/docs/LTX_FORMAT.md b/packages/sqlite-js-stream/docs/LTX_FORMAT.md similarity index 100% rename from docs/LTX_FORMAT.md rename to packages/sqlite-js-stream/docs/LTX_FORMAT.md diff --git a/docs/SQLITE_INTERNALS.md b/packages/sqlite-js-stream/docs/SQLITE_INTERNALS.md similarity index 100% rename from docs/SQLITE_INTERNALS.md rename to packages/sqlite-js-stream/docs/SQLITE_INTERNALS.md diff --git a/packages/sqlite-js-stream/package.json b/packages/sqlite-js-stream/package.json new file mode 100644 index 0000000..2f65568 --- /dev/null +++ b/packages/sqlite-js-stream/package.json @@ -0,0 +1,73 @@ +{ + "name": "@platformatic/sqlite-js-stream", + "version": "0.5.0", + "description": "Node.js port of Litestream: SQLite streaming replication & disaster recovery via LTX files", + "type": "module", + "packageManager": "pnpm@12.16.1", + "engines": { + "node": ">=24" + }, + "license": "Apache-2.0", + "author": "Matteo Collina", + "homepage": "https://github.com/platformatic/sqlite-js-stream/tree/main/packages/sqlite-js-stream", + "repository": { + "type": "git", + "url": "git+https://github.com/platformatic/sqlite-js-stream.git", + "directory": "packages/sqlite-js-stream" + }, + "bugs": { + "url": "https://github.com/platformatic/sqlite-js-stream/issues" + }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js", + "default": "./dist/index.js" + }, + "./package.json": "./package.json" + }, + "files": [ + "dist", + "src", + "README.md", + "LICENSE", + "NOTICE", + "CHANGELOG.md" + ], + "publishConfig": { + "access": "public", + "registry": "https://registry.npmjs.org" + }, + "scripts": { + "test": "node --test 'test/**/*.test.ts'", + "build": "tsc -p tsconfig.build.json", + "prepack": "pnpm build", + "bench": "node bench/bench-sync-restore.ts" + }, + "dependencies": { + "lz4js": "^0.2.0" + }, + "peerDependencies": { + "@platformatic/sqlite-js-stream-paged-vfs": "workspace:^" + }, + "peerDependenciesMeta": { + "@platformatic/sqlite-js-stream-paged-vfs": { + "optional": true + } + }, + "devDependencies": { + "@platformatic/sqlite-js-stream-paged-vfs": "workspace:*", + "@types/node": "^26.0.0", + "typescript": "^5.7.0" + }, + "keywords": [ + "sqlite", + "litestream", + "replication", + "backup", + "ltx", + "disaster-recovery" + ] +} diff --git a/src/backends/file/index.ts b/packages/sqlite-js-stream/src/backends/file/index.ts similarity index 100% rename from src/backends/file/index.ts rename to packages/sqlite-js-stream/src/backends/file/index.ts diff --git a/src/backends/file/leaser.ts b/packages/sqlite-js-stream/src/backends/file/leaser.ts similarity index 100% rename from src/backends/file/leaser.ts rename to packages/sqlite-js-stream/src/backends/file/leaser.ts diff --git a/src/backends/file/replica-client.ts b/packages/sqlite-js-stream/src/backends/file/replica-client.ts similarity index 100% rename from src/backends/file/replica-client.ts rename to packages/sqlite-js-stream/src/backends/file/replica-client.ts diff --git a/src/db.ts b/packages/sqlite-js-stream/src/db.ts similarity index 100% rename from src/db.ts rename to packages/sqlite-js-stream/src/db.ts diff --git a/src/errors.ts b/packages/sqlite-js-stream/src/errors.ts similarity index 100% rename from src/errors.ts rename to packages/sqlite-js-stream/src/errors.ts diff --git a/src/index.ts b/packages/sqlite-js-stream/src/index.ts similarity index 100% rename from src/index.ts rename to packages/sqlite-js-stream/src/index.ts diff --git a/src/leaser.ts b/packages/sqlite-js-stream/src/leaser.ts similarity index 100% rename from src/leaser.ts rename to packages/sqlite-js-stream/src/leaser.ts diff --git a/src/ltx/byte-reader.ts b/packages/sqlite-js-stream/src/ltx/byte-reader.ts similarity index 100% rename from src/ltx/byte-reader.ts rename to packages/sqlite-js-stream/src/ltx/byte-reader.ts diff --git a/src/ltx/bytes.ts b/packages/sqlite-js-stream/src/ltx/bytes.ts similarity index 100% rename from src/ltx/bytes.ts rename to packages/sqlite-js-stream/src/ltx/bytes.ts diff --git a/src/ltx/checksum.ts b/packages/sqlite-js-stream/src/ltx/checksum.ts similarity index 100% rename from src/ltx/checksum.ts rename to packages/sqlite-js-stream/src/ltx/checksum.ts diff --git a/src/ltx/compaction-level.ts b/packages/sqlite-js-stream/src/ltx/compaction-level.ts similarity index 100% rename from src/ltx/compaction-level.ts rename to packages/sqlite-js-stream/src/ltx/compaction-level.ts diff --git a/src/ltx/compactor.ts b/packages/sqlite-js-stream/src/ltx/compactor.ts similarity index 100% rename from src/ltx/compactor.ts rename to packages/sqlite-js-stream/src/ltx/compactor.ts diff --git a/src/ltx/constants.ts b/packages/sqlite-js-stream/src/ltx/constants.ts similarity index 100% rename from src/ltx/constants.ts rename to packages/sqlite-js-stream/src/ltx/constants.ts diff --git a/src/ltx/crc64.ts b/packages/sqlite-js-stream/src/ltx/crc64.ts similarity index 100% rename from src/ltx/crc64.ts rename to packages/sqlite-js-stream/src/ltx/crc64.ts diff --git a/src/ltx/decoder.ts b/packages/sqlite-js-stream/src/ltx/decoder.ts similarity index 100% rename from src/ltx/decoder.ts rename to packages/sqlite-js-stream/src/ltx/decoder.ts diff --git a/src/ltx/encoder.ts b/packages/sqlite-js-stream/src/ltx/encoder.ts similarity index 100% rename from src/ltx/encoder.ts rename to packages/sqlite-js-stream/src/ltx/encoder.ts diff --git a/src/ltx/file-info.ts b/packages/sqlite-js-stream/src/ltx/file-info.ts similarity index 100% rename from src/ltx/file-info.ts rename to packages/sqlite-js-stream/src/ltx/file-info.ts diff --git a/src/ltx/filename.ts b/packages/sqlite-js-stream/src/ltx/filename.ts similarity index 100% rename from src/ltx/filename.ts rename to packages/sqlite-js-stream/src/ltx/filename.ts diff --git a/src/ltx/header.ts b/packages/sqlite-js-stream/src/ltx/header.ts similarity index 100% rename from src/ltx/header.ts rename to packages/sqlite-js-stream/src/ltx/header.ts diff --git a/src/ltx/index.ts b/packages/sqlite-js-stream/src/ltx/index.ts similarity index 100% rename from src/ltx/index.ts rename to packages/sqlite-js-stream/src/ltx/index.ts diff --git a/src/ltx/lz4.ts b/packages/sqlite-js-stream/src/ltx/lz4.ts similarity index 100% rename from src/ltx/lz4.ts rename to packages/sqlite-js-stream/src/ltx/lz4.ts diff --git a/src/ltx/page-index.ts b/packages/sqlite-js-stream/src/ltx/page-index.ts similarity index 100% rename from src/ltx/page-index.ts rename to packages/sqlite-js-stream/src/ltx/page-index.ts diff --git a/src/ltx/page.ts b/packages/sqlite-js-stream/src/ltx/page.ts similarity index 100% rename from src/ltx/page.ts rename to packages/sqlite-js-stream/src/ltx/page.ts diff --git a/src/ltx/sink.ts b/packages/sqlite-js-stream/src/ltx/sink.ts similarity index 100% rename from src/ltx/sink.ts rename to packages/sqlite-js-stream/src/ltx/sink.ts diff --git a/src/ltx/stream.ts b/packages/sqlite-js-stream/src/ltx/stream.ts similarity index 100% rename from src/ltx/stream.ts rename to packages/sqlite-js-stream/src/ltx/stream.ts diff --git a/src/ltx/streaming-compactor.ts b/packages/sqlite-js-stream/src/ltx/streaming-compactor.ts similarity index 100% rename from src/ltx/streaming-compactor.ts rename to packages/sqlite-js-stream/src/ltx/streaming-compactor.ts diff --git a/src/ltx/trailer.ts b/packages/sqlite-js-stream/src/ltx/trailer.ts similarity index 100% rename from src/ltx/trailer.ts rename to packages/sqlite-js-stream/src/ltx/trailer.ts diff --git a/src/ltx/types.ts b/packages/sqlite-js-stream/src/ltx/types.ts similarity index 100% rename from src/ltx/types.ts rename to packages/sqlite-js-stream/src/ltx/types.ts diff --git a/src/lz4js.d.ts b/packages/sqlite-js-stream/src/lz4js.d.ts similarity index 100% rename from src/lz4js.d.ts rename to packages/sqlite-js-stream/src/lz4js.d.ts diff --git a/src/paged-restore.ts b/packages/sqlite-js-stream/src/paged-restore.ts similarity index 100% rename from src/paged-restore.ts rename to packages/sqlite-js-stream/src/paged-restore.ts diff --git a/src/replica-client.ts b/packages/sqlite-js-stream/src/replica-client.ts similarity index 100% rename from src/replica-client.ts rename to packages/sqlite-js-stream/src/replica-client.ts diff --git a/src/replica-url.ts b/packages/sqlite-js-stream/src/replica-url.ts similarity index 100% rename from src/replica-url.ts rename to packages/sqlite-js-stream/src/replica-url.ts diff --git a/src/replica.ts b/packages/sqlite-js-stream/src/replica.ts similarity index 100% rename from src/replica.ts rename to packages/sqlite-js-stream/src/replica.ts diff --git a/src/store.ts b/packages/sqlite-js-stream/src/store.ts similarity index 100% rename from src/store.ts rename to packages/sqlite-js-stream/src/store.ts diff --git a/src/util/fs.ts b/packages/sqlite-js-stream/src/util/fs.ts similarity index 100% rename from src/util/fs.ts rename to packages/sqlite-js-stream/src/util/fs.ts diff --git a/src/util/logger.ts b/packages/sqlite-js-stream/src/util/logger.ts similarity index 100% rename from src/util/logger.ts rename to packages/sqlite-js-stream/src/util/logger.ts diff --git a/src/util/stream.ts b/packages/sqlite-js-stream/src/util/stream.ts similarity index 100% rename from src/util/stream.ts rename to packages/sqlite-js-stream/src/util/stream.ts diff --git a/src/wal/checksum.ts b/packages/sqlite-js-stream/src/wal/checksum.ts similarity index 100% rename from src/wal/checksum.ts rename to packages/sqlite-js-stream/src/wal/checksum.ts diff --git a/src/wal/constants.ts b/packages/sqlite-js-stream/src/wal/constants.ts similarity index 100% rename from src/wal/constants.ts rename to packages/sqlite-js-stream/src/wal/constants.ts diff --git a/src/wal/index.ts b/packages/sqlite-js-stream/src/wal/index.ts similarity index 100% rename from src/wal/index.ts rename to packages/sqlite-js-stream/src/wal/index.ts diff --git a/src/wal/reader.ts b/packages/sqlite-js-stream/src/wal/reader.ts similarity index 100% rename from src/wal/reader.ts rename to packages/sqlite-js-stream/src/wal/reader.ts diff --git a/test/backends/file/delete-prefix.test.ts b/packages/sqlite-js-stream/test/backends/file/delete-prefix.test.ts similarity index 100% rename from test/backends/file/delete-prefix.test.ts rename to packages/sqlite-js-stream/test/backends/file/delete-prefix.test.ts diff --git a/test/backends/file/replica-client.test.ts b/packages/sqlite-js-stream/test/backends/file/replica-client.test.ts similarity index 100% rename from test/backends/file/replica-client.test.ts rename to packages/sqlite-js-stream/test/backends/file/replica-client.test.ts diff --git a/test/db/db-incremental.test.ts b/packages/sqlite-js-stream/test/db/db-incremental.test.ts similarity index 100% rename from test/db/db-incremental.test.ts rename to packages/sqlite-js-stream/test/db/db-incremental.test.ts diff --git a/test/errors.test.ts b/packages/sqlite-js-stream/test/errors.test.ts similarity index 100% rename from test/errors.test.ts rename to packages/sqlite-js-stream/test/errors.test.ts diff --git a/test/helpers/byte.ts b/packages/sqlite-js-stream/test/helpers/byte.ts similarity index 100% rename from test/helpers/byte.ts rename to packages/sqlite-js-stream/test/helpers/byte.ts diff --git a/test/helpers/continuous-worker.ts b/packages/sqlite-js-stream/test/helpers/continuous-worker.ts similarity index 100% rename from test/helpers/continuous-worker.ts rename to packages/sqlite-js-stream/test/helpers/continuous-worker.ts diff --git a/test/helpers/delete-prefix-suite.ts b/packages/sqlite-js-stream/test/helpers/delete-prefix-suite.ts similarity index 100% rename from test/helpers/delete-prefix-suite.ts rename to packages/sqlite-js-stream/test/helpers/delete-prefix-suite.ts diff --git a/test/helpers/fencing-suite.ts b/packages/sqlite-js-stream/test/helpers/fencing-suite.ts similarity index 100% rename from test/helpers/fencing-suite.ts rename to packages/sqlite-js-stream/test/helpers/fencing-suite.ts diff --git a/test/helpers/make-db.ts b/packages/sqlite-js-stream/test/helpers/make-db.ts similarity index 100% rename from test/helpers/make-db.ts rename to packages/sqlite-js-stream/test/helpers/make-db.ts diff --git a/test/helpers/make-ltx.ts b/packages/sqlite-js-stream/test/helpers/make-ltx.ts similarity index 100% rename from test/helpers/make-ltx.ts rename to packages/sqlite-js-stream/test/helpers/make-ltx.ts diff --git a/test/helpers/paged-restore-child.ts b/packages/sqlite-js-stream/test/helpers/paged-restore-child.ts similarity index 100% rename from test/helpers/paged-restore-child.ts rename to packages/sqlite-js-stream/test/helpers/paged-restore-child.ts diff --git a/test/helpers/replication-worker.ts b/packages/sqlite-js-stream/test/helpers/replication-worker.ts similarity index 100% rename from test/helpers/replication-worker.ts rename to packages/sqlite-js-stream/test/helpers/replication-worker.ts diff --git a/test/helpers/sql-worker.ts b/packages/sqlite-js-stream/test/helpers/sql-worker.ts similarity index 100% rename from test/helpers/sql-worker.ts rename to packages/sqlite-js-stream/test/helpers/sql-worker.ts diff --git a/test/integration/checkpoint-concurrency.test.ts b/packages/sqlite-js-stream/test/integration/checkpoint-concurrency.test.ts similarity index 100% rename from test/integration/checkpoint-concurrency.test.ts rename to packages/sqlite-js-stream/test/integration/checkpoint-concurrency.test.ts diff --git a/test/integration/compaction-retention.test.ts b/packages/sqlite-js-stream/test/integration/compaction-retention.test.ts similarity index 100% rename from test/integration/compaction-retention.test.ts rename to packages/sqlite-js-stream/test/integration/compaction-retention.test.ts diff --git a/test/integration/compaction-scheduling.test.ts b/packages/sqlite-js-stream/test/integration/compaction-scheduling.test.ts similarity index 100% rename from test/integration/compaction-scheduling.test.ts rename to packages/sqlite-js-stream/test/integration/compaction-scheduling.test.ts diff --git a/test/integration/continuous-sync.test.ts b/packages/sqlite-js-stream/test/integration/continuous-sync.test.ts similarity index 100% rename from test/integration/continuous-sync.test.ts rename to packages/sqlite-js-stream/test/integration/continuous-sync.test.ts diff --git a/test/integration/follow.test.ts b/packages/sqlite-js-stream/test/integration/follow.test.ts similarity index 100% rename from test/integration/follow.test.ts rename to packages/sqlite-js-stream/test/integration/follow.test.ts diff --git a/test/integration/incremental-checkpoint.test.ts b/packages/sqlite-js-stream/test/integration/incremental-checkpoint.test.ts similarity index 100% rename from test/integration/incremental-checkpoint.test.ts rename to packages/sqlite-js-stream/test/integration/incremental-checkpoint.test.ts diff --git a/test/integration/incremental-restore.test.ts b/packages/sqlite-js-stream/test/integration/incremental-restore.test.ts similarity index 100% rename from test/integration/incremental-restore.test.ts rename to packages/sqlite-js-stream/test/integration/incremental-restore.test.ts diff --git a/test/integration/leasing.test.ts b/packages/sqlite-js-stream/test/integration/leasing.test.ts similarity index 100% rename from test/integration/leasing.test.ts rename to packages/sqlite-js-stream/test/integration/leasing.test.ts diff --git a/test/integration/open-reader.test.ts b/packages/sqlite-js-stream/test/integration/open-reader.test.ts similarity index 100% rename from test/integration/open-reader.test.ts rename to packages/sqlite-js-stream/test/integration/open-reader.test.ts diff --git a/test/integration/paged-restore.test.ts b/packages/sqlite-js-stream/test/integration/paged-restore.test.ts similarity index 100% rename from test/integration/paged-restore.test.ts rename to packages/sqlite-js-stream/test/integration/paged-restore.test.ts diff --git a/test/integration/pitr-restore.test.ts b/packages/sqlite-js-stream/test/integration/pitr-restore.test.ts similarity index 100% rename from test/integration/pitr-restore.test.ts rename to packages/sqlite-js-stream/test/integration/pitr-restore.test.ts diff --git a/test/integration/replicate-restore.test.ts b/packages/sqlite-js-stream/test/integration/replicate-restore.test.ts similarity index 100% rename from test/integration/replicate-restore.test.ts rename to packages/sqlite-js-stream/test/integration/replicate-restore.test.ts diff --git a/test/integration/snapshot-store.test.ts b/packages/sqlite-js-stream/test/integration/snapshot-store.test.ts similarity index 100% rename from test/integration/snapshot-store.test.ts rename to packages/sqlite-js-stream/test/integration/snapshot-store.test.ts diff --git a/test/integration/streaming-restore.test.ts b/packages/sqlite-js-stream/test/integration/streaming-restore.test.ts similarity index 100% rename from test/integration/streaming-restore.test.ts rename to packages/sqlite-js-stream/test/integration/streaming-restore.test.ts diff --git a/test/integration/superseded.test.ts b/packages/sqlite-js-stream/test/integration/superseded.test.ts similarity index 100% rename from test/integration/superseded.test.ts rename to packages/sqlite-js-stream/test/integration/superseded.test.ts diff --git a/test/integration/worker-sync.test.ts b/packages/sqlite-js-stream/test/integration/worker-sync.test.ts similarity index 100% rename from test/integration/worker-sync.test.ts rename to packages/sqlite-js-stream/test/integration/worker-sync.test.ts diff --git a/test/ltx/checksum.test.ts b/packages/sqlite-js-stream/test/ltx/checksum.test.ts similarity index 100% rename from test/ltx/checksum.test.ts rename to packages/sqlite-js-stream/test/ltx/checksum.test.ts diff --git a/test/ltx/compactor.test.ts b/packages/sqlite-js-stream/test/ltx/compactor.test.ts similarity index 100% rename from test/ltx/compactor.test.ts rename to packages/sqlite-js-stream/test/ltx/compactor.test.ts diff --git a/test/ltx/crc64.test.ts b/packages/sqlite-js-stream/test/ltx/crc64.test.ts similarity index 100% rename from test/ltx/crc64.test.ts rename to packages/sqlite-js-stream/test/ltx/crc64.test.ts diff --git a/test/ltx/encoder-decoder.test.ts b/packages/sqlite-js-stream/test/ltx/encoder-decoder.test.ts similarity index 100% rename from test/ltx/encoder-decoder.test.ts rename to packages/sqlite-js-stream/test/ltx/encoder-decoder.test.ts diff --git a/test/ltx/filename.test.ts b/packages/sqlite-js-stream/test/ltx/filename.test.ts similarity index 100% rename from test/ltx/filename.test.ts rename to packages/sqlite-js-stream/test/ltx/filename.test.ts diff --git a/test/ltx/go-interop.test.ts b/packages/sqlite-js-stream/test/ltx/go-interop.test.ts similarity index 100% rename from test/ltx/go-interop.test.ts rename to packages/sqlite-js-stream/test/ltx/go-interop.test.ts diff --git a/test/ltx/golden.test.ts b/packages/sqlite-js-stream/test/ltx/golden.test.ts similarity index 100% rename from test/ltx/golden.test.ts rename to packages/sqlite-js-stream/test/ltx/golden.test.ts diff --git a/test/ltx/header.test.ts b/packages/sqlite-js-stream/test/ltx/header.test.ts similarity index 100% rename from test/ltx/header.test.ts rename to packages/sqlite-js-stream/test/ltx/header.test.ts diff --git a/test/ltx/lz4.test.ts b/packages/sqlite-js-stream/test/ltx/lz4.test.ts similarity index 100% rename from test/ltx/lz4.test.ts rename to packages/sqlite-js-stream/test/ltx/lz4.test.ts diff --git a/test/ltx/page-index.test.ts b/packages/sqlite-js-stream/test/ltx/page-index.test.ts similarity index 100% rename from test/ltx/page-index.test.ts rename to packages/sqlite-js-stream/test/ltx/page-index.test.ts diff --git a/test/ltx/page-trailer.test.ts b/packages/sqlite-js-stream/test/ltx/page-trailer.test.ts similarity index 100% rename from test/ltx/page-trailer.test.ts rename to packages/sqlite-js-stream/test/ltx/page-trailer.test.ts diff --git a/test/ltx/stream.test.ts b/packages/sqlite-js-stream/test/ltx/stream.test.ts similarity index 100% rename from test/ltx/stream.test.ts rename to packages/sqlite-js-stream/test/ltx/stream.test.ts diff --git a/test/ltx/streaming-compactor.test.ts b/packages/sqlite-js-stream/test/ltx/streaming-compactor.test.ts similarity index 100% rename from test/ltx/streaming-compactor.test.ts rename to packages/sqlite-js-stream/test/ltx/streaming-compactor.test.ts diff --git a/test/smoke.test.ts b/packages/sqlite-js-stream/test/smoke.test.ts similarity index 100% rename from test/smoke.test.ts rename to packages/sqlite-js-stream/test/smoke.test.ts diff --git a/test/wal/wal.test.ts b/packages/sqlite-js-stream/test/wal/wal.test.ts similarity index 100% rename from test/wal/wal.test.ts rename to packages/sqlite-js-stream/test/wal/wal.test.ts diff --git a/testdata/go/snapshot.ltx b/packages/sqlite-js-stream/testdata/go/snapshot.ltx similarity index 100% rename from testdata/go/snapshot.ltx rename to packages/sqlite-js-stream/testdata/go/snapshot.ltx diff --git a/testdata/go/sparse-non-snapshot.ltx b/packages/sqlite-js-stream/testdata/go/sparse-non-snapshot.ltx similarity index 100% rename from testdata/go/sparse-non-snapshot.ltx rename to packages/sqlite-js-stream/testdata/go/sparse-non-snapshot.ltx diff --git a/tools/golden/go.mod b/packages/sqlite-js-stream/tools/golden/go.mod similarity index 100% rename from tools/golden/go.mod rename to packages/sqlite-js-stream/tools/golden/go.mod diff --git a/tools/golden/go.sum b/packages/sqlite-js-stream/tools/golden/go.sum similarity index 100% rename from tools/golden/go.sum rename to packages/sqlite-js-stream/tools/golden/go.sum diff --git a/tools/golden/main.go b/packages/sqlite-js-stream/tools/golden/main.go similarity index 100% rename from tools/golden/main.go rename to packages/sqlite-js-stream/tools/golden/main.go diff --git a/tsconfig.build.json b/packages/sqlite-js-stream/tsconfig.build.json similarity index 59% rename from tsconfig.build.json rename to packages/sqlite-js-stream/tsconfig.build.json index f7bde81..2804928 100644 --- a/tsconfig.build.json +++ b/packages/sqlite-js-stream/tsconfig.build.json @@ -1,5 +1,5 @@ { - "extends": "./tsconfig.json", + "extends": "../../tsconfig.json", "compilerOptions": { "noEmit": false, "declaration": true, @@ -8,5 +8,5 @@ "rootDir": "src" }, "include": ["src/**/*.ts"], - "exclude": ["test", "packages", "node_modules", "dist", "bench"] + "exclude": ["test", "bench", "tools", "testdata", "docs", "node_modules", "dist"] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2f52a23..25fddda 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -165,14 +165,7 @@ settings: importers: .: - dependencies: - lz4js: - specifier: ^0.2.0 - version: 0.2.0 devDependencies: - '@platformatic/sqlite-js-stream-paged-vfs': - specifier: workspace:* - version: link:packages/sqlite-js-stream-paged-vfs '@types/node': specifier: ^26.0.0 version: 26.6.6 @@ -190,7 +183,7 @@ importers: dependencies: '@platformatic/sqlite-js-stream': specifier: workspace:* - version: link:../.. + version: link:../sqlite-js-stream postgres: specifier: ^4.0.0 version: 4.0.0 @@ -209,8 +202,24 @@ importers: version: 3.1162.0 '@platformatic/sqlite-js-stream': specifier: workspace:* - version: link:../.. + version: link:../sqlite-js-stream + devDependencies: + '@types/node': + specifier: ^26.0.0 + version: 26.6.6 + typescript: + specifier: ^5.7.0 + version: 5.9.3 + + packages/sqlite-js-stream: + dependencies: + lz4js: + specifier: ^0.2.0 + version: 0.2.0 devDependencies: + '@platformatic/sqlite-js-stream-paged-vfs': + specifier: workspace:* + version: link:../sqlite-js-stream-paged-vfs '@types/node': specifier: ^26.0.0 version: 26.6.6 diff --git a/scripts/bump-version.mjs b/scripts/bump-version.mjs index f58258b..a79f00f 100644 --- a/scripts/bump-version.mjs +++ b/scripts/bump-version.mjs @@ -4,8 +4,10 @@ * Usage: node scripts/bump-version.mjs [github-actor] * * The root package and every `packages/*` package are released in lockstep, - * so they all receive the same version. `workspace:*` dependencies are - * rewritten to the concrete version by `pnpm publish`. + * so they all receive the same version. The root package is private (not + * published); the base version is read from the core package and every + * `packages/*` package that ships a `package.json` is bumped. `workspace:*` + * dependencies are rewritten to the concrete version by `pnpm publish`. */ import { execFileSync } from 'node:child_process' import { readdir, readFile, writeFile } from 'node:fs/promises' @@ -47,7 +49,7 @@ async function getVersion () { if (!input) throw new Error('Missing version argument (version, patch, minor or major).') if (['major', 'minor', 'patch'].includes(input)) { - const { version } = await readJson('package.json') + const { version } = await readJson('packages/sqlite-js-stream/package.json') return inc(version, input) } @@ -60,10 +62,17 @@ async function getVersion () { async function getPackageFiles () { const entries = await readdir('packages', { withFileTypes: true }) - return [ - 'package.json', - ...entries.filter((e) => e.isDirectory()).map((e) => path.join('packages', e.name, 'package.json')), - ] + const files = [] + for (const entry of entries.filter((e) => e.isDirectory())) { + const file = path.join('packages', entry.name, 'package.json') + try { + await readFile(file, 'utf8') + files.push(file) + } catch { + // Ignore directories without a package.json (e.g. leftover node_modules). + } + } + return files } const userInfo = getUserInfo() diff --git a/tsconfig.json b/tsconfig.json index 1f77689..c6de4b9 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -19,6 +19,6 @@ "forceConsistentCasingInFileNames": true, "noEmit": true }, - "include": ["src/**/*.ts", "test/**/*.ts", "packages/**/*.ts"], - "exclude": ["node_modules", "dist"] + "include": ["packages/**/*.ts"], + "exclude": ["node_modules", "dist", "**/dist/**", "**/target/**"] }