diff --git a/README.md b/README.md
index 5931b1c..5a6923f 100644
--- a/README.md
+++ b/README.md
@@ -233,7 +233,7 @@ window.onload = connectWebSocket
```
SQL Dump
-You can request a `database_dump.sql` file that exports your database schema and data into a single file.
+You can request a `database_dump.sql` file that exports your database schema and data into a single file. Dumps are streamed incrementally in bounded memory pages:
@@ -243,6 +243,15 @@ curl --location 'https://starbasedb.YOUR-ID-HERE.workers.dev/export/dump' \
+For large databases, offload the dump directly to Cloudflare R2 storage without hitting worker execution time limits:
+
+
+
+curl --location 'https://starbasedb.YOUR-ID-HERE.workers.dev/export/dump?location=r2&callback=https://example.com/webhook' \
+--header 'Authorization: Bearer ABC123'
+
+
+
JSON Data Export
diff --git a/src/export/dump.test.ts b/src/export/dump.test.ts
index ca65b43..e01eba9 100644
--- a/src/export/dump.test.ts
+++ b/src/export/dump.test.ts
@@ -1,5 +1,11 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
-import { dumpDatabaseRoute } from './dump'
+import {
+ dumpDatabaseRoute,
+ escapeSqlValue,
+ formatDumpTimestamp,
+ generateDumpChunks,
+ uploadDumpToR2,
+} from './dump'
import { executeOperation } from '.'
import { createResponse } from '../utils'
import type { DataSource } from '../types'
@@ -39,107 +45,335 @@ beforeEach(() => {
})
describe('Database Dump Module', () => {
- it('should return a database dump when tables exist', async () => {
- vi.mocked(executeOperation)
- .mockResolvedValueOnce([{ name: 'users' }, { name: 'orders' }])
- .mockResolvedValueOnce([
- { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' },
- ])
- .mockResolvedValueOnce([
- { id: 1, name: 'Alice' },
- { id: 2, name: 'Bob' },
- ])
- .mockResolvedValueOnce([
- { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' },
- ])
- .mockResolvedValueOnce([
- { id: 1, total: 99.99 },
- { id: 2, total: 49.5 },
- ])
-
- const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
-
- expect(response).toBeInstanceOf(Response)
- expect(response.headers.get('Content-Type')).toBe(
- 'application/x-sqlite3'
- )
- expect(response.headers.get('Content-Disposition')).toBe(
- 'attachment; filename="database_dump.sql"'
- )
-
- const dumpText = await response.text()
- expect(dumpText).toContain(
- 'CREATE TABLE users (id INTEGER, name TEXT);'
- )
- expect(dumpText).toContain("INSERT INTO users VALUES (1, 'Alice');")
- expect(dumpText).toContain("INSERT INTO users VALUES (2, 'Bob');")
- expect(dumpText).toContain(
- 'CREATE TABLE orders (id INTEGER, total REAL);'
- )
- expect(dumpText).toContain('INSERT INTO orders VALUES (1, 99.99);')
- expect(dumpText).toContain('INSERT INTO orders VALUES (2, 49.5);')
+ describe('formatDumpTimestamp', () => {
+ it('should format UTC date components with zero-padding', () => {
+ const date = new Date(Date.UTC(2024, 0, 5, 8, 9, 7))
+ expect(formatDumpTimestamp(date)).toBe('20240105-080907')
+ })
+
+ it('should handle double digit months and days', () => {
+ const date = new Date(Date.UTC(2024, 11, 25, 23, 59, 59))
+ expect(formatDumpTimestamp(date)).toBe('20241225-235959')
+ })
})
- it('should handle empty databases (no tables)', async () => {
- vi.mocked(executeOperation).mockResolvedValueOnce([])
+ describe('escapeSqlValue', () => {
+ it('should format null and undefined as NULL', () => {
+ expect(escapeSqlValue(null)).toBe('NULL')
+ expect(escapeSqlValue(undefined)).toBe('NULL')
+ })
+
+ it('should format numbers and bigints as unquoted strings', () => {
+ expect(escapeSqlValue(42)).toBe('42')
+ expect(escapeSqlValue(3.14159)).toBe('3.14159')
+ expect(escapeSqlValue(BigInt(9007199254740991))).toBe(
+ '9007199254740991'
+ )
+ })
+
+ it('should format booleans as 1 and 0', () => {
+ expect(escapeSqlValue(true)).toBe('1')
+ expect(escapeSqlValue(false)).toBe('0')
+ })
- const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
+ it('should format ArrayBuffer and Uint8Array as hex literals', () => {
+ const bytes = new Uint8Array([0xde, 0xad, 0xbe, 0xef])
+ expect(escapeSqlValue(bytes)).toBe("X'deadbeef'")
+ expect(escapeSqlValue(bytes.buffer)).toBe("X'deadbeef'")
+ })
- expect(response).toBeInstanceOf(Response)
- expect(response.headers.get('Content-Type')).toBe(
- 'application/x-sqlite3'
- )
- const dumpText = await response.text()
- expect(dumpText).toBe('SQLite format 3\0')
+ it('should escape single quotes inside string values', () => {
+ expect(escapeSqlValue("Alice's book")).toBe("'Alice''s book'")
+ expect(escapeSqlValue("O''Reilly")).toBe("'O''''Reilly'")
+ expect(escapeSqlValue('Simple text')).toBe("'Simple text'")
+ })
})
- it('should handle databases with tables but no data', async () => {
- vi.mocked(executeOperation)
- .mockResolvedValueOnce([{ name: 'users' }])
- .mockResolvedValueOnce([
- { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' },
- ])
- .mockResolvedValueOnce([])
-
- const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
-
- expect(response).toBeInstanceOf(Response)
- const dumpText = await response.text()
- expect(dumpText).toContain(
- 'CREATE TABLE users (id INTEGER, name TEXT);'
- )
- expect(dumpText).not.toContain('INSERT INTO users VALUES')
+ describe('generateDumpChunks', () => {
+ it('should paginate table rows using LIMIT and OFFSET', async () => {
+ const tables = ['users']
+
+ vi.mocked(executeOperation)
+ // Schema query
+ .mockResolvedValueOnce([
+ { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' },
+ ])
+ // First page (pageSize = 2)
+ .mockResolvedValueOnce([
+ { id: 1, name: 'Alice' },
+ { id: 2, name: 'Bob' },
+ ])
+ // Second page
+ .mockResolvedValueOnce([{ id: 3, name: 'Charlie' }])
+
+ const chunks: string[] = []
+ for await (const chunk of generateDumpChunks(
+ tables,
+ mockDataSource,
+ mockConfig,
+ 2
+ )) {
+ chunks.push(chunk)
+ }
+
+ const fullDump = chunks.join('')
+ expect(fullDump).toContain('SQLite format 3\0')
+ expect(fullDump).toContain(
+ 'CREATE TABLE users (id INTEGER, name TEXT);'
+ )
+ expect(fullDump).toContain("INSERT INTO users VALUES (1, 'Alice');")
+ expect(fullDump).toContain("INSERT INTO users VALUES (2, 'Bob');")
+ expect(fullDump).toContain(
+ "INSERT INTO users VALUES (3, 'Charlie');"
+ )
+
+ // Verify LIMIT and OFFSET were queried
+ expect(executeOperation).toHaveBeenCalledWith(
+ [{ sql: 'SELECT * FROM "users" LIMIT 2 OFFSET 0;' }],
+ mockDataSource,
+ mockConfig
+ )
+ expect(executeOperation).toHaveBeenCalledWith(
+ [{ sql: 'SELECT * FROM "users" LIMIT 2 OFFSET 2;' }],
+ mockDataSource,
+ mockConfig
+ )
+ })
})
- it('should escape single quotes properly in string values', async () => {
- vi.mocked(executeOperation)
- .mockResolvedValueOnce([{ name: 'users' }])
- .mockResolvedValueOnce([
- { sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' },
- ])
- .mockResolvedValueOnce([{ id: 1, bio: "Alice's adventure" }])
-
- const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
-
- expect(response).toBeInstanceOf(Response)
- const dumpText = await response.text()
- expect(dumpText).toContain(
- "INSERT INTO users VALUES (1, 'Alice''s adventure');"
- )
+ describe('dumpDatabaseRoute streaming', () => {
+ it('should return a database dump when tables exist', async () => {
+ vi.mocked(executeOperation)
+ .mockResolvedValueOnce([{ name: 'users' }, { name: 'orders' }])
+ .mockResolvedValueOnce([
+ { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' },
+ ])
+ .mockResolvedValueOnce([
+ { id: 1, name: 'Alice' },
+ { id: 2, name: 'Bob' },
+ ])
+ .mockResolvedValueOnce([
+ { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' },
+ ])
+ .mockResolvedValueOnce([
+ { id: 1, total: 99.99 },
+ { id: 2, total: 49.5 },
+ ])
+
+ const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
+
+ expect(response).toBeInstanceOf(Response)
+ expect(response.headers.get('Content-Type')).toBe(
+ 'application/x-sqlite3'
+ )
+ expect(response.headers.get('Content-Disposition')).toBe(
+ 'attachment; filename="database_dump.sql"'
+ )
+
+ const dumpText = await response.text()
+ expect(dumpText).toContain(
+ 'CREATE TABLE users (id INTEGER, name TEXT);'
+ )
+ expect(dumpText).toContain("INSERT INTO users VALUES (1, 'Alice');")
+ expect(dumpText).toContain("INSERT INTO users VALUES (2, 'Bob');")
+ expect(dumpText).toContain(
+ 'CREATE TABLE orders (id INTEGER, total REAL);'
+ )
+ expect(dumpText).toContain('INSERT INTO orders VALUES (1, 99.99);')
+ expect(dumpText).toContain('INSERT INTO orders VALUES (2, 49.5);')
+ })
+
+ it('should handle empty databases (no tables)', async () => {
+ vi.mocked(executeOperation).mockResolvedValueOnce([])
+
+ const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
+
+ expect(response).toBeInstanceOf(Response)
+ expect(response.headers.get('Content-Type')).toBe(
+ 'application/x-sqlite3'
+ )
+ const dumpText = await response.text()
+ expect(dumpText).toBe('SQLite format 3\0')
+ })
+
+ it('should handle databases with tables but no data', async () => {
+ vi.mocked(executeOperation)
+ .mockResolvedValueOnce([{ name: 'users' }])
+ .mockResolvedValueOnce([
+ { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' },
+ ])
+ .mockResolvedValueOnce([])
+
+ const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
+
+ expect(response).toBeInstanceOf(Response)
+ const dumpText = await response.text()
+ expect(dumpText).toContain(
+ 'CREATE TABLE users (id INTEGER, name TEXT);'
+ )
+ expect(dumpText).not.toContain('INSERT INTO users VALUES')
+ })
+
+ it('should escape single quotes properly in string values', async () => {
+ vi.mocked(executeOperation)
+ .mockResolvedValueOnce([{ name: 'users' }])
+ .mockResolvedValueOnce([
+ { sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' },
+ ])
+ .mockResolvedValueOnce([{ id: 1, bio: "Alice's adventure" }])
+
+ const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
+
+ expect(response).toBeInstanceOf(Response)
+ const dumpText = await response.text()
+ expect(dumpText).toContain(
+ "INSERT INTO users VALUES (1, 'Alice''s adventure');"
+ )
+ })
+
+ it('should return a 500 response when an error occurs', async () => {
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.mocked(executeOperation).mockRejectedValue(
+ new Error('Database Error')
+ )
+
+ const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
+
+ expect(response.status).toBe(500)
+ const jsonResponse: { error: string } = await response.json()
+ expect(jsonResponse.error).toBe('Failed to create database dump')
+ })
})
- it('should return a 500 response when an error occurs', async () => {
- const consoleErrorMock = vi
- .spyOn(console, 'error')
- .mockImplementation(() => {})
- vi.mocked(executeOperation).mockRejectedValue(
- new Error('Database Error')
- )
+ describe('R2 Offload and uploadDumpToR2', () => {
+ it('should return 400 when location=r2 is requested without a bucket binding', async () => {
+ vi.mocked(executeOperation).mockResolvedValueOnce([])
+
+ const request = new Request(
+ 'https://api.starbasedb.com/export/dump?location=r2'
+ )
+ const response = await dumpDatabaseRoute(
+ mockDataSource,
+ mockConfig,
+ request
+ )
+
+ expect(response.status).toBe(400)
+ const body = (await response.json()) as any
+ expect(body.error).toContain('DATABASE_DUMP_BUCKET')
+ })
+
+ it('should execute multipart upload to R2 and trigger callback URL if provided', async () => {
+ const mockUploadPart = vi.fn().mockImplementation((partNumber) => ({
+ partNumber,
+ etag: `etag-${partNumber}`,
+ }))
+ const mockComplete = vi.fn().mockResolvedValue({})
+ const mockAbort = vi.fn().mockResolvedValue({})
+
+ const mockBucket = {
+ createMultipartUpload: vi.fn().mockResolvedValue({
+ uploadPart: mockUploadPart,
+ complete: mockComplete,
+ abort: mockAbort,
+ }),
+ } as any
+
+ const fetchSpy = vi
+ .spyOn(globalThis, 'fetch')
+ .mockResolvedValue(new Response('OK'))
+
+ vi.mocked(executeOperation)
+ .mockResolvedValueOnce([
+ { sql: 'CREATE TABLE users (id INT);' },
+ ])
+ .mockResolvedValueOnce([{ id: 1 }])
+
+ await uploadDumpToR2(
+ mockBucket,
+ 'dump_test.sql',
+ ['users'],
+ mockDataSource,
+ mockConfig,
+ 'https://webhook.site/test'
+ )
+
+ expect(mockBucket.createMultipartUpload).toHaveBeenCalledWith(
+ 'dump_test.sql'
+ )
+ expect(mockUploadPart).toHaveBeenCalled()
+ expect(mockComplete).toHaveBeenCalled()
+ expect(fetchSpy).toHaveBeenCalledWith(
+ 'https://webhook.site/test',
+ expect.objectContaining({
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ })
+ )
+ })
+
+ it('should abort multipart upload if an error occurs during uploadDumpToR2', async () => {
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ const mockAbort = vi.fn().mockResolvedValue({})
+ const mockBucket = {
+ createMultipartUpload: vi.fn().mockResolvedValue({
+ uploadPart: vi
+ .fn()
+ .mockRejectedValue(new Error('Network upload failed')),
+ complete: vi.fn(),
+ abort: mockAbort,
+ }),
+ } as any
+
+ vi.mocked(executeOperation).mockRejectedValue(
+ new Error('Query execution error')
+ )
+
+ await expect(
+ uploadDumpToR2(
+ mockBucket,
+ 'dump_failed.sql',
+ ['users'],
+ mockDataSource,
+ mockConfig
+ )
+ ).rejects.toThrow('Query execution error')
+
+ expect(mockAbort).toHaveBeenCalled()
+ })
+
+ it('should return 202 Accepted when executionContext.waitUntil is available', async () => {
+ const waitUntil = vi.fn()
+ const mockBucket = {
+ createMultipartUpload: vi.fn().mockResolvedValue({
+ uploadPart: vi.fn().mockResolvedValue({
+ partNumber: 1,
+ etag: 'etag-1',
+ }),
+ complete: vi.fn().mockResolvedValue({}),
+ abort: vi.fn(),
+ }),
+ } as any
+
+ mockDataSource.dumpBucket = mockBucket
+ mockDataSource.executionContext = { waitUntil } as any
+
+ vi.mocked(executeOperation).mockResolvedValueOnce([])
- const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
+ const request = new Request(
+ 'https://api.starbasedb.com/export/dump?location=r2'
+ )
+ const response = await dumpDatabaseRoute(
+ mockDataSource,
+ mockConfig,
+ request
+ )
- expect(response.status).toBe(500)
- const jsonResponse: { error: string } = await response.json()
- expect(jsonResponse.error).toBe('Failed to create database dump')
+ expect(response.status).toBe(202)
+ const body = (await response.json()) as any
+ expect(body.result.status).toBe('accepted')
+ expect(body.result.key).toMatch(/^dump_\d{8}-\d{6}\.sql$/)
+ expect(waitUntil).toHaveBeenCalled()
+ })
})
})
diff --git a/src/export/dump.ts b/src/export/dump.ts
index 91a2e89..9dd331a 100644
--- a/src/export/dump.ts
+++ b/src/export/dump.ts
@@ -3,67 +3,314 @@ import { StarbaseDBConfiguration } from '../handler'
import { DataSource } from '../types'
import { createResponse } from '../utils'
-export async function dumpDatabaseRoute(
+// SQLite file header written at the top of every dump.
+const SQLITE_HEADER = 'SQLite format 3\0'
+
+// Number of rows fetched per page. Paging keeps the amount of data held in
+// memory bounded regardless of how large the underlying table is, so even a
+// multi-gigabyte table can be dumped without loading it all at once.
+const DUMP_PAGE_SIZE = 1000
+
+// R2 requires every part of a multipart upload (except the final one) to be at
+// least 5 MiB. We buffer encoded chunks until they cross this threshold before
+// flushing a part, which lets us stream an arbitrarily large dump into a single
+// R2 object without ever holding the whole file in memory.
+const R2_MIN_PART_SIZE = 5 * 1024 * 1024
+
+/**
+ * Format a timestamp into the `YYYYMMDD-HHMMSS` form used for dump filenames,
+ * e.g. `2024-01-01 17:00:00` UTC becomes `20240101-170000`.
+ */
+export function formatDumpTimestamp(date: Date): string {
+ const pad = (value: number) => String(value).padStart(2, '0')
+ const year = date.getUTCFullYear()
+ const month = pad(date.getUTCMonth() + 1)
+ const day = pad(date.getUTCDate())
+ const hours = pad(date.getUTCHours())
+ const minutes = pad(date.getUTCMinutes())
+ const seconds = pad(date.getUTCSeconds())
+ return `${year}${month}${day}-${hours}${minutes}${seconds}`
+}
+
+/**
+ * Convert a single column value into its SQL literal representation. Handles the
+ * boundary cases: NULL/undefined, booleans, and binary BLOB values, in addition to
+ * escaping single quotes inside strings.
+ */
+export function escapeSqlValue(value: unknown): string {
+ if (value === null || value === undefined) {
+ return 'NULL'
+ }
+
+ if (typeof value === 'number' || typeof value === 'bigint') {
+ return String(value)
+ }
+
+ if (typeof value === 'boolean') {
+ return value ? '1' : '0'
+ }
+
+ if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) {
+ const bytes =
+ value instanceof ArrayBuffer
+ ? new Uint8Array(value)
+ : new Uint8Array(
+ value.buffer,
+ value.byteOffset,
+ value.byteLength
+ )
+ let hex = ''
+ for (const byte of bytes) {
+ hex += byte.toString(16).padStart(2, '0')
+ }
+ return `X'${hex}'`
+ }
+
+ return `'${String(value).replace(/'/g, "''")}'`
+}
+
+/**
+ * Stream the database dump as a sequence of string chunks. The full database is
+ * never materialised in memory at once: table data is paged with LIMIT/OFFSET so
+ * each iteration only holds a single page of rows.
+ */
+export async function* generateDumpChunks(
+ tables: string[],
dataSource: DataSource,
- config: StarbaseDBConfiguration
-): Promise {
- try {
- // Get all table names
- const tablesResult = await executeOperation(
- [{ sql: "SELECT name FROM sqlite_master WHERE type='table';" }],
+ config: StarbaseDBConfiguration,
+ pageSize: number = DUMP_PAGE_SIZE
+): AsyncGenerator {
+ yield SQLITE_HEADER
+
+ for (const table of tables) {
+ // Get table schema
+ const schemaResult = await executeOperation(
+ [
+ {
+ sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`,
+ },
+ ],
dataSource,
config
)
- const tables = tablesResult.map((row: any) => row.name)
- let dumpContent = 'SQLite format 3\0' // SQLite file header
+ if (schemaResult.length) {
+ const schema = schemaResult[0].sql
+ yield `\n-- Table: ${table}\n${schema};\n\n`
+ }
- // Iterate through all tables
- for (const table of tables) {
- // Get table schema
- const schemaResult = await executeOperation(
+ // Page through the table data so a large table never has to be loaded
+ // into memory in its entirety.
+ let offset = 0
+ while (true) {
+ const rows = await executeOperation(
[
{
- sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`,
+ sql: `SELECT * FROM "${table}" LIMIT ${pageSize} OFFSET ${offset};`,
},
],
dataSource,
config
)
- if (schemaResult.length) {
- const schema = schemaResult[0].sql
- dumpContent += `\n-- Table: ${table}\n${schema};\n\n`
+ if (!rows.length) {
+ break
+ }
+
+ let chunk = ''
+ for (const row of rows) {
+ const values = Object.values(row).map(escapeSqlValue)
+ chunk += `INSERT INTO ${table} VALUES (${values.join(', ')});\n`
+ }
+ yield chunk
+
+ // A short page means we have reached the end of the table.
+ if (rows.length < pageSize) {
+ break
+ }
+ offset += pageSize
+ }
+
+ yield '\n'
+ }
+}
+
+/**
+ * Fetch the list of user table names from the database.
+ */
+async function getTableNames(
+ dataSource: DataSource,
+ config: StarbaseDBConfiguration
+): Promise {
+ const tablesResult = await executeOperation(
+ [{ sql: "SELECT name FROM sqlite_master WHERE type='table';" }],
+ dataSource,
+ config
+ )
+
+ return tablesResult.map((row: any) => row.name)
+}
+
+/**
+ * Upload a database dump to R2 using a multipart upload. Encoded chunks are
+ * buffered until they exceed R2's minimum part size (5 MiB) and then flushed as a part,
+ * allowing a dump of any size to be written without buffering the whole file.
+ * When a `callbackUrl` is provided it is notified once the upload completes.
+ */
+export async function uploadDumpToR2(
+ bucket: R2Bucket,
+ key: string,
+ tables: string[],
+ dataSource: DataSource,
+ config: StarbaseDBConfiguration,
+ callbackUrl?: string
+): Promise {
+ const encoder = new TextEncoder()
+ const upload = await bucket.createMultipartUpload(key)
+ const parts: R2UploadedPart[] = []
+ let pending: Uint8Array[] = []
+ let pendingBytes = 0
+
+ const flush = async () => {
+ if (pendingBytes === 0) {
+ return
+ }
+
+ const part = new Uint8Array(pendingBytes)
+ let offset = 0
+ for (const buffer of pending) {
+ part.set(buffer, offset)
+ offset += buffer.length
+ }
+
+ const uploaded = await upload.uploadPart(parts.length + 1, part)
+ parts.push(uploaded)
+ pending = []
+ pendingBytes = 0
+ }
+
+ try {
+ for await (const chunk of generateDumpChunks(
+ tables,
+ dataSource,
+ config
+ )) {
+ const encoded = encoder.encode(chunk)
+ pending.push(encoded)
+ pendingBytes += encoded.length
+
+ if (pendingBytes >= R2_MIN_PART_SIZE) {
+ await flush()
+ }
+ }
+
+ // Flush whatever remains as the final part (which may be < 5 MiB).
+ await flush()
+ await upload.complete(parts)
+ } catch (error) {
+ // Abort the multipart upload so we never leave a partial object behind.
+ try {
+ await upload.abort()
+ } catch (abortError) {
+ console.error('Failed to abort R2 multipart upload:', abortError)
+ }
+ throw error
+ }
+
+ if (callbackUrl) {
+ try {
+ await fetch(callbackUrl, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ key, status: 'completed' }),
+ })
+ } catch (error) {
+ console.error('Failed to notify dump callback URL:', error)
+ }
+ }
+}
+
+export async function dumpDatabaseRoute(
+ dataSource: DataSource,
+ config: StarbaseDBConfiguration,
+ request?: Request
+): Promise {
+ try {
+ const url = request ? new URL(request.url) : undefined
+ const wantsR2 =
+ url?.searchParams.get('location') === 'r2' ||
+ url?.searchParams.get('r2') === 'true'
+ const callbackUrl = url?.searchParams.get('callback') ?? undefined
+
+ // Resolve the table list up-front so connection/auth errors surface as a
+ // normal error response before we start streaming a body.
+ const tables = await getTableNames(dataSource, config)
+
+ // When the caller asks to offload to R2 we stream the dump into an R2
+ // object instead of the response body. This removes the 30 second
+ // request limit from the equation for very large databases: the upload
+ // continues in the background and the optional callback URL notifies the
+ // caller once the file is ready.
+ if (wantsR2) {
+ const bucket = dataSource.dumpBucket
+ if (!bucket) {
+ return createResponse(
+ undefined,
+ 'R2 bucket binding (DATABASE_DUMP_BUCKET) is required to offload a dump.',
+ 400
+ )
}
- // Get table data
- const dataResult = await executeOperation(
- [{ sql: `SELECT * FROM ${table};` }],
+ const key = `dump_${formatDumpTimestamp(new Date())}.sql`
+ const work = uploadDumpToR2(
+ bucket,
+ key,
+ tables,
dataSource,
- config
+ config,
+ callbackUrl
)
- for (const row of dataResult) {
- const values = Object.values(row).map((value) =>
- typeof value === 'string'
- ? `'${value.replace(/'/g, "''")}'`
- : value
+ const ctx = dataSource.executionContext
+ if (ctx?.waitUntil) {
+ ctx.waitUntil(work)
+ return createResponse(
+ { key, status: 'accepted' },
+ undefined,
+ 202
)
- dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n`
}
- dumpContent += '\n'
+ await work
+ return createResponse({ key, status: 'completed' }, undefined, 200)
}
- // Create a Blob from the dump content
- const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' })
+ // Default behaviour: stream the dump straight back to the caller. The
+ // response is produced incrementally so the database is never buffered
+ // in memory in its entirety.
+ const encoder = new TextEncoder()
+ const iterator = generateDumpChunks(tables, dataSource, config)
+ const stream = new ReadableStream({
+ async pull(controller) {
+ try {
+ const { value, done } = await iterator.next()
+ if (done) {
+ controller.close()
+ } else {
+ controller.enqueue(encoder.encode(value))
+ }
+ } catch (error) {
+ controller.error(error)
+ }
+ },
+ })
const headers = new Headers({
'Content-Type': 'application/x-sqlite3',
'Content-Disposition': 'attachment; filename="database_dump.sql"',
})
- return new Response(blob, { headers })
+ return new Response(stream, { headers })
} catch (error: any) {
console.error('Database Dump Error:', error)
return createResponse(undefined, 'Failed to create database dump', 500)
diff --git a/src/handler.ts b/src/handler.ts
index 3fa0085..ec0590a 100644
--- a/src/handler.ts
+++ b/src/handler.ts
@@ -120,8 +120,12 @@ export class StarbaseDB {
}
if (this.getFeature('export')) {
- this.app.get('/export/dump', this.isInternalSource, async () => {
- return dumpDatabaseRoute(this.dataSource, this.config)
+ this.app.get('/export/dump', this.isInternalSource, async (c) => {
+ return dumpDatabaseRoute(
+ this.dataSource,
+ this.config,
+ c.req.raw
+ )
})
this.app.get(
diff --git a/src/index.ts b/src/index.ts
index 4d08932..b8a39e7 100644
--- a/src/index.ts
+++ b/src/index.ts
@@ -56,6 +56,9 @@ export interface Env {
HYPERDRIVE: Hyperdrive
+ // Optional R2 bucket binding used to offload large database dumps.
+ DATABASE_DUMP_BUCKET?: R2Bucket
+
// ## DO NOT REMOVE: TEMPLATE INTERFACE ##
}
@@ -184,6 +187,10 @@ export default {
}
}
+ if (env.DATABASE_DUMP_BUCKET) {
+ dataSource.dumpBucket = env.DATABASE_DUMP_BUCKET
+ }
+
const config: StarbaseDBConfiguration = {
outerbaseApiKey: env.OUTERBASE_API_KEY,
role,
diff --git a/src/types.ts b/src/types.ts
index b0a59f4..f6afb3d 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -64,6 +64,8 @@ export type DataSource = {
cacheTTL?: number
registry?: StarbasePluginRegistry
executionContext?: ExecutionContext
+ // Optional R2 bucket used to offload large database dumps.
+ dumpBucket?: R2Bucket
}
export enum RegionLocationHint {
diff --git a/wrangler.toml b/wrangler.toml
index 395c4ac..d7d1f4d 100644
--- a/wrangler.toml
+++ b/wrangler.toml
@@ -78,3 +78,8 @@ AUTH_JWKS_ENDPOINT = ""
# [[hyperdrive]]
# binding = "HYPERDRIVE"
# id = ""
+
+# Cloudflare R2 bucket for offloading large database dumps
+# [[r2_buckets]]
+# binding = "DATABASE_DUMP_BUCKET"
+# bucket_name = "my-starbase-dumps"