diff --git a/src/export/dump.test.ts b/src/export/dump.test.ts index ca65b43..d62b500 100644 --- a/src/export/dump.test.ts +++ b/src/export/dump.test.ts @@ -1,145 +1,229 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest' -import { dumpDatabaseRoute } from './dump' -import { executeOperation } from '.' -import { createResponse } from '../utils' -import type { DataSource } from '../types' -import type { StarbaseDBConfiguration } from '../handler' +import { describe, expect, it, vi } from 'vitest' + +const executeOperation = vi.fn() vi.mock('.', () => ({ - executeOperation: vi.fn(), + executeOperation, })) vi.mock('../utils', () => ({ - createResponse: vi.fn( - (data, message, status) => - new Response(JSON.stringify({ result: data, error: message }), { - status, - headers: { 'Content-Type': 'application/json' }, - }) - ), + createResponse: (_data: unknown, message: string, status: number) => + new Response(message, { status }), })) -let mockDataSource: DataSource -let mockConfig: StarbaseDBConfiguration - -beforeEach(() => { - vi.clearAllMocks() - - mockDataSource = { - source: 'external', - external: { dialect: 'sqlite' }, - rpc: { executeQuery: vi.fn() }, - } as any - - mockConfig = { - outerbaseApiKey: 'mock-api-key', - role: 'admin', - features: { allowlist: true, rls: true, rest: true }, - } -}) +import { dumpDatabaseRoute } from './dump' -describe('Database Dump Module', () => { - it('should return a database dump when tables exist', async () => { - vi.mocked(executeOperation) - .mockResolvedValueOnce([{ name: 'users' }, { name: 'orders' }]) - .mockResolvedValueOnce([ - { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, - ]) - .mockResolvedValueOnce([ - { id: 1, name: 'Alice' }, - { id: 2, name: 'Bob' }, - ]) - .mockResolvedValueOnce([ - { sql: 'CREATE TABLE orders (id INTEGER, total REAL);' }, - ]) - .mockResolvedValueOnce([ - { id: 1, total: 99.99 }, - { id: 2, total: 49.5 }, - ]) - - const response = await dumpDatabaseRoute(mockDataSource, mockConfig) - - expect(response).toBeInstanceOf(Response) - expect(response.headers.get('Content-Type')).toBe( - 'application/x-sqlite3' - ) - expect(response.headers.get('Content-Disposition')).toBe( - 'attachment; filename="database_dump.sql"' +async function responseText(response: Response): Promise { + return await response.text() +} + +describe('dumpDatabaseRoute', () => { + it('streams a valid SQL dump without constructing a Blob', async () => { + executeOperation.mockImplementation(async (queries: any[]) => { + const { sql, params } = queries[0] + + if (sql.includes("FROM sqlite_master") && sql.includes("type = 'table'")) { + return [{ name: 'users' }] + } + + if (sql.includes('WHERE type = \'table\' AND name = ?')) { + expect(params).toEqual(['users']) + return [{ sql: 'CREATE TABLE users (id INTEGER, name TEXT)' }] + } + + if (sql.includes('SELECT _rowid_')) { + const after = params[0] + + if (after === 0) { + return [ + { + __starbase_rowid__: 1, + id: 1, + name: "O'Reilly", + }, + ] + } + + return [] + } + + throw new Error(`Unexpected SQL: ${sql}`) + }) + + const response = await dumpDatabaseRoute( + {} as any, + { role: 'admin' }, + 1 ) - const dumpText = await response.text() - expect(dumpText).toContain( - 'CREATE TABLE users (id INTEGER, name TEXT);' + expect(response.status).toBe(200) + expect(response.headers.get('Content-Type')).toContain('application/sql') + expect(response.headers.get('Content-Disposition')).toContain( + 'database_dump.sql' ) - expect(dumpText).toContain("INSERT INTO users VALUES (1, 'Alice');") - expect(dumpText).toContain("INSERT INTO users VALUES (2, 'Bob');") - expect(dumpText).toContain( - 'CREATE TABLE orders (id INTEGER, total REAL);' + + expect(await responseText(response)).toBe( + [ + 'BEGIN TRANSACTION;', + '', + '-- Table: "users"', + 'CREATE TABLE users (id INTEGER, name TEXT);', + `INSERT INTO "users" ("id", "name") VALUES (1, 'O''Reilly');`, + 'COMMIT;', + '', + ].join('\n') ) - expect(dumpText).toContain('INSERT INTO orders VALUES (1, 99.99);') - expect(dumpText).toContain('INSERT INTO orders VALUES (2, 49.5);') }) - it('should handle empty databases (no tables)', async () => { - vi.mocked(executeOperation).mockResolvedValueOnce([]) + it('uses keyset pagination and never emits OFFSET for rowid tables', async () => { + const observedSql: string[] = [] - const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + executeOperation.mockImplementation(async (queries: any[]) => { + const { sql, params } = queries[0] + observedSql.push(sql) - expect(response).toBeInstanceOf(Response) - expect(response.headers.get('Content-Type')).toBe( - 'application/x-sqlite3' - ) - const dumpText = await response.text() - expect(dumpText).toBe('SQLite format 3\0') - }) + if (sql.includes("FROM sqlite_master") && sql.includes("type = 'table'")) { + return [{ name: 'events' }] + } + + if (sql.includes('WHERE type = \'table\' AND name = ?')) { + return [{ sql: 'CREATE TABLE events (id INTEGER)' }] + } + + if (sql.includes('SELECT _rowid_')) { + const after = params[0] + + if (after === 0) { + return [ + { __starbase_rowid__: 10, id: 10 }, + { __starbase_rowid__: 20, id: 20 }, + ] + } - it('should handle databases with tables but no data', async () => { - vi.mocked(executeOperation) - .mockResolvedValueOnce([{ name: 'users' }]) - .mockResolvedValueOnce([ - { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, - ]) - .mockResolvedValueOnce([]) + if (after === 20) { + return [{ __starbase_rowid__: 30, id: 30 }] + } - const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + return [] + } - expect(response).toBeInstanceOf(Response) - const dumpText = await response.text() - expect(dumpText).toContain( - 'CREATE TABLE users (id INTEGER, name TEXT);' + throw new Error(`Unexpected SQL: ${sql}`) + }) + + const response = await dumpDatabaseRoute( + {} as any, + { role: 'admin' }, + 2 ) - expect(dumpText).not.toContain('INSERT INTO users VALUES') + + const output = await responseText(response) + + expect(output.match(/INSERT INTO "events"/g)).toHaveLength(3) + expect(observedSql.some((sql) => sql.includes('_rowid_ > ?'))).toBe(true) + expect(observedSql.some((sql) => sql.includes('OFFSET'))).toBe(false) }) - it('should escape single quotes properly in string values', async () => { - vi.mocked(executeOperation) - .mockResolvedValueOnce([{ name: 'users' }]) - .mockResolvedValueOnce([ - { sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' }, - ]) - .mockResolvedValueOnce([{ id: 1, bio: "Alice's adventure" }]) + it('falls back to bounded OFFSET pagination for WITHOUT ROWID tables', async () => { + executeOperation.mockImplementation(async (queries: any[]) => { + const { sql, params } = queries[0] + + if (sql.includes("FROM sqlite_master") && sql.includes("type = 'table'")) { + return [{ name: 'lookup' }] + } + + if (sql.includes('WHERE type = \'table\' AND name = ?')) { + return [ + { + sql: 'CREATE TABLE lookup (key TEXT PRIMARY KEY) WITHOUT ROWID', + }, + ] + } + + if (sql.includes('SELECT _rowid_')) { + throw new Error('no such column: _rowid_') + } + + if (sql.includes('LIMIT ? OFFSET ?')) { + return params[1] === 0 ? [{ key: 'first' }] : [] + } + + throw new Error(`Unexpected SQL: ${sql}`) + }) + + const response = await dumpDatabaseRoute( + {} as any, + { role: 'admin' }, + 1 + ) - const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + const output = await responseText(response) - expect(response).toBeInstanceOf(Response) - const dumpText = await response.text() - expect(dumpText).toContain( - "INSERT INTO users VALUES (1, 'Alice''s adventure');" + expect(output).toContain( + `INSERT INTO "lookup" ("key") VALUES ('first');` ) }) - it('should return a 500 response when an error occurs', async () => { - const consoleErrorMock = vi - .spyOn(console, 'error') - .mockImplementation(() => {}) - vi.mocked(executeOperation).mockRejectedValue( - new Error('Database Error') + it('serializes nulls, booleans, blobs, numbers, and quoted strings correctly', async () => { + executeOperation.mockImplementation(async (queries: any[]) => { + const { sql, params } = queries[0] + + if (sql.includes("FROM sqlite_master") && sql.includes("type = 'table'")) { + return [{ name: 'values' }] + } + + if (sql.includes('WHERE type = \'table\' AND name = ?')) { + return [{ sql: 'CREATE TABLE values (a, b, c, d, e)' }] + } + + if (sql.includes('SELECT _rowid_')) { + return params[0] === 0 + ? [ + { + __starbase_rowid__: 1, + a: null, + b: true, + c: 42.5, + d: new Uint8Array([0, 15, 255]), + e: "it's safe", + }, + ] + : [] + } + + throw new Error(`Unexpected SQL: ${sql}`) + }) + + const response = await dumpDatabaseRoute( + {} as any, + { role: 'admin' }, + 10 ) - const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + expect(await responseText(response)).toContain( + `VALUES (NULL, 1, 42.5, X'000fff', 'it''s safe');` + ) + }) + + it('returns a normal 500 response when metadata resolution fails', async () => { + executeOperation.mockRejectedValueOnce(new Error('storage unavailable')) + + const response = await dumpDatabaseRoute( + {} as any, + { role: 'admin' } + ) expect(response.status).toBe(500) - const jsonResponse: { error: string } = await response.json() - expect(jsonResponse.error).toBe('Failed to create database dump') + expect(await response.text()).toBe('Failed to create database dump') + }) + + it('rejects invalid page sizes before querying the Durable Object', async () => { + const response = await dumpDatabaseRoute( + {} as any, + { role: 'admin' }, + 0 + ) + + expect(response.status).toBe(400) + expect(executeOperation).not.toHaveBeenCalled() }) }) diff --git a/src/export/dump.ts b/src/export/dump.ts index 91a2e89..417e055 100644 --- a/src/export/dump.ts +++ b/src/export/dump.ts @@ -3,68 +3,259 @@ import { StarbaseDBConfiguration } from '../handler' import { DataSource } from '../types' import { createResponse } from '../utils' -export async function dumpDatabaseRoute( +const encoder = new TextEncoder() +const DEFAULT_PAGE_SIZE = 500 + +type Row = Record + +function quoteIdentifier(identifier: string): string { + return `"${identifier.replace(/"/g, '""')}"` +} + +function sqlLiteral(value: unknown): string { + if (value === null || value === undefined) return 'NULL' + + if (typeof value === 'number') { + if (!Number.isFinite(value)) { + throw new Error('Cannot export non-finite numeric SQLite values') + } + return String(value) + } + + if (typeof value === 'bigint') return value.toString() + + if (typeof value === 'boolean') return value ? '1' : '0' + + if (value instanceof Uint8Array) { + let hex = '' + for (const byte of value) hex += byte.toString(16).padStart(2, '0') + return `X'${hex}'` + } + + return `'${String(value).replace(/'/g, "''")}'` +} + +async function breathe(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)) +} + +async function getTables( dataSource: DataSource, config: StarbaseDBConfiguration -): Promise { - try { - // Get all table names - const tablesResult = await executeOperation( - [{ sql: "SELECT name FROM sqlite_master WHERE type='table';" }], - dataSource, - config - ) - - const tables = tablesResult.map((row: any) => row.name) - let dumpContent = 'SQLite format 3\0' // SQLite file header - - // Iterate through all tables - for (const table of tables) { - // Get table schema - const schemaResult = await executeOperation( - [ - { - sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`, - }, - ], - dataSource, - config - ) - - if (schemaResult.length) { - const schema = schemaResult[0].sql - dumpContent += `\n-- Table: ${table}\n${schema};\n\n` - } +): Promise { + const result = await executeOperation( + [ + { + sql: ` + SELECT name + FROM sqlite_master + WHERE type = 'table' + AND name NOT LIKE 'sqlite_%' + ORDER BY name + `, + }, + ], + dataSource, + config + ) + + return result.map((row: Row) => String(row.name)) +} + +async function getSchema( + table: string, + dataSource: DataSource, + config: StarbaseDBConfiguration +): Promise { + const result = await executeOperation( + [ + { + sql: ` + SELECT sql + FROM sqlite_master + WHERE type = 'table' AND name = ? + `, + params: [table], + }, + ], + dataSource, + config + ) + + return result.length > 0 && typeof result[0].sql === 'string' + ? result[0].sql + : null +} + +async function fetchRowIdPage( + table: string, + afterRowId: number, + pageSize: number, + dataSource: DataSource, + config: StarbaseDBConfiguration +): Promise> { + return executeOperation( + [ + { + sql: ` + SELECT _rowid_ AS __starbase_rowid__, * + FROM ${quoteIdentifier(table)} + WHERE _rowid_ > ? + ORDER BY _rowid_ + LIMIT ? + `, + params: [afterRowId, pageSize], + }, + ], + dataSource, + config + ) as Promise> +} - // Get table data - const dataResult = await executeOperation( - [{ sql: `SELECT * FROM ${table};` }], - dataSource, - config - ) - - for (const row of dataResult) { - const values = Object.values(row).map((value) => - typeof value === 'string' - ? `'${value.replace(/'/g, "''")}'` - : value +async function fetchOffsetPage( + table: string, + offset: number, + pageSize: number, + dataSource: DataSource, + config: StarbaseDBConfiguration +): Promise { + return executeOperation( + [ + { + sql: ` + SELECT * + FROM ${quoteIdentifier(table)} + LIMIT ? OFFSET ? + `, + params: [pageSize, offset], + }, + ], + dataSource, + config + ) as Promise +} + +async function* dumpChunks( + tables: string[], + dataSource: DataSource, + config: StarbaseDBConfiguration, + pageSize: number +): AsyncGenerator { + yield 'BEGIN TRANSACTION;\n' + + for (const table of tables) { + const schema = await getSchema(table, dataSource, config) + if (!schema) continue + + yield `\n-- Table: ${quoteIdentifier(table)}\n${schema};\n` + + let rowId = 0 + let offset = 0 + let usesRowId = true + + while (true) { + let page: Row[] + + if (usesRowId) { + try { + page = await fetchRowIdPage( + table, + rowId, + pageSize, + dataSource, + config + ) + } catch { + usesRowId = false + offset = 0 + continue + } + } else { + page = await fetchOffsetPage( + table, + offset, + pageSize, + dataSource, + config ) - dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` } - dumpContent += '\n' + if (page.length === 0) break + + let output = '' + + for (const sourceRow of page) { + const row = { ...sourceRow } + const currentRowId = row.__starbase_rowid__ + delete row.__starbase_rowid__ + + const columns = Object.keys(row).map(quoteIdentifier).join(', ') + const values = Object.values(row).map(sqlLiteral).join(', ') + + output += `INSERT INTO ${quoteIdentifier(table)} (${columns}) VALUES (${values});\n` + + if (usesRowId) rowId = Number(currentRowId) + } + + yield output + + if (usesRowId) { + if (page.length < pageSize) break + } else { + offset += page.length + if (page.length < pageSize) break + } + + await breathe() } + } - // Create a Blob from the dump content - const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' }) + yield 'COMMIT;\n' +} - const headers = new Headers({ - 'Content-Type': 'application/x-sqlite3', - 'Content-Disposition': 'attachment; filename="database_dump.sql"', +export async function dumpDatabaseRoute( + dataSource: DataSource, + config: StarbaseDBConfiguration, + pageSize = DEFAULT_PAGE_SIZE +): Promise { + if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 10_000) { + return createResponse(undefined, 'Invalid export page size', 400) + } + + try { + // Resolve table metadata before committing an HTTP 200 stream. + const tables = await getTables(dataSource, config) + const iterator = dumpChunks(tables, dataSource, config, pageSize) + + const body = new ReadableStream({ + async pull(controller) { + try { + const next = await iterator.next() + + if (next.done) { + controller.close() + return + } + + controller.enqueue(encoder.encode(next.value)) + } catch (error) { + controller.error(error) + } + }, + async cancel() { + await iterator.return?.() + }, }) - return new Response(blob, { headers }) - } catch (error: any) { + return new Response(body, { + headers: { + 'Content-Type': 'application/sql; charset=utf-8', + 'Content-Disposition': + 'attachment; filename="database_dump.sql"', + 'Cache-Control': 'no-store', + }, + }) + } catch (error) { console.error('Database Dump Error:', error) return createResponse(undefined, 'Failed to create database dump', 500) }