Skip to content

fix: #59 - Database dumps do not work on large databases #311

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
isearchxsolve wants to merge 1 commit into from
+438 −163
Closed

fix: #59 - Database dumps do not work on large databases #311

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
308 changes: 196 additions & 112 deletions src/export/dump.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,145 +1,229 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { dumpDatabaseRoute } from './dump'
import { executeOperation } from '.'
import { createResponse } from '../utils'
import type { DataSource } from '../types'
import type { StarbaseDBConfiguration } from '../handler'
import { describe, expect, it, vi } from 'vitest'

const executeOperation = vi.fn()

vi.mock('.', () => ({
executeOperation: vi.fn(),
executeOperation,
}))

vi.mock('../utils', () => ({
createResponse: vi.fn(
(data, message, status) =>
new Response(JSON.stringify({ result: data, error: message }), {
status,
headers: { 'Content-Type': 'application/json' },
})
),
createResponse: (_data: unknown, message: string, status: number) =>
new Response(message, { status }),
}))

let mockDataSource: DataSource
let mockConfig: StarbaseDBConfiguration

beforeEach(() => {
vi.clearAllMocks()

mockDataSource = {
source: 'external',
external: { dialect: 'sqlite' },
rpc: { executeQuery: vi.fn() },
} as any

mockConfig = {
outerbaseApiKey: 'mock-api-key',
role: 'admin',
features: { allowlist: true, rls: true, rest: true },
}
})
import { dumpDatabaseRoute } from './dump'

describe('Database Dump Module', () => {
it('should return a database dump when tables exist', async () => {
vi.mocked(executeOperation)
.mockResolvedValueOnce([{ name: 'users' }, { name: 'orders' }])
.mockResolvedValueOnce([
{ sql: 'CREATE TABLE users (id INTEGER, name TEXT);' },
])
.mockResolvedValueOnce([
{ id: 1, name: 'Alice' },
{ id: 2, name: 'Bob' },
])
.mockResolvedValueOnce([
{ sql: 'CREATE TABLE orders (id INTEGER, total REAL);' },
])
.mockResolvedValueOnce([
{ id: 1, total: 99.99 },
{ id: 2, total: 49.5 },
])

const response = await dumpDatabaseRoute(mockDataSource, mockConfig)

expect(response).toBeInstanceOf(Response)
expect(response.headers.get('Content-Type')).toBe(
'application/x-sqlite3'
)
expect(response.headers.get('Content-Disposition')).toBe(
'attachment; filename="database_dump.sql"'
async function responseText(response: Response): Promise<string> {
return await response.text()
}

describe('dumpDatabaseRoute', () => {
it('streams a valid SQL dump without constructing a Blob', async () => {
executeOperation.mockImplementation(async (queries: any[]) => {
const { sql, params } = queries[0]

if (sql.includes("FROM sqlite_master") && sql.includes("type = 'table'")) {
return [{ name: 'users' }]
}

if (sql.includes('WHERE type = \'table\' AND name = ?')) {
expect(params).toEqual(['users'])
return [{ sql: 'CREATE TABLE users (id INTEGER, name TEXT)' }]
}

if (sql.includes('SELECT _rowid_')) {
const after = params[0]

if (after === 0) {
return [
{
__starbase_rowid__: 1,
id: 1,
name: "O'Reilly",
},
]
}

return []
}

throw new Error(`Unexpected SQL: ${sql}`)
})

const response = await dumpDatabaseRoute(
{} as any,
{ role: 'admin' },
1
)

const dumpText = await response.text()
expect(dumpText).toContain(
'CREATE TABLE users (id INTEGER, name TEXT);'
expect(response.status).toBe(200)
expect(response.headers.get('Content-Type')).toContain('application/sql')
expect(response.headers.get('Content-Disposition')).toContain(
'database_dump.sql'
)
expect(dumpText).toContain("INSERT INTO users VALUES (1, 'Alice');")
expect(dumpText).toContain("INSERT INTO users VALUES (2, 'Bob');")
expect(dumpText).toContain(
'CREATE TABLE orders (id INTEGER, total REAL);'

expect(await responseText(response)).toBe(
[
'BEGIN TRANSACTION;',
'',
'-- Table: "users"',
'CREATE TABLE users (id INTEGER, name TEXT);',
`INSERT INTO "users" ("id", "name") VALUES (1, 'O''Reilly');`,
'COMMIT;',
'',
].join('\n')
)
expect(dumpText).toContain('INSERT INTO orders VALUES (1, 99.99);')
expect(dumpText).toContain('INSERT INTO orders VALUES (2, 49.5);')
})

it('should handle empty databases (no tables)', async () => {
vi.mocked(executeOperation).mockResolvedValueOnce([])
it('uses keyset pagination and never emits OFFSET for rowid tables', async () => {
const observedSql: string[] = []

const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
executeOperation.mockImplementation(async (queries: any[]) => {
const { sql, params } = queries[0]
observedSql.push(sql)

expect(response).toBeInstanceOf(Response)
expect(response.headers.get('Content-Type')).toBe(
'application/x-sqlite3'
)
const dumpText = await response.text()
expect(dumpText).toBe('SQLite format 3\0')
})
if (sql.includes("FROM sqlite_master") && sql.includes("type = 'table'")) {
return [{ name: 'events' }]
}

if (sql.includes('WHERE type = \'table\' AND name = ?')) {
return [{ sql: 'CREATE TABLE events (id INTEGER)' }]
}

if (sql.includes('SELECT _rowid_')) {
const after = params[0]

if (after === 0) {
return [
{ __starbase_rowid__: 10, id: 10 },
{ __starbase_rowid__: 20, id: 20 },
]
}

it('should handle databases with tables but no data', async () => {
vi.mocked(executeOperation)
.mockResolvedValueOnce([{ name: 'users' }])
.mockResolvedValueOnce([
{ sql: 'CREATE TABLE users (id INTEGER, name TEXT);' },
])
.mockResolvedValueOnce([])
if (after === 20) {
return [{ __starbase_rowid__: 30, id: 30 }]
}

const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
return []
}

expect(response).toBeInstanceOf(Response)
const dumpText = await response.text()
expect(dumpText).toContain(
'CREATE TABLE users (id INTEGER, name TEXT);'
throw new Error(`Unexpected SQL: ${sql}`)
})

const response = await dumpDatabaseRoute(
{} as any,
{ role: 'admin' },
2
)
expect(dumpText).not.toContain('INSERT INTO users VALUES')

const output = await responseText(response)

expect(output.match(/INSERT INTO "events"/g)).toHaveLength(3)
expect(observedSql.some((sql) => sql.includes('_rowid_ > ?'))).toBe(true)
expect(observedSql.some((sql) => sql.includes('OFFSET'))).toBe(false)
})

it('should escape single quotes properly in string values', async () => {
vi.mocked(executeOperation)
.mockResolvedValueOnce([{ name: 'users' }])
.mockResolvedValueOnce([
{ sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' },
])
.mockResolvedValueOnce([{ id: 1, bio: "Alice's adventure" }])
it('falls back to bounded OFFSET pagination for WITHOUT ROWID tables', async () => {
executeOperation.mockImplementation(async (queries: any[]) => {
const { sql, params } = queries[0]

if (sql.includes("FROM sqlite_master") && sql.includes("type = 'table'")) {
return [{ name: 'lookup' }]
}

if (sql.includes('WHERE type = \'table\' AND name = ?')) {
return [
{
sql: 'CREATE TABLE lookup (key TEXT PRIMARY KEY) WITHOUT ROWID',
},
]
}

if (sql.includes('SELECT _rowid_')) {
throw new Error('no such column: _rowid_')
}

if (sql.includes('LIMIT ? OFFSET ?')) {
return params[1] === 0 ? [{ key: 'first' }] : []
}

throw new Error(`Unexpected SQL: ${sql}`)
})

const response = await dumpDatabaseRoute(
{} as any,
{ role: 'admin' },
1
)

const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
const output = await responseText(response)

expect(response).toBeInstanceOf(Response)
const dumpText = await response.text()
expect(dumpText).toContain(
"INSERT INTO users VALUES (1, 'Alice''s adventure');"
expect(output).toContain(
`INSERT INTO "lookup" ("key") VALUES ('first');`
)
})

it('should return a 500 response when an error occurs', async () => {
const consoleErrorMock = vi
.spyOn(console, 'error')
.mockImplementation(() => {})
vi.mocked(executeOperation).mockRejectedValue(
new Error('Database Error')
it('serializes nulls, booleans, blobs, numbers, and quoted strings correctly', async () => {
executeOperation.mockImplementation(async (queries: any[]) => {
const { sql, params } = queries[0]

if (sql.includes("FROM sqlite_master") && sql.includes("type = 'table'")) {
return [{ name: 'values' }]
}

if (sql.includes('WHERE type = \'table\' AND name = ?')) {
return [{ sql: 'CREATE TABLE values (a, b, c, d, e)' }]
}

if (sql.includes('SELECT _rowid_')) {
return params[0] === 0
? [
{
__starbase_rowid__: 1,
a: null,
b: true,
c: 42.5,
d: new Uint8Array([0, 15, 255]),
e: "it's safe",
},
]
: []
}

throw new Error(`Unexpected SQL: ${sql}`)
})

const response = await dumpDatabaseRoute(
{} as any,
{ role: 'admin' },
10
)

const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
expect(await responseText(response)).toContain(
`VALUES (NULL, 1, 42.5, X'000fff', 'it''s safe');`
)
})

it('returns a normal 500 response when metadata resolution fails', async () => {
executeOperation.mockRejectedValueOnce(new Error('storage unavailable'))

const response = await dumpDatabaseRoute(
{} as any,
{ role: 'admin' }
)

expect(response.status).toBe(500)
const jsonResponse: { error: string } = await response.json()
expect(jsonResponse.error).toBe('Failed to create database dump')
expect(await response.text()).toBe('Failed to create database dump')
})

it('rejects invalid page sizes before querying the Durable Object', async () => {
const response = await dumpDatabaseRoute(
{} as any,
{ role: 'admin' },
0
)

expect(response.status).toBe(400)
expect(executeOperation).not.toHaveBeenCalled()
})
})
Loading