diff --git a/src/export/csv.test.ts b/src/export/csv.test.ts index b186aeb..50d6213 100644 --- a/src/export/csv.test.ts +++ b/src/export/csv.test.ts @@ -1,14 +1,17 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { exportTableToCsvRoute } from './csv' -import { getTableData, createExportResponse } from './index' -import { createResponse } from '../utils' +import { executeOperation } from './index' +import type * as IndexModule from './index' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' -vi.mock('./index', () => ({ - getTableData: vi.fn(), - createExportResponse: vi.fn(), -})) +vi.mock('./index', async () => { + const actual = await vi.importActual('./index') + return { + ...actual, + executeOperation: vi.fn(), + } +}) vi.mock('../utils', () => ({ createResponse: vi.fn( @@ -43,16 +46,12 @@ beforeEach(() => { describe('CSV Export Module', () => { it('should return a CSV file when table data exists', async () => { - vi.mocked(getTableData).mockResolvedValue([ - { id: 1, name: 'Alice', age: 30 }, - { id: 2, name: 'Bob', age: 25 }, - ]) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) - ) + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'users' }]) + .mockResolvedValueOnce([ + { id: 1, name: 'Alice', age: 30 }, + { id: 2, name: 'Bob', age: 25 }, + ]) const response = await exportTableToCsvRoute( 'users', @@ -60,21 +59,17 @@ describe('CSV Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( - 'users', - mockDataSource, - mockConfig - ) - expect(createExportResponse).toHaveBeenCalledWith( - 'id,name,age\n1,Alice,30\n2,Bob,25\n', - 'users_export.csv', - 'text/csv' - ) expect(response.headers.get('Content-Type')).toBe('text/csv') + expect(response.headers.get('Content-Disposition')).toBe( + 'attachment; filename="users_export.csv"' + ) + + const csvText = await response.text() + expect(csvText).toBe('id,name,age\n1,Alice,30\n2,Bob,25\n') }) it('should return 404 if table does not exist', async () => { - vi.mocked(getTableData).mockResolvedValue(null) + vi.mocked(executeOperation).mockResolvedValueOnce([]) const response = await exportTableToCsvRoute( 'non_existent_table', @@ -82,27 +77,17 @@ describe('CSV Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( - 'non_existent_table', - mockDataSource, - mockConfig - ) expect(response.status).toBe(404) - const jsonResponse: { error: string } = await response.json() expect(jsonResponse.error).toBe( "Table 'non_existent_table' does not exist." ) }) - it('should handle empty table (return only headers)', async () => { - vi.mocked(getTableData).mockResolvedValue([]) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) - ) + it('should handle empty table (return empty content)', async () => { + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'empty_table' }]) + .mockResolvedValueOnce([]) const response = await exportTableToCsvRoute( 'empty_table', @@ -110,49 +95,112 @@ describe('CSV Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( - 'empty_table', + expect(response.status).toBe(200) + expect(response.headers.get('Content-Type')).toBe('text/csv') + expect(response.headers.get('Content-Disposition')).toBe( + 'attachment; filename="empty_table_export.csv"' + ) + + const csvText = await response.text() + expect(csvText).toBe('') + }) + + it('should escape commas and quotes in CSV values', async () => { + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'special_chars' }]) + .mockResolvedValueOnce([ + { id: 1, name: 'Sahithi, is', bio: 'my forever "penguin"' }, + ]) + + const response = await exportTableToCsvRoute( + 'special_chars', mockDataSource, mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - '', - 'empty_table_export.csv', - 'text/csv' - ) + expect(response.headers.get('Content-Type')).toBe('text/csv') + const csvText = await response.text() + expect(csvText).toBe( + 'id,name,bio\n1,"Sahithi, is","my forever ""penguin"""\n' + ) }) - it('should escape commas and quotes in CSV values', async () => { - vi.mocked(getTableData).mockResolvedValue([ - { id: 1, name: 'Sahithi, is', bio: 'my forever "penguin"' }, - ]) + it('should paginate large tables across multiple queries', async () => { + const firstPage = Array.from({ length: 1000 }, (_, i) => ({ + id: i + 1, + name: `User${i + 1}`, + })) + const secondPage = [{ id: 1001, name: 'User1001' }] - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-csv-content', { - headers: { 'Content-Type': 'text/csv' }, - }) - ) + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'users' }]) + .mockResolvedValueOnce(firstPage) + .mockResolvedValueOnce(secondPage) const response = await exportTableToCsvRoute( - 'special_chars', + 'users', mockDataSource, mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - 'id,name,bio\n1,"Sahithi, is","my forever ""penguin"""\n', - 'special_chars_export.csv', - 'text/csv' + const csvText = await response.text() + expect(csvText).toContain('id,name\n') + expect(csvText).toContain('1,User1\n') + expect(csvText).toContain('1000,User1000\n') + expect(csvText).toContain('1001,User1001\n') + + const issuedSql = vi + .mocked(executeOperation) + .mock.calls.map((c) => (c[0] as Array<{ sql: string }>)[0].sql) + expect(issuedSql).toContain('SELECT * FROM users LIMIT 1000 OFFSET 0;') + expect(issuedSql).toContain( + 'SELECT * FROM users LIMIT 1000 OFFSET 1000;' ) - expect(response.headers.get('Content-Type')).toBe('text/csv') + }) + + it('should stream chunks incrementally without buffering entire dataset', async () => { + const page1 = Array.from({ length: 1000 }, (_, i) => ({ + id: i + 1, + name: `User${i + 1}`, + })) + const page2 = [{ id: 1001, name: 'User1001' }] + + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'users' }]) + .mockResolvedValueOnce(page1) + .mockResolvedValueOnce(page2) + + const response = await exportTableToCsvRoute( + 'users', + mockDataSource, + mockConfig + ) + + expect(response.body).toBeInstanceOf(ReadableStream) + const reader = response.body!.getReader() + const decoder = new TextDecoder() + let receivedChunks = 0 + let combined = '' + + while (true) { + const { done, value } = await reader.read() + if (done) break + receivedChunks++ + combined += decoder.decode(value, { stream: true }) + } + + expect(receivedChunks).toBeGreaterThan(0) + expect(combined).toContain('id,name\n1,User1\n') + expect(combined).toContain('1001,User1001\n') }) it('should return 500 on an unexpected error', async () => { const consoleErrorMock = vi .spyOn(console, 'error') .mockImplementation(() => {}) - vi.mocked(getTableData).mockRejectedValue(new Error('Database Error')) + vi.mocked(executeOperation).mockRejectedValue( + new Error('Database Error') + ) const response = await exportTableToCsvRoute( 'users', diff --git a/src/export/csv.ts b/src/export/csv.ts index 22a4591..7bdd842 100644 --- a/src/export/csv.ts +++ b/src/export/csv.ts @@ -1,4 +1,8 @@ -import { getTableData, createExportResponse } from './index' +import { + executeOperation, + createExportResponse, + EXPORT_PAGE_SIZE, +} from './index' import { createResponse } from '../utils' import { DataSource } from '../types' import { StarbaseDBConfiguration } from '../handler' @@ -9,9 +13,19 @@ export async function exportTableToCsvRoute( config: StarbaseDBConfiguration ): Promise { try { - const data = await getTableData(tableName, dataSource, config) + // Verify if the table exists up front + const tableExistsResult = await executeOperation( + [ + { + sql: `SELECT name FROM sqlite_master WHERE type='table' AND name=?;`, + params: [tableName], + }, + ], + dataSource, + config + ) - if (data === null) { + if (!tableExistsResult || tableExistsResult.length === 0) { return createResponse( undefined, `Table '${tableName}' does not exist.`, @@ -19,37 +33,76 @@ export async function exportTableToCsvRoute( ) } - // Convert the result to CSV - let csvContent = '' - if (data.length > 0) { - // Add headers - csvContent += Object.keys(data[0]).join(',') + '\n' - - // Add data rows - data.forEach((row: any) => { - csvContent += - Object.values(row) - .map((value) => { - if ( - typeof value === 'string' && - (value.includes(',') || - value.includes('"') || - value.includes('\n')) - ) { - return `"${value.replace(/"/g, '""')}"` - } - return value - }) - .join(',') + '\n' - }) - } + const encoder = new TextEncoder() + const stream = new ReadableStream({ + async start(controller) { + try { + let offset = 0 + let headerWritten = false + + while (true) { + const rows = await executeOperation( + [ + { + sql: `SELECT * FROM ${tableName} LIMIT ${EXPORT_PAGE_SIZE} OFFSET ${offset};`, + }, + ], + dataSource, + config + ) + + if (!rows.length) { + break + } + + let chunk = '' + if (!headerWritten) { + chunk += Object.keys(rows[0]).join(',') + '\n' + headerWritten = true + } + + for (const row of rows) { + chunk += + Object.values(row) + .map((value) => { + if ( + typeof value === 'string' && + (value.includes(',') || + value.includes('"') || + value.includes('\n') || + value.includes('\r')) + ) { + return `"${value.replace(/"/g, '""')}"` + } + return value === null || + value === undefined + ? '' + : value + }) + .join(',') + '\n' + } + + controller.enqueue(encoder.encode(chunk)) + + if (rows.length < EXPORT_PAGE_SIZE) { + break + } + offset += EXPORT_PAGE_SIZE + } + + controller.close() + } catch (error) { + controller.error(error) + } + }, + }) return createExportResponse( - csvContent, + stream, `${tableName}_export.csv`, 'text/csv' ) - } catch (error: any) { + } catch (error: unknown) { console.error('CSV Export Error:', error) return createResponse(undefined, 'Failed to export table to CSV', 500) } diff --git a/src/export/dump.test.ts b/src/export/dump.test.ts index ca65b43..aaa32a8 100644 --- a/src/export/dump.test.ts +++ b/src/export/dump.test.ts @@ -128,6 +128,92 @@ describe('Database Dump Module', () => { ) }) + it('should paginate large tables across multiple queries', async () => { + const firstPage = Array.from({ length: 1000 }, (_, i) => ({ + id: i + 1, + name: `User${i + 1}`, + })) + const secondPage = [{ id: 1001, name: 'User1001' }] + + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'users' }]) + .mockResolvedValueOnce([ + { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, + ]) + .mockResolvedValueOnce(firstPage) + .mockResolvedValueOnce(secondPage) + + const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + const dumpText = await response.text() + + expect(dumpText).toContain("INSERT INTO users VALUES (1, 'User1');") + expect(dumpText).toContain( + "INSERT INTO users VALUES (1000, 'User1000');" + ) + expect(dumpText).toContain( + "INSERT INTO users VALUES (1001, 'User1001');" + ) + + const issuedSql = vi + .mocked(executeOperation) + .mock.calls.map((c) => (c[0] as Array<{ sql: string }>)[0].sql) + expect(issuedSql).toContain('SELECT * FROM users LIMIT 1000 OFFSET 0;') + expect(issuedSql).toContain( + 'SELECT * FROM users LIMIT 1000 OFFSET 1000;' + ) + }) + + it('should serialize NULL values as the NULL keyword', async () => { + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'users' }]) + .mockResolvedValueOnce([ + { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, + ]) + .mockResolvedValueOnce([{ id: 1, name: null }]) + + const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + const dumpText = await response.text() + + expect(dumpText).toContain('INSERT INTO users VALUES (1, NULL);') + }) + + it('should stream chunks incrementally without buffering entire dataset', async () => { + const firstPage = Array.from({ length: 1000 }, (_, i) => ({ + id: i + 1, + name: `User${i + 1}`, + })) + const secondPage = [{ id: 1001, name: 'User1001' }] + + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'users' }]) + .mockResolvedValueOnce([ + { sql: 'CREATE TABLE users (id INTEGER, name TEXT);' }, + ]) + .mockResolvedValueOnce(firstPage) + .mockResolvedValueOnce(secondPage) + + const response = await dumpDatabaseRoute(mockDataSource, mockConfig) + + expect(response.body).toBeInstanceOf(ReadableStream) + const reader = response.body!.getReader() + const decoder = new TextDecoder() + let receivedChunks = 0 + let combined = '' + + while (true) { + const { done, value } = await reader.read() + if (done) break + receivedChunks++ + combined += decoder.decode(value, { stream: true }) + } + + expect(receivedChunks).toBeGreaterThan(0) + expect(combined).toContain("INSERT INTO users VALUES (1, 'User1');") + expect(combined).toContain( + "INSERT INTO users VALUES (1001, 'User1001');" + ) + }) + it('should return a 500 response when an error occurs', async () => { const consoleErrorMock = vi .spyOn(console, 'error') diff --git a/src/export/dump.ts b/src/export/dump.ts index 91a2e89..f539de8 100644 --- a/src/export/dump.ts +++ b/src/export/dump.ts @@ -3,68 +3,102 @@ import { StarbaseDBConfiguration } from '../handler' import { DataSource } from '../types' import { createResponse } from '../utils' +const DUMP_PAGE_SIZE = 1000 + +function formatValue(value: unknown): string { + if (value === null || value === undefined) return 'NULL' + if (typeof value === 'string') return `'${value.replace(/'/g, "''")}'` + return String(value) +} + export async function dumpDatabaseRoute( dataSource: DataSource, config: StarbaseDBConfiguration ): Promise { try { - // Get all table names + // Resolve the list of tables up front so any failure surfaces as a + // 500 response rather than mid-stream error. const tablesResult = await executeOperation( [{ sql: "SELECT name FROM sqlite_master WHERE type='table';" }], dataSource, config ) - const tables = tablesResult.map((row: any) => row.name) - let dumpContent = 'SQLite format 3\0' // SQLite file header + const tables = (tablesResult as Array<{ name: string }>).map( + (row) => row.name + ) + + const encoder = new TextEncoder() + const stream = new ReadableStream({ + async start(controller) { + try { + controller.enqueue(encoder.encode('SQLite format 3\0')) + + for (const table of tables) { + // Get table schema + const schemaResult = await executeOperation( + [ + { + sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`, + }, + ], + dataSource, + config + ) - // Iterate through all tables - for (const table of tables) { - // Get table schema - const schemaResult = await executeOperation( - [ - { - sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`, - }, - ], - dataSource, - config - ) + if (schemaResult.length && schemaResult[0]?.sql) { + const schema = schemaResult[0].sql + controller.enqueue( + encoder.encode( + `\n-- Table: ${table}\n${schema};\n\n` + ) + ) + } - if (schemaResult.length) { - const schema = schemaResult[0].sql - dumpContent += `\n-- Table: ${table}\n${schema};\n\n` - } + // Page through table rows to avoid loading large tables into memory + let offset = 0 + while (true) { + const dataResult = await executeOperation( + [ + { + sql: `SELECT * FROM ${table} LIMIT ${DUMP_PAGE_SIZE} OFFSET ${offset};`, + }, + ], + dataSource, + config + ) - // Get table data - const dataResult = await executeOperation( - [{ sql: `SELECT * FROM ${table};` }], - dataSource, - config - ) + if (!dataResult.length) break - for (const row of dataResult) { - const values = Object.values(row).map((value) => - typeof value === 'string' - ? `'${value.replace(/'/g, "''")}'` - : value - ) - dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` - } + let chunk = '' + for (const row of dataResult) { + const values = + Object.values(row).map(formatValue) + chunk += `INSERT INTO ${table} VALUES (${values.join(', ')});\n` + } + controller.enqueue(encoder.encode(chunk)) - dumpContent += '\n' - } + if (dataResult.length < DUMP_PAGE_SIZE) break + offset += DUMP_PAGE_SIZE + } - // Create a Blob from the dump content - const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' }) + controller.enqueue(encoder.encode('\n')) + } + + controller.close() + } catch (error) { + controller.error(error) + } + }, + }) const headers = new Headers({ 'Content-Type': 'application/x-sqlite3', 'Content-Disposition': 'attachment; filename="database_dump.sql"', }) - return new Response(blob, { headers }) - } catch (error: any) { + return new Response(stream, { headers }) + } catch (error: unknown) { console.error('Database Dump Error:', error) return createResponse(undefined, 'Failed to create database dump', 500) } diff --git a/src/export/index.test.ts b/src/export/index.test.ts index 48de76e..88fa86b 100644 --- a/src/export/index.test.ts +++ b/src/export/index.test.ts @@ -154,5 +154,28 @@ describe('Database Operations Module', () => { 'attachment; filename="notes.txt"' ) }) + + it('should create a valid response for a ReadableStream', async () => { + const stream = new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode('streamed data') + ) + controller.close() + }, + }) + const response = createExportResponse( + stream, + 'stream.csv', + 'text/csv' + ) + + expect(response.headers.get('Content-Type')).toBe('text/csv') + expect(response.headers.get('Content-Disposition')).toBe( + 'attachment; filename="stream.csv"' + ) + const text = await response.text() + expect(text).toBe('streamed data') + }) }) }) diff --git a/src/export/index.ts b/src/export/index.ts index 9c40119..5396596 100644 --- a/src/export/index.ts +++ b/src/export/index.ts @@ -2,6 +2,8 @@ import { DataSource } from '../types' import { executeTransaction } from '../operation' import { StarbaseDBConfiguration } from '../handler' +export const EXPORT_PAGE_SIZE = 1000 + export async function executeOperation( queries: { sql: string; params?: any[] }[], dataSource: DataSource, @@ -48,23 +50,29 @@ export async function getTableData( config ) return dataResult - } catch (error: any) { + } catch (error: unknown) { console.error('Table Data Fetch Error:', error) throw error } } export function createExportResponse( - data: any, + data: BodyInit | unknown, fileName: string, contentType: string ): Response { - const blob = new Blob([data], { type: contentType }) - const headers = new Headers({ 'Content-Type': contentType, 'Content-Disposition': `attachment; filename="${fileName}"`, }) + if (data instanceof ReadableStream) { + return new Response(data, { headers }) + } + + const blob = + data instanceof Blob + ? data + : new Blob([String(data ?? '')], { type: contentType }) return new Response(blob, { headers }) } diff --git a/src/export/json.test.ts b/src/export/json.test.ts index 3fe4a8c..90c7be0 100644 --- a/src/export/json.test.ts +++ b/src/export/json.test.ts @@ -1,14 +1,17 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { exportTableToJsonRoute } from './json' -import { getTableData, createExportResponse } from './index' -import { createResponse } from '../utils' +import { executeOperation } from './index' +import type * as IndexModule from './index' import type { DataSource } from '../types' import type { StarbaseDBConfiguration } from '../handler' -vi.mock('./index', () => ({ - getTableData: vi.fn(), - createExportResponse: vi.fn(), -})) +vi.mock('./index', async () => { + const actual = await vi.importActual('./index') + return { + ...actual, + executeOperation: vi.fn(), + } +}) vi.mock('../utils', () => ({ createResponse: vi.fn( @@ -33,7 +36,7 @@ beforeEach(() => { } as any mockConfig = { - outerbaseApiKey: 'mock-api-key', + outerbaseApiKey: '[REDACTED:auth_header]', role: 'admin', features: { allowlist: true, rls: true, rest: true }, } @@ -41,7 +44,7 @@ beforeEach(() => { describe('JSON Export Module', () => { it('should return a 404 response if table does not exist', async () => { - vi.mocked(getTableData).mockResolvedValue(null) + vi.mocked(executeOperation).mockResolvedValueOnce([]) const response = await exportTableToJsonRoute( 'missing_table', @@ -59,13 +62,9 @@ describe('JSON Export Module', () => { { id: 1, name: 'Alice' }, { id: 2, name: 'Bob' }, ] - vi.mocked(getTableData).mockResolvedValue(mockData) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) - ) + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'users' }]) + .mockResolvedValueOnce(mockData) const response = await exportTableToJsonRoute( 'users', @@ -73,27 +72,21 @@ describe('JSON Export Module', () => { mockConfig ) - expect(getTableData).toHaveBeenCalledWith( - 'users', - mockDataSource, - mockConfig - ) - expect(createExportResponse).toHaveBeenCalledWith( - JSON.stringify(mockData, null, 4), - 'users_export.json', - 'application/json' - ) + expect(response.status).toBe(200) expect(response.headers.get('Content-Type')).toBe('application/json') + expect(response.headers.get('Content-Disposition')).toBe( + 'attachment; filename="users_export.json"' + ) + + const jsonText = await response.text() + expect(JSON.parse(jsonText)).toEqual(mockData) + expect(jsonText).toBe(JSON.stringify(mockData, null, 4)) }) it('should return an empty JSON array when table has no data', async () => { - vi.mocked(getTableData).mockResolvedValue([]) - - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) - ) + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'empty_table' }]) + .mockResolvedValueOnce([]) const response = await exportTableToJsonRoute( 'empty_table', @@ -101,46 +94,112 @@ describe('JSON Export Module', () => { mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - '[]', - 'empty_table_export.json', - 'application/json' - ) + expect(response.status).toBe(200) expect(response.headers.get('Content-Type')).toBe('application/json') + expect(response.headers.get('Content-Disposition')).toBe( + 'attachment; filename="empty_table_export.json"' + ) + + const jsonText = await response.text() + expect(jsonText).toBe('[]') }) it('should escape special characters in JSON properly', async () => { const specialCharsData = [ - { id: 1, name: 'Sahithi "The Best"' }, - { id: 2, description: 'New\nLine' }, + { id: 1, text: 'Hello "World"\nNew line\tTab \\ Backslash' }, ] - vi.mocked(getTableData).mockResolvedValue(specialCharsData) + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'special_chars' }]) + .mockResolvedValueOnce(specialCharsData) - vi.mocked(createExportResponse).mockReturnValue( - new Response('mocked-json-content', { - headers: { 'Content-Type': 'application/json' }, - }) + const response = await exportTableToJsonRoute( + 'special_chars', + mockDataSource, + mockConfig ) + expect(response.headers.get('Content-Type')).toBe('application/json') + const jsonText = await response.text() + expect(JSON.parse(jsonText)).toEqual(specialCharsData) + expect(jsonText).toBe(JSON.stringify(specialCharsData, null, 4)) + }) + + it('should paginate large tables across multiple queries', async () => { + const firstPage = Array.from({ length: 1000 }, (_, i) => ({ + id: i + 1, + name: `User${i + 1}`, + })) + const secondPage = [{ id: 1001, name: 'User1001' }] + + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'users' }]) + .mockResolvedValueOnce(firstPage) + .mockResolvedValueOnce(secondPage) + const response = await exportTableToJsonRoute( - 'special_chars', + 'users', mockDataSource, mockConfig ) - expect(createExportResponse).toHaveBeenCalledWith( - JSON.stringify(specialCharsData, null, 4), - 'special_chars_export.json', - 'application/json' + const jsonText = await response.text() + const parsed = JSON.parse(jsonText) + expect(parsed).toHaveLength(1001) + expect(parsed[0]).toEqual({ id: 1, name: 'User1' }) + expect(parsed[1000]).toEqual({ id: 1001, name: 'User1001' }) + + const issuedSql = vi + .mocked(executeOperation) + .mock.calls.map((c) => (c[0] as Array<{ sql: string }>)[0].sql) + expect(issuedSql).toContain('SELECT * FROM users LIMIT 1000 OFFSET 0;') + expect(issuedSql).toContain( + 'SELECT * FROM users LIMIT 1000 OFFSET 1000;' ) - expect(response.headers.get('Content-Type')).toBe('application/json') + }) + + it('should stream chunks incrementally without buffering entire dataset', async () => { + const page1 = Array.from({ length: 1000 }, (_, i) => ({ + id: i + 1, + name: `User${i + 1}`, + })) + const page2 = [{ id: 1001, name: 'User1001' }] + + vi.mocked(executeOperation) + .mockResolvedValueOnce([{ name: 'users' }]) + .mockResolvedValueOnce(page1) + .mockResolvedValueOnce(page2) + + const response = await exportTableToJsonRoute( + 'users', + mockDataSource, + mockConfig + ) + + expect(response.body).toBeInstanceOf(ReadableStream) + const reader = response.body!.getReader() + const decoder = new TextDecoder() + let receivedChunks = 0 + let combined = '' + + while (true) { + const { done, value } = await reader.read() + if (done) break + receivedChunks++ + combined += decoder.decode(value, { stream: true }) + } + + expect(receivedChunks).toBeGreaterThan(0) + const parsed = JSON.parse(combined) + expect(parsed).toHaveLength(1001) }) it('should return a 500 response when an error occurs', async () => { const consoleErrorMock = vi .spyOn(console, 'error') .mockImplementation(() => {}) - vi.mocked(getTableData).mockRejectedValue(new Error('Database Error')) + vi.mocked(executeOperation).mockRejectedValue( + new Error('Database Error') + ) const response = await exportTableToJsonRoute( 'users', diff --git a/src/export/json.ts b/src/export/json.ts index c0ab811..1ad5699 100644 --- a/src/export/json.ts +++ b/src/export/json.ts @@ -1,4 +1,8 @@ -import { getTableData, createExportResponse } from './index' +import { + executeOperation, + createExportResponse, + EXPORT_PAGE_SIZE, +} from './index' import { createResponse } from '../utils' import { DataSource } from '../types' import { StarbaseDBConfiguration } from '../handler' @@ -9,9 +13,19 @@ export async function exportTableToJsonRoute( config: StarbaseDBConfiguration ): Promise { try { - const data = await getTableData(tableName, dataSource, config) + // Verify if the table exists up front + const tableExistsResult = await executeOperation( + [ + { + sql: `SELECT name FROM sqlite_master WHERE type='table' AND name=?;`, + params: [tableName], + }, + ], + dataSource, + config + ) - if (data === null) { + if (!tableExistsResult || tableExistsResult.length === 0) { return createResponse( undefined, `Table '${tableName}' does not exist.`, @@ -19,15 +33,74 @@ export async function exportTableToJsonRoute( ) } - // Convert the result to JSON - const jsonData = JSON.stringify(data, null, 4) + const encoder = new TextEncoder() + const stream = new ReadableStream({ + async start(controller) { + try { + let offset = 0 + let isFirstRow = true + + while (true) { + const rows = await executeOperation( + [ + { + sql: `SELECT * FROM ${tableName} LIMIT ${EXPORT_PAGE_SIZE} OFFSET ${offset};`, + }, + ], + dataSource, + config + ) + + if (!rows.length) { + if (isFirstRow) { + controller.enqueue(encoder.encode('[]')) + } + break + } + + let chunk = '' + if (isFirstRow) { + chunk += '[\n' + } + + for (const row of rows) { + const formattedRow = JSON.stringify(row, null, 4) + .split('\n') + .map((line) => ' ' + line) + .join('\n') + + if (!isFirstRow) { + chunk += ',\n' + } + chunk += formattedRow + isFirstRow = false + } + + controller.enqueue(encoder.encode(chunk)) + + if (rows.length < EXPORT_PAGE_SIZE) { + break + } + offset += EXPORT_PAGE_SIZE + } + + if (!isFirstRow) { + controller.enqueue(encoder.encode('\n]')) + } + + controller.close() + } catch (error) { + controller.error(error) + } + }, + }) return createExportResponse( - jsonData, + stream, `${tableName}_export.json`, 'application/json' ) - } catch (error: any) { + } catch (error: unknown) { console.error('JSON Export Error:', error) return createResponse(undefined, 'Failed to export table to JSON', 500) }