Skip to content

Commit d4b7c86

Browse files
committed
perf(files): seek folder indexes and order file timestamps on the raw column
Folder-scoped file pages spelled their folder predicate as `folder_id IS NULL`, `= $1`, or `IN (...)`, none of which can seek the `(workspace_id, coalesce(folder_id, ''), original_name)` index, so a name-sorted root or folder page read every live file in the workspace and sorted them. The predicate is now spelled on `coalesce(folder_id, '')`, identical in meaning because no folder id is empty (an empty id is now refused), and the name key bounds its scan so later pages seek too. The uploadedAt and updatedAt keysets ordered on `date_trunc('milliseconds', ...)`, which no index on the column can serve. They now order on the raw column with a microsecond cursor; a cursor minted before this keeps the truncated keyset it was minted under, so a walk in progress neither skips nor repeats rows.
1 parent 19b8d53 commit d4b7c86

2 files changed

Lines changed: 121 additions & 35 deletions

File tree

‎apps/sim/lib/uploads/contexts/workspace/file-manager.ts‎

Lines changed: 79 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import type { ShareRecord } from '@/lib/api/contracts/public-shares'
3030
import type { V2FileSortBy } from '@/lib/api/contracts/v2/files'
3131
import {
3232
type CursorKey,
33+
classifyTimestampCursor,
3334
encodeKeyset,
3435
INVALID_CURSOR_MESSAGE,
3536
type KeysetKey,
@@ -38,6 +39,8 @@ import {
3839
type ListSortOrder,
3940
listOrderBy,
4041
numberKey,
42+
preciseTimestampKey,
43+
preciseTimestampText,
4144
searchFilter,
4245
textKey,
4346
timestampKey,
@@ -150,6 +153,7 @@ import {
150153
} from '@/lib/workspace-files/page-source-embed'
151154
import {
152155
contentWritableWorkspaceFileContextCondition,
156+
workspaceFileFolderIdCondition,
153157
workspaceFileFolderScopeCondition,
154158
} from '@/lib/workspace-files/query-scope'
155159
import { getWorkspaceWithOwner } from '@/lib/workspaces/permissions/utils'
@@ -1819,25 +1823,78 @@ export async function listWorkspaceFiles(
18191823
}
18201824

18211825
/**
1822-
* The keysets behind {@link queryWorkspaceFiles}' sortable fields. `satisfies`
1823-
* makes this total over the contract enum: a new sortable field in the contract
1824-
* fails to compile until it has a keyset here, rather than silently falling
1825-
* through to an unordered scan.
1826-
*
1827-
* Every key column is `NOT NULL`, and `id` closes each keyset so a page
1828-
* boundary inside a run of equal names/sizes/timestamps is still stable.
1826+
* A page row: the listing columns plus each timestamp key's full-precision cursor text (see
1827+
* {@link preciseTimestampText}), so the keyset orders and compares on the raw column.
18291828
*/
1830-
const fileId = textKey<OwnedFileRecord>(workspaceFiles.id, (row) => row.id)
1829+
const workspaceFilePageColumns = {
1830+
...workspaceFileListColumns,
1831+
uploadedAtCursor: preciseTimestampText(workspaceFiles.uploadedAt),
1832+
updatedAtCursor: preciseTimestampText(workspaceFiles.updatedAt),
1833+
} as const
18311834

1832-
const WORKSPACE_FILE_SORTS = {
1833-
name: [textKey(workspaceFiles.originalName, (row) => row.name), fileId],
1834-
size: [
1835-
numberKey(sql<number>`${workspaceFiles.sizeBytes}`.mapWith(Number), (row) => row.size),
1836-
fileId,
1837-
],
1838-
uploadedAt: [timestampKey(workspaceFiles.uploadedAt, (row) => row.uploadedAt), fileId],
1839-
updatedAt: [timestampKey(workspaceFiles.updatedAt, (row) => row.updatedAt), fileId],
1840-
} satisfies Record<V2FileSortBy, readonly KeysetKey<OwnedFileRecord>[]>
1835+
type WorkspaceFilePageRow = WorkspaceFileListRow & {
1836+
uploadedAtCursor: string
1837+
updatedAtCursor: string
1838+
}
1839+
1840+
/**
1841+
* A timestamp sort key on the raw column, so an index on `(…, <column>, id)` serves both the
1842+
* order and the cursor seek. A cursor minted before that carries a millisecond date (see
1843+
* {@link classifyTimestampCursor}) and was read under millisecond-truncated ordering; resuming
1844+
* it under the raw ordering would skip or repeat rows sharing that millisecond, so it keeps the
1845+
* truncated key it was minted under, as does every cursor its walk mints.
1846+
*/
1847+
function fileTimestampKey(
1848+
column: typeof workspaceFiles.uploadedAt | typeof workspaceFiles.updatedAt,
1849+
read: (row: WorkspaceFilePageRow) => Date,
1850+
readCursor: (row: WorkspaceFilePageRow) => string,
1851+
after: CursorKey[] | undefined
1852+
): KeysetKey<WorkspaceFilePageRow> {
1853+
return after !== undefined && classifyTimestampCursor(after[0]) === 'date'
1854+
? timestampKey(column, read)
1855+
: preciseTimestampKey(column, readCursor)
1856+
}
1857+
1858+
/**
1859+
* The keyset behind {@link queryFileRecords}' sortable fields. The `Record` makes it total over
1860+
* the contract enum: a new sortable field fails to compile until it has a keyset here.
1861+
*
1862+
* Every key column is `NOT NULL`, and `id` closes each keyset so a page boundary inside a run of
1863+
* equal names/sizes/timestamps is still stable. The name key bounds its scan so a later page
1864+
* seeks the `(owner, folder, original_name)` index instead of filtering down from its top.
1865+
*/
1866+
function workspaceFileSortKeys(
1867+
sortBy: V2FileSortBy,
1868+
after: CursorKey[] | undefined
1869+
): KeysetKey<WorkspaceFilePageRow>[] {
1870+
const fileId = textKey<WorkspaceFilePageRow>(workspaceFiles.id, (row) => row.id)
1871+
const sorts: Record<V2FileSortBy, () => KeysetKey<WorkspaceFilePageRow>> = {
1872+
name: () => ({
1873+
...textKey<WorkspaceFilePageRow>(workspaceFiles.originalName, (row) => row.originalName),
1874+
boundsScan: true,
1875+
}),
1876+
size: () =>
1877+
numberKey<WorkspaceFilePageRow>(
1878+
sql<number>`${workspaceFiles.sizeBytes}`.mapWith(Number),
1879+
(row) => getWorkspaceFileSize(row)
1880+
),
1881+
uploadedAt: () =>
1882+
fileTimestampKey(
1883+
workspaceFiles.uploadedAt,
1884+
(row) => row.uploadedAt,
1885+
(row) => row.uploadedAtCursor,
1886+
after
1887+
),
1888+
updatedAt: () =>
1889+
fileTimestampKey(
1890+
workspaceFiles.updatedAt,
1891+
(row) => row.updatedAt,
1892+
(row) => row.updatedAtCursor,
1893+
after
1894+
),
1895+
}
1896+
return [sorts[sortBy](), fileId]
1897+
}
18411898

18421899
export interface QueryWorkspaceFilesOptions {
18431900
/** Exact file ids. */
@@ -1876,10 +1933,7 @@ export interface QueryWorkspaceFilesResult {
18761933
function workspaceFileFolderCondition(
18771934
folderId: string | null | readonly string[] | undefined
18781935
): SQL | undefined {
1879-
if (folderId === undefined) return undefined
1880-
if (folderId === null) return isNull(workspaceFiles.folderId)
1881-
if (Array.isArray(folderId)) return inArray(workspaceFiles.folderId, folderId)
1882-
return eq(workspaceFiles.folderId, folderId as string)
1936+
return folderId === undefined ? undefined : workspaceFileFolderIdCondition(folderId)
18831937
}
18841938

18851939
/**
@@ -1931,7 +1985,7 @@ export async function queryFileRecords<const O extends EditableFileOwner>(
19311985
if (folderId !== undefined && folderScope !== undefined) {
19321986
throw new OrchestrationError('validation', 'Specify either folderId or folderScope, not both')
19331987
}
1934-
const keys: readonly KeysetKey<OwnedFileRecord>[] = WORKSPACE_FILE_SORTS[sortBy]
1988+
const keys = workspaceFileSortKeys(sortBy, after)
19351989

19361990
let resumeAfter: SQL | undefined
19371991
if (after) {
@@ -1955,8 +2009,8 @@ export async function queryFileRecords<const O extends EditableFileOwner>(
19552009
resumeAfter,
19562010
]
19572011

1958-
const rows = await client
1959-
.select(workspaceFileListColumns)
2012+
const rows: WorkspaceFilePageRow[] = await client
2013+
.select(workspaceFilePageColumns)
19602014
.from(workspaceFiles)
19612015
.where(and(...conditions))
19622016
.orderBy(...listOrderBy(keysetColumns(keys), sortOrder))
@@ -1969,7 +2023,7 @@ export async function queryFileRecords<const O extends EditableFileOwner>(
19692023
: []
19702024
const paths = buildWorkspaceFileFolderPathMap(folders)
19712025
const files = page.map((file) => mapFileRecord(file, owner, paths))
1972-
const last = files.at(-1)
2026+
const last = page.at(-1)
19732027

19742028
return { files, nextKeys: hasMore && last ? encodeKeyset(keys, last) : null }
19752029
}

‎apps/sim/lib/workspace-files/query-scope.ts‎

Lines changed: 42 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { workspaceFiles } from '@sim/db/schema'
2-
import { eq, inArray, isNull, or, type SQL, sql } from 'drizzle-orm'
2+
import { eq, inArray, isNull, type SQL, sql } from 'drizzle-orm'
3+
import { OrchestrationError } from '@/lib/core/orchestration/types'
34
import type { FolderIdScope } from '@/lib/folders/scope'
45
import { OWNED_FILE_CONTEXTS } from '@/lib/workspace-files/owned-files'
56

@@ -18,20 +19,51 @@ export const contentWritableWorkspaceFileContextCondition = inArray(workspaceFil
1819
...OWNED_FILE_CONTEXTS,
1920
])
2021

22+
/**
23+
* The folder key the folder-scoped `workspace_files` indexes are built on: the folder id, or `''`
24+
* for the root. `workspace_files_workspace_folder_name_active_unique` (and its Project twin) lead
25+
* with `coalesce(folder_id, '')`, so a folder predicate spelled as `folder_id IS NULL` or
26+
* `folder_id = $1` cannot seek them and reads every live file of the owner instead.
27+
*/
28+
export const workspaceFileFolderKey = sql<string>`coalesce(${workspaceFiles.folderId}, '')`
29+
30+
/** The {@link workspaceFileFolderKey} value for a folder id, `null` being the root. */
31+
function folderKeyValue(folderId: string | null): string {
32+
if (folderId === '') {
33+
throw new OrchestrationError('validation', 'Folder id must not be empty')
34+
}
35+
return folderId ?? ''
36+
}
37+
38+
/**
39+
* The `workspaceFiles` predicate for one folder (`null` for the root), or several folder ids,
40+
* on {@link workspaceFileFolderKey}. Identical in meaning to `folder_id IS NULL` / `= $1` /
41+
* `IN (...)` because no folder id is `''` (ids are generated, and an empty id is refused here).
42+
* An empty list matches nothing.
43+
*/
44+
export function workspaceFileFolderIdCondition(
45+
folderId: string | null | readonly (string | null)[]
46+
): SQL {
47+
if (typeof folderId === 'string' || folderId === null) {
48+
return sql`${workspaceFileFolderKey} = ${folderKeyValue(folderId)}`
49+
}
50+
const keys = [...new Set(folderId.map(folderKeyValue))]
51+
if (keys.length === 0) return sql`false`
52+
return inArray(workspaceFileFolderKey, keys)
53+
}
54+
2155
/**
2256
* The `workspaceFiles` predicate for a resolved folder scope, or none when the
23-
* scope is absent. A root scope selects the files that carry no folder id, so
24-
* the two halves are OR-ed. An empty scope selects nothing: every path resolved
25-
* to a folder holding no subtree, and answering it unrestricted would widen it
26-
* to the whole workspace.
57+
* scope is absent. A root scope selects the files that carry no folder id. An
58+
* empty scope selects nothing: every path resolved to a folder holding no
59+
* subtree, and answering it unrestricted would widen it to the whole workspace.
2760
*/
2861
export function workspaceFileFolderScopeCondition(
2962
scope: FolderIdScope | undefined
3063
): SQL | undefined {
3164
if (!scope) return undefined
32-
const ids = [...scope.folderIds]
33-
const inScope = ids.length > 0 ? inArray(workspaceFiles.folderId, ids) : undefined
34-
const atRoot = scope.includeRootItems ? isNull(workspaceFiles.folderId) : undefined
35-
if (inScope && atRoot) return or(inScope, atRoot)
36-
return inScope ?? atRoot ?? sql`false`
65+
return workspaceFileFolderIdCondition([
66+
...scope.folderIds,
67+
...(scope.includeRootItems ? [null] : []),
68+
])
3769
}

0 commit comments

Comments
 (0)