Skip to content

Commit 41ce612

Browse files
committed
refactor(files): simplify the paged file reads and share their helpers
- The resumable raw-column timestamp key the file sorts and the public logs keyset each spelled out is one `resumableTimestampKey` in list-query. - A page selects only its active timestamp sort's cursor text, and reuses the folder tree a folder filter already loaded to hydrate folder paths instead of reading the tree again. - A lookup reads ids, keys, and references in one query with one folder-path hydration; the route's contract is its only cap. - One `attachShares` joins share state for the full list, the bounded page, and the layout seed; the full list reads files and shares concurrently. - One capped folder-index loader serves the page and folder-stats reads, and folder-id filters resolve through `resolveFolderScope`. - Search uses `v2LimitSchema` and its own type-ahead rate-limit bucket, and notes that terms under three characters cannot use the trigram index. - The list cursor binds the root folder as `null`, like the Projects list. - The list, search, and lookup responses share one files body schema.
1 parent ab271d9 commit 41ce612

10 files changed

Lines changed: 247 additions & 229 deletions

File tree

‎apps/sim/app/api/workspaces/[id]/files/route.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import {
22
createWorkspaceFileContract,
3+
DEFAULT_WORKSPACE_FILE_PAGE_SIZE,
34
listWorkspaceFilesContract,
45
WORKSPACE_FILE_PAGE_PARAMS,
56
} from '@/lib/api/contracts/workspace-files'
@@ -24,8 +25,6 @@ import { fileOperations } from '@/lib/workspace-files/application/operations'
2425
import { MAX_WORKSPACE_FILE_INLINE_BODY_BYTES } from '@/lib/workspace-files/orchestration'
2526
import { readSortedCursor, writeSortedCursor } from '@/app/api/v2/lib/response'
2627

27-
const DEFAULT_WORKSPACE_FILE_PAGE_SIZE = 100
28-
2928
export const dynamic = 'force-dynamic'
3029

3130
/** Every param that changes which files, in which order, a page walk returns. */
@@ -35,7 +34,7 @@ function fileCursorFilters(
3534
) {
3635
return cursorScopeKey(cursorRoute(listWorkspaceFilesContract, { id: workspaceId }), {
3736
scope: query.scope,
38-
folderId: query.folderId === null ? 'root' : query.folderId,
37+
folderId: query.folderId,
3938
})
4039
}
4140

@@ -47,6 +46,8 @@ export const GET = defineInternalJsonRoute({
4746
contract: listWorkspaceFilesContract,
4847
auth: internalSessionAuth,
4948
operation: fileOperations.list,
49+
// One policy per route: paged requests share the unpaged list's exemption until the builder
50+
// can choose a policy per request.
5051
rateLimit: internalRateLimits.none({ reason: 'Preserve existing internal file-list behavior' }),
5152
errorPolicy: internalFileErrorPolicies.default,
5253
mapInput: ({ params, query }) => {

‎apps/sim/app/api/workspaces/[id]/files/search/route.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ import { fileOperations } from '@/lib/workspace-files/application/operations'
1010

1111
export const dynamic = 'force-dynamic'
1212

13+
/** Sized for debounced type-ahead: a burst while typing, about two searches a second sustained. */
14+
const SEARCH_TYPE_AHEAD_LIMIT = { maxTokens: 120, refillRate: 120, refillIntervalMs: 60_000 }
15+
1316
/**
1417
* GET /api/workspaces/[id]/files/search — live files whose name contains `q`, by name, each
1518
* carrying its `folderPath`; within `folderId` (and, by default, its subfolders) when given.
@@ -18,7 +21,10 @@ export const GET = defineInternalJsonRoute({
1821
contract: searchWorkspaceFilesContract,
1922
auth: internalSessionAuth,
2023
operation: fileOperations.list,
21-
rateLimit: internalRateLimits.user({ bucketName: 'workspace-files.read' }),
24+
rateLimit: internalRateLimits.user({
25+
bucketName: 'workspace-files.search',
26+
config: SEARCH_TYPE_AHEAD_LIMIT,
27+
}),
2228
errorPolicy: internalFileErrorPolicies.default,
2329
mapInput: ({ params, query }) => ({
2430
workspaceId: params.id,

‎apps/sim/lib/api/contracts/workspace-files.ts‎

Lines changed: 22 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import {
1010
} from '@/lib/api/contracts/primitives'
1111
import { shareRecordSchema } from '@/lib/api/contracts/public-shares'
1212
import { defineRouteContract } from '@/lib/api/contracts/types'
13-
import { V2_SEARCH_MAX_LENGTH } from '@/lib/api/contracts/v2/shared'
13+
import { V2_SEARCH_MAX_LENGTH, v2LimitSchema } from '@/lib/api/contracts/v2/shared'
1414
import { LIST_SORT_ORDERS } from '@/lib/api/list-query'
1515

1616
/**
@@ -35,6 +35,9 @@ const WORKSPACE_FILE_SORT_FIELDS = ['name', 'size', 'uploadedAt', 'updatedAt'] a
3535
/** The largest page `GET /api/workspaces/[id]/files` serves when paging. */
3636
const MAX_WORKSPACE_FILE_PAGE_SIZE = 500
3737

38+
/** The page size a paged list request that omits `limit` gets. */
39+
export const DEFAULT_WORKSPACE_FILE_PAGE_SIZE = 100
40+
3841
/** A folder filter in the browser's addressing: `root` for the workspace root, else a folder id. */
3942
const workspaceFileFolderFilterSchema = z
4043
.union([z.literal('root'), folderIdSchema])
@@ -209,8 +212,12 @@ const workspaceFileSuccessSchema = z.object({
209212
success: z.boolean(),
210213
})
211214

212-
const listWorkspaceFilesResponseSchema = workspaceFileSuccessSchema.extend({
215+
/** The body every file-list read answers with. */
216+
const workspaceFilesResponseSchema = workspaceFileSuccessSchema.extend({
213217
files: z.array(workspaceFileRecordSchema),
218+
})
219+
220+
const listWorkspaceFilesResponseSchema = workspaceFilesResponseSchema.extend({
214221
/** Present only on a paged request: the next page's cursor, `null` on the last page. */
215222
nextCursor: z.string().nullable().optional(),
216223
})
@@ -234,41 +241,33 @@ export const listWorkspaceFilesContract = defineRouteContract({
234241
},
235242
})
236243

237-
const MAX_WORKSPACE_FILE_SEARCH_RESULTS = 50
238-
239244
const searchWorkspaceFilesQuerySchema = z.object({
245+
/**
246+
* A term of one or two characters holds no trigram, so it cannot use the name index and
247+
* scans the workspace's live files; three or more probe `workspace_files_workspace_name_trgm_idx`.
248+
*/
240249
q: z.string().trim().min(1, 'q cannot be empty').max(V2_SEARCH_MAX_LENGTH, 'q is too long'),
241250
/** Confine the search to this folder (`root` for the workspace root). Omit to search everywhere. */
242251
folderId: workspaceFileFolderFilterSchema.optional(),
243252
/** Whether the folder's subfolders are searched too. Ignored without `folderId`. */
244253
recursive: booleanQueryFlagSchema.default(true),
245-
limit: z.coerce
246-
.number({ error: 'limit must be a number' })
247-
.int('limit must be a whole number')
248-
.min(1, 'limit must be at least 1')
249-
.max(
250-
MAX_WORKSPACE_FILE_SEARCH_RESULTS,
251-
`limit cannot exceed ${MAX_WORKSPACE_FILE_SEARCH_RESULTS}`
252-
)
253-
.default(20),
254+
limit: v2LimitSchema({ max: 50, fallback: 20, description: 'Maximum files to return.' }),
254255
})
255256

256-
const searchWorkspaceFilesResponseSchema = workspaceFileSuccessSchema.extend({
257-
/** Live files whose name contains `q` (case-insensitive), by name; each carries `folderPath`. */
258-
files: z.array(workspaceFileRecordSchema),
259-
})
260-
261-
/** GET — case-insensitive file-name search within a workspace, optionally within one folder. */
257+
/**
258+
* GET — case-insensitive file-name search within a workspace, optionally within one folder:
259+
* live files whose name contains `q`, by name, each carrying `folderPath`.
260+
*/
262261
export const searchWorkspaceFilesContract = defineRouteContract({
263262
method: 'GET',
264263
path: '/api/workspaces/[id]/files/search',
265264
params: workspaceFilesParamsSchema,
266265
query: searchWorkspaceFilesQuerySchema,
267-
response: { mode: 'json', schema: searchWorkspaceFilesResponseSchema },
266+
response: { mode: 'json', schema: workspaceFilesResponseSchema },
268267
})
269268

270269
/** The most entries each of a lookup's `ids`, `keys`, and `references` may carry. */
271-
export const MAX_WORKSPACE_FILE_LOOKUP_ENTRIES = 100
270+
const MAX_WORKSPACE_FILE_LOOKUP_ENTRIES = 100
272271

273272
const lookupEntriesSchema = (field: string) =>
274273
z
@@ -296,9 +295,8 @@ const lookupWorkspaceFilesBodySchema = z
296295
{ message: 'Provide at least one id, key, or reference' }
297296
)
298297

299-
const lookupWorkspaceFilesResponseSchema = workspaceFileSuccessSchema.extend({
300-
/** Every live file any id, key, or reference resolved to, once each. Misses are absent. */
301-
files: z.array(workspaceFileRecordSchema),
298+
/** `files` holds every live file any id, key, or reference resolved to, once each. */
299+
const lookupWorkspaceFilesResponseSchema = workspaceFilesResponseSchema.extend({
302300
/** Per requested reference, in request order (deduplicated): the file it resolved to, or `null`. */
303301
references: z.array(z.object({ reference: z.string(), fileId: z.string().nullable() })),
304302
})

‎apps/sim/lib/api/list-query.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,28 @@ export function preciseTimestampKey<Row>(
235235
}
236236
}
237237

238+
/**
239+
* A `timestamp` key that orders on the raw column, so an index on `(…, <column>, id)` serves
240+
* both the order and the cursor seek, while still finishing a walk begun before that.
241+
*
242+
* A cursor minted under the millisecond {@link timestampKey} carries a date rather than the
243+
* precise form (see {@link classifyTimestampCursor}) and was read under truncated ordering,
244+
* where rows sharing a millisecond fall in `id` order rather than stored order. Resuming one
245+
* under the raw ordering would skip or repeat rows inside that millisecond, so it keeps the
246+
* truncated key it was minted under, and so does every cursor that page mints, until the walk
247+
* ends. `readCursor` returns the row's {@link preciseTimestampText}.
248+
*/
249+
export function resumableTimestampKey<Row>(
250+
column: Column,
251+
read: (row: Row) => Date,
252+
readCursor: (row: Row) => string,
253+
cursorKeys: CursorKey[] | undefined
254+
): KeysetKey<Row> {
255+
return cursorKeys !== undefined && classifyTimestampCursor(cursorKeys[0]) === 'date'
256+
? timestampKey(column, read)
257+
: preciseTimestampKey(column, readCursor)
258+
}
259+
238260
export function sortDirection(order: ListSortOrder): typeof asc {
239261
return order === 'asc' ? asc : desc
240262
}

‎apps/sim/lib/logs/public-queries.ts‎

Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,10 @@ import {
2323
type ListSortOrder,
2424
listOrderBy,
2525
numberKey,
26-
preciseTimestampKey,
2726
preciseTimestampText,
27+
resumableTimestampKey,
2828
resumeKeyset,
2929
textKey,
30-
timestampKey,
3130
} from '@/lib/api/list-query'
3231
import { MATERIALIZE_CONCURRENCY } from '@/lib/core/utils/concurrency'
3332
import { MAX_DURABLE_LARGE_VALUE_BYTES } from '@/lib/execution/payloads/limits'
@@ -250,27 +249,22 @@ type PublicLogKeysetRow = Pick<
250249
* Both log tables are ordered by it, and both spell it identically, so the
251250
* merged sequence a unioned page returns resumes from one set of cursor keys.
252251
*
253-
* It orders on the raw column so the `(workspace_id, started_at)` and
254-
* `(workflow_id, started_at)` indexes serve the sort and a page reads only its
255-
* own rows. A cursor minted before that carries a date rather than the precise
256-
* form (see {@link classifyTimestampCursor}) and was read under
257-
* millisecond-truncated ordering, where rows sharing a millisecond
258-
* fall in `id` order rather than stored order. Resuming one under the raw
259-
* ordering would skip or repeat rows inside that millisecond, so it keeps the
260-
* truncated keyset it was minted under, and so does every cursor that page
261-
* mints, until the walk ends.
252+
* It orders on the raw column (see {@link resumableTimestampKey}) so the
253+
* `(workspace_id, started_at)` and `(workflow_id, started_at)` indexes serve the
254+
* sort and a page reads only its own rows.
262255
*/
263256
function startedAtKeyset<Row extends { id: string; startedAt: Date; startedAtCursor: string }>(
264257
startedAtColumn: Column,
265258
idColumn: Column,
266259
cursorKeys: CursorKey[] | undefined
267260
): KeysetKey<Row>[] {
268-
const resumesDateCursor =
269-
cursorKeys !== undefined && classifyTimestampCursor(cursorKeys[0]) === 'date'
270261
return [
271-
resumesDateCursor
272-
? timestampKey<Row>(startedAtColumn, (row) => row.startedAt)
273-
: preciseTimestampKey<Row>(startedAtColumn, (row) => row.startedAtCursor),
262+
resumableTimestampKey<Row>(
263+
startedAtColumn,
264+
(row) => row.startedAt,
265+
(row) => row.startedAtCursor,
266+
cursorKeys
267+
),
274268
textKey<Row>(idColumn, (row) => row.id),
275269
]
276270
}
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import type { ShareRecord } from '@/lib/api/contracts/public-shares'
2+
3+
/** Each item with its public share state, `null` when it was never shared. */
4+
export function attachShares<T extends { id: string }>(
5+
items: readonly T[],
6+
shares: ReadonlyMap<string, ShareRecord>
7+
): (T & { share: ShareRecord | null })[] {
8+
return items.map((item) => ({ ...item, share: shares.get(item.id) ?? null }))
9+
}

0 commit comments

Comments
 (0)