perf(db): index a workspace's live files in upload order

The live file list filtered on workspace, owner, context and liveness and
sorted by uploaded_at, and no index served that filter in that order. The
planner combined the context and search keyset indexes, which match the
live workspace files of every workspace, filtered to one workspace on the
heap, then sorted, on disk for a large workspace. A bounded read such as
the SSR seed or an uploadedAt keyset page paid for the whole workspace.

workspace_files_workspace_active_uploaded_idx keys on (workspace_id,
uploaded_at, id) under the predicate the other live workspace-file indexes
share, which the list's filter implies. The key stays narrow: carrying the
name and storage key columns would bring an entry near the btree size limit
and fail the write. The contract-pending note on
workspace_files_workspace_active_keyset_idx now names this index as the one
that takes over its workspace-scoped reads.

The list now orders upload-time ties by id, so its order is deterministic.
explainCaptured in @sim/db/testing/explain EXPLAINs the statement a call
sends, with its bound parameters, for plan assertions in integration tests.

On a prod-shaped local copy with stale statistics, the largest workspace's
301-row seed went from ~210 ms to under 1 ms and an uploadedAt keyset page
from ~205 ms to under 0.1 ms, both ordered index scans with no sort. The
unbounded full list keeps its bitmap plan: its estimate is about an eighth
of the real row count, at which the planner prices this index above it.
Lookups by key, name and id and the name-sorted folder page keep their
plans.

Author: waleedlatif1

apps/sim/lib/uploads/contexts/workspace/__integration__/file-list.integration.ts

@@ -0,0 +1,137 @@
+/**
+ * Real PostgreSQL reads of a workspace's whole file list: which rows each scope returns, their
+ * order, and the plan the live list's SQL gets.
+ */
+import { db } from '@sim/db'
+import { user, workspace, workspaceFiles } from '@sim/db/schema'
+import { explainCaptured } from '@sim/db/testing/explain'
+import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures'
+import { uploadsSetupMock } from '@sim/testing/mocks/uploads-setup.mock'
+import { generateId } from '@sim/utils/id'
+import { eq, inArray } from 'drizzle-orm'
+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/uploads/core/setup.server', () => uploadsSetupMock)
+
+import { listWorkspaceFiles } from '@/lib/uploads/contexts/workspace/file-manager'
+import { WORKSPACE_FILE_SEED_MAX } from '@/app/workspace/[workspaceId]/lib/seed-workspace-files'
+
+const ownerId = generateId()
+const workspaceId = generateId()
+const otherWorkspaceId = generateId()
+const tick = (seconds: number) => new Date(Date.UTC(2026, 0, 1, 0, 0, seconds))
+
+type FileRow = typeof workspaceFiles.$inferInsert
+
+function fileRow(name: string, values: Partial<FileRow> = {}): FileRow {
+  const id = `${name}-${workspaceId}`
+  return {
+    id,
+    key: `workspace/${workspaceId}/${id}`,
+    userId: ownerId,
+    workspaceId,
+    context: 'workspace',
+    originalName: `${name}.txt`,
+    contentType: 'text/plain',
+    sizeBytes: 1,
+    ...values,
+  }
+}
+
+describe('workspace file list in PostgreSQL', () => {
+  /**
+   * Rows inserted, and deleted, out of id order, with uploaded_at ties inside each scope and
+   * across them, so an order that is not (uploaded_at, id) shows up as a reordered list.
+   */
+  const live = [
+    fileRow('live-d', { uploadedAt: tick(1) }),
+    fileRow('live-c', { uploadedAt: tick(2) }),
+    fileRow('live-b', { uploadedAt: tick(2) }),
+    fileRow('live-a', { uploadedAt: tick(2) }),
+    fileRow('live-e', { uploadedAt: tick(4) }),
+  ]
+  const archived = [
+    fileRow('gone-b', { uploadedAt: tick(2), deletedAt: tick(10) }),
+    fileRow('gone-c', { uploadedAt: tick(3), deletedAt: tick(11) }),
+    fileRow('gone-a', { uploadedAt: tick(3), deletedAt: tick(12) }),
+  ]
+  const neverListed = [
+    fileRow('chat-upload', { context: 'mothership', uploadedAt: tick(2) }),
+    fileRow('execution-output', { context: 'execution', uploadedAt: tick(2) }),
+    fileRow('other-workspace', {
+      workspaceId: otherWorkspaceId,
+      key: `workspace/${otherWorkspaceId}/other`,
+      uploadedAt: tick(2),
+    }),
+  ]
+  const byUploadThenId = (rows: FileRow[]) =>
+    [...rows]
+      .sort(
+        (a, b) =>
+          (a.uploadedAt?.getTime() ?? 0) - (b.uploadedAt?.getTime() ?? 0) ||
+          (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)
+      )
+      .map((row) => row.id)
+
+  beforeAll(async () => {
+    await db.insert(user).values({
+      id: ownerId,
+      email: `${ownerId}@file-list.invalid`,
+      name: 'File list fixture',
+      emailVerified: true,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    })
+    await insertWorkspaceFixture(
+      db,
+      [workspaceId, otherWorkspaceId].map((id) => ({
+        id,
+        ownerId,
+        billedAccountUserId: ownerId,
+        name: 'File list fixture',
+      }))
+    )
+    await db.insert(workspaceFiles).values([...live, ...archived, ...neverListed])
+  })
+
+  afterAll(async () => {
+    try {
+      await db.delete(workspaceFiles).where(eq(workspaceFiles.userId, ownerId))
+      await deleteWorkspaceFixture(db, inArray(workspace.id, [workspaceId, otherWorkspaceId]))
+      await db.delete(user).where(eq(user.id, ownerId))
+    } finally {
+      await db.$client.end()
+    }
+  })
+
+  it.each([
+    { scope: 'active', label: 'live files', expected: byUploadThenId(live) },
+    { scope: 'archived', label: 'Recently Deleted files', expected: byUploadThenId(archived) },
+    {
+      scope: 'all',
+      label: 'live and deleted files together',
+      expected: byUploadThenId([...live, ...archived]),
+    },
+  ] as const)('lists $label by upload time, ties by id', async ({ scope, expected }) => {
+    const files = await listWorkspaceFiles(workspaceId, { scope, throwOnError: true })
+    expect(files.map((file) => file.id)).toEqual(expected)
+  })
+
+  /**
+   * With every sorting plan ruled out, an ordered plan exists only if an index's predicate is
+   * implied by the list's filter and its key gives the list's order.
+   */
+  it('reads the live list, whole or bounded, in order from its workspace index', async () => {
+    for (const limit of [undefined, WORKSPACE_FILE_SEED_MAX + 1]) {
+      const nodes = await explainCaptured(
+        db,
+        () => listWorkspaceFiles(workspaceId, { limit, throwOnError: true }),
+        { match: /from "workspace_files"/, gucs: { enable_sort: 'off' } }
+      )
+      expect(nodes.map((node) => node['Node Type'])).not.toContain('Sort')
+      expect(nodes.map((node) => node['Index Name'])).toContain(
+        'workspace_files_workspace_active_uploaded_idx'
+      )
+    }
+  })
+})

apps/sim/lib/uploads/contexts/workspace/file-manager.ts

@@ -1794,7 +1794,8 @@ async function hydrateWorkspaceFilePaths(
 }
 
 /**
- * List all files for a workspace
+ * List all files for a workspace, oldest upload first; `id` orders files uploaded at the same
+ * instant, so the order is stable across reads.
  */
 export async function listWorkspaceFiles(
   workspaceId: string,
@@ -1806,7 +1807,7 @@ export async function listWorkspaceFiles(
       .select(workspaceFileListColumns)
       .from(workspaceFiles)
       .where(workspaceFileScopeCondition(workspaceId, scope))
-      .orderBy(workspaceFiles.uploadedAt)
+      .orderBy(workspaceFiles.uploadedAt, workspaceFiles.id)
     const files = await (limit === undefined ? query : query.limit(limit))
 
     return await hydrateWorkspaceFilePaths(files, workspaceId, options)

packages/db/migrations/0442_workspace_files_active_uploaded_idx.sql

@@ -0,0 +1,35 @@
+-- A workspace's live file list, in upload order. No index served the list's filter in that
+-- order, so the planner matched every workspace's live files, filtered to one workspace on the
+-- heap and sorted them, on disk for a large workspace; a bounded read (the SSR seed, an
+-- uploadedAt keyset page) paid for the whole workspace.
+--
+-- The predicate is the one the other live workspace-file indexes share, which the list's filter
+-- implies. The list's owner conditions (project_id and organization_id IS NULL) stay heap checks
+-- that discard nothing: the project and organization binding CHECKs rule both out for
+-- workspace-context rows. The key stays narrow: carrying the listed name and storage key columns
+-- would bring an entry near the btree size limit and fail the write.
+--
+-- Replay-safety: workspace_files takes uploads and edits continuously, so the build is
+-- CONCURRENTLY, below an embedded COMMIT, and a failure there replays the whole file from the
+-- top. A failed concurrent build leaves an INVALID index that IF NOT EXISTS would keep: the replay
+-- moves only that one aside to be dropped, so a valid index built by an earlier attempt is kept.
+DO $$
+BEGIN
+  IF EXISTS (
+    SELECT 1 FROM pg_index WHERE indexrelid = to_regclass('workspace_files_workspace_active_uploaded_idx_failed')
+      AND (indisvalid OR indrelid <> to_regclass('workspace_files'))
+  ) THEN
+    RAISE EXCEPTION 'Refusing to drop unexpected recovery index workspace_files_workspace_active_uploaded_idx_failed';
+  END IF;
+  IF EXISTS (
+    SELECT 1 FROM pg_index WHERE indexrelid = to_regclass('workspace_files_workspace_active_uploaded_idx')
+      AND indrelid = to_regclass('workspace_files') AND NOT indisvalid
+  ) THEN
+    ALTER INDEX "workspace_files_workspace_active_uploaded_idx" RENAME TO "workspace_files_workspace_active_uploaded_idx_failed";
+  END IF;
+END $$;--> statement-breakpoint
+COMMIT;--> statement-breakpoint
+SET lock_timeout = 0;--> statement-breakpoint
+DROP INDEX CONCURRENTLY IF EXISTS "workspace_files_workspace_active_uploaded_idx_failed";--> statement-breakpoint
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "workspace_files_workspace_active_uploaded_idx" ON "workspace_files" USING btree ("workspace_id","uploaded_at","id") WHERE "workspace_files"."deleted_at" IS NULL AND "workspace_files"."context" = 'workspace' AND "workspace_files"."workspace_id" IS NOT NULL;--> statement-breakpoint
+SET lock_timeout = '5s';