fix(account-deletion): never purge objects that live data still names

Review of the durable storage sweep found ways it, or the in-memory purge it
backs up, could delete an object another tenant still uses.

Reference guard. Every account-deletion purge now runs one check before
deleting a key: the durable prefix sweep (per listed key), the durable
exact-key cleanup, and the post-commit in-memory purge. A key is kept when a
row still names it (workspace_files.key, workspace_file.key,
workspace_file_version.key, document.storage_key, execution_large_values.key)
or a stored asset URL names it: user.image, workspace.logo_url,
organization.logo, organization.whitelabel_settings, organization.metadata,
custom_block.icon_url, chat.customizations (deployed-chat logo and avatar) and
oauth_client.icon. URLs are mapped to keys with extractStorageKey, so serve
paths with or without a provider segment and direct bucket URLs both match;
over-matching only keeps an object. References embedded in free-form JSON or
text (workflow sub-block values, chat messages, Markdown links) are not
consulted; for the deleted account's own chat attachments that is the intended
erasure. This keeps whitelabel logos that were uploaded under an admin's
personal workspace prefix, and legacy kb/ objects shared across workspaces,
which the in-memory purge used to delete unchecked.

Exact-key cleanup also keeps any key under a workspace-keyed root whose
workspace still exists, since its members can read it by key through the
metadata fallback. Workspace-less rows are queued only for the Copilot
context: the workspace binding check was added NOT VALID, so legacy
shared-context rows without a workspace can name a surviving workspace's
object. Profile pictures are taken only from the account's own user.image,
never from rows, because whitelabel logos were once stored as
profile-pictures rows under an admin's account; organization logos are never
queued either.

Prefix deletion now counts only objects actually removed. A tolerated ENOENT
(locally, any file whose on-disk name no sanitized key resolves to) used to
count as progress, so a page of such files re-listed forever without spending
an attempt. The workspace sweep and the Project prefix cleanup (which had the
same flaw) now defer and spend an attempt when a full page removes nothing.
A local metadata sidecar is kept whenever its object is. Versions of the
account's private files are queued in keyset pages that stop at the cap
instead of one unbounded query inside the deletion transaction.

Author: waleedlatif1

apps/sim/lib/projects/files/prefix-cleanup.test.ts

@@ -0,0 +1,79 @@
+import { mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { dirname, join } from 'node:path'
+import { resetDbChainMock } from '@sim/testing'
+import { uploadsConfigMock } from '@sim/testing/mocks/uploads-config.mock'
+import { uploadsMetadataMock } from '@sim/testing/mocks/uploads-metadata.mock'
+import { setUploadDirServer, uploadsSetupMock } from '@sim/testing/mocks/uploads-setup.mock'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/uploads/core/setup.server', () => uploadsSetupMock)
+vi.mock('@/lib/uploads/config', () => uploadsConfigMock)
+vi.mock('@/lib/uploads/server/metadata', () => uploadsMetadataMock)
+
+import type { OutboxEventContext } from '@/lib/core/outbox/service'
+import { PROJECT_FILE_PREFIX_CLEANUP_EVENT } from '@/lib/projects/files/outbox-events'
+import { projectFilePrefixCleanupOutboxHandlers } from '@/lib/projects/files/prefix-cleanup'
+
+const cleanup = projectFilePrefixCleanupOutboxHandlers[PROJECT_FILE_PREFIX_CLEANUP_EVENT]
+
+let root: string
+
+function outboxContext(deadlineAt?: number): OutboxEventContext {
+  return {
+    eventId: 'event-1',
+    eventType: PROJECT_FILE_PREFIX_CLEANUP_EVENT,
+    attempts: 0,
+    maxAttempts: 10,
+    signal: new AbortController().signal,
+    ...(deadlineAt ? { deadlineAt } : {}),
+    checkpointPayload: vi.fn(async () => {}),
+  }
+}
+
+async function put(key: string): Promise<void> {
+  await mkdir(dirname(join(root, key)), { recursive: true })
+  await writeFile(join(root, key), 'x')
+}
+
+async function exists(key: string): Promise<boolean> {
+  try {
+    await stat(join(root, key))
+    return true
+  } catch {
+    return false
+  }
+}
+
+describe('Project prefix cleanup on local storage', () => {
+  beforeEach(async () => {
+    resetDbChainMock()
+    root = await mkdtemp(join(tmpdir(), 'sim-project-prefix-'))
+    setUploadDirServer(root)
+  })
+
+  afterEach(async () => {
+    await rm(root, { recursive: true, force: true })
+  })
+
+  it('deletes the retired Project prefix and completes', async () => {
+    await put('project/p1/a.txt')
+    await put('project/p1/nested/b.txt')
+    await put('project/p10/keep.txt')
+
+    await expect(cleanup({ projectId: 'p1' }, outboxContext())).resolves.toBeUndefined()
+
+    expect(await exists('project/p1/a.txt')).toBe(false)
+    expect(await exists('project/p1/nested/b.txt')).toBe(false)
+    expect(await exists('project/p10/keep.txt')).toBe(true)
+  })
+
+  it('spends an attempt when a full page holds only objects it cannot remove', async () => {
+    for (let index = 0; index < 520; index++) await put(`project/p1/legacy file ${index}.txt`)
+
+    const result = await cleanup({ projectId: 'p1' }, outboxContext(Date.now() + 60_000))
+
+    expect(result).toMatchObject({ outcome: 'deferred' })
+    expect(result).not.toMatchObject({ consumeAttempt: false })
+  })
+})

apps/sim/lib/projects/files/prefix-cleanup.ts

@@ -7,6 +7,7 @@ import { isRecordLike } from '@sim/utils/object'
 import { and, asc, eq, gt, inArray, isNull, sql } from 'drizzle-orm'
 import {
   continueOutboxHandler,
+  deferOutboxHandler,
   enqueueOutboxEvent,
   type OutboxEventContext,
   type OutboxHandler,
@@ -332,7 +333,9 @@ export async function listStorageKeyPage(
 /**
  * Deletes listed keys, each of which must sit under `prefix`, and prunes local directories the
  * deletions empty, never above the prefix itself. Stops before another deletion once the handler
- * deadline is near; `complete` is false when it did.
+ * deadline is near; `complete` is false when it did. `deleted` counts only objects actually
+ * removed: a key already gone (or, locally, one whose on-disk name no storage key resolves to) is
+ * no progress, so a caller re-listing the same page can tell that it is stuck.
  */
 export async function deleteStorageKeysUnderPrefix(
   prefix: string,
@@ -341,17 +344,21 @@ export async function deleteStorageKeysUnderPrefix(
   outbox: Pick<OutboxEventContext, 'signal' | 'deadlineAt'>
 ): Promise<{ deleted: number; complete: boolean }> {
   if (prefix.length < 2 || !prefix.endsWith('/')) throw new Error('Unbounded storage prefix')
+  let attempted = 0
   let deleted = 0
   for (const key of keys) {
     if (!key.startsWith(prefix))
       throw new Error('Storage provider returned a key outside the prefix')
     outbox.signal.throwIfAborted()
-    if (deleted > 0 && outbox.deadlineAt && Date.now() + 5000 >= outbox.deadlineAt)
+    if (attempted > 0 && outbox.deadlineAt && Date.now() + 5000 >= outbox.deadlineAt)
       return { deleted, complete: false }
+    attempted++
+    let removed = true
     try {
       await deleteFile({ key, context, signal: outbox.signal })
     } catch (error) {
       if (describeError(error).code !== 'ENOENT') throw error
+      removed = false
     }
     if (!USE_S3_STORAGE && !USE_BLOB_STORAGE && !USE_GCS_STORAGE) {
       const root = join(UPLOAD_DIR_SERVER, prefix.slice(0, -1))
@@ -361,16 +368,21 @@ export async function deleteStorageKeysUnderPrefix(
         directory = dirname(directory)
       }
     }
-    deleted++
+    if (removed) deleted++
   }
   return { deleted, complete: true }
 }
 
-async function sweepPrefix(projectId: string, context: OutboxEventContext): Promise<boolean> {
+/** `stalled` when a page that must be re-listed removed nothing, so continuing cannot progress. */
+async function sweepPrefix(
+  projectId: string,
+  context: OutboxEventContext
+): Promise<'done' | 'continue' | 'stalled'> {
   const prefix = prefixFor(projectId)
   const { keys } = await listStorageKeyPage(prefix, 'project', context.signal)
-  const { complete } = await deleteStorageKeysUnderPrefix(prefix, 'project', keys, context)
-  return complete && keys.length < PAGE_SIZE
+  const { deleted, complete } = await deleteStorageKeysUnderPrefix(prefix, 'project', keys, context)
+  if (complete && keys.length < PAGE_SIZE) return 'done'
+  return deleted > 0 ? 'continue' : 'stalled'
 }
 
 const cleanupPrefix: OutboxHandler<unknown> = async (raw, context) => {
@@ -410,8 +422,10 @@ const cleanupPrefix: OutboxHandler<unknown> = async (raw, context) => {
       return continueOutboxHandler('Continuing retired Project upload cleanup')
     }
   }
-  if (!(await sweepPrefix(payload.projectId, context)))
-    return continueOutboxHandler('Continuing Project storage cleanup')
+  const sweep = await sweepPrefix(payload.projectId, context)
+  if (sweep === 'continue') return continueOutboxHandler('Continuing Project storage cleanup')
+  if (sweep === 'stalled')
+    return deferOutboxHandler('Project storage cleanup removed nothing from a full listing page')
 }
 
 /**

apps/sim/lib/users/account-deletion-attachments.test.ts

@@ -90,6 +90,18 @@ describe('account deletion of private organization Assistant images', () => {
     }
   )
 
+  it('keeps an object a surviving row still references', async () => {
+    queueTableRows(schemaMock.uploadSession, [
+      { id: 'upload-1', key: IMAGE_KEY },
+      { id: 'upload-2', key: FAILED_IMAGE_KEY },
+    ])
+    queueTableRows(schemaMock.document, [{ key: IMAGE_KEY }])
+
+    await deleteUserAccount('user-1')
+
+    expect(mockDeleteFiles).toHaveBeenCalledExactlyOnceWith([FAILED_IMAGE_KEY], 'mothership')
+  })
+
   it('retains ownership while an issued upload URL could recreate a purged object', async () => {
     queueTableRows(schemaMock.uploadSession, [{ id: 'upload-1', key: IMAGE_KEY }])
 

apps/sim/lib/users/account-deletion.ts

@@ -54,7 +54,10 @@ import {
 } from '@/lib/table/rows/executions'
 import type { StorageContext } from '@/lib/uploads'
 import { isUsingCloudStorage, StorageService } from '@/lib/uploads'
-import { enqueueAccountStorageCleanupInTx } from '@/lib/users/storage-sweep'
+import {
+  createStorageReferenceGuard,
+  enqueueAccountStorageCleanupInTx,
+} from '@/lib/users/storage-sweep'
 import {
   listSharedResourceWorkspaceIdsForUser,
   listUnresolvedSharedResourceWorkspaces,
@@ -638,11 +641,25 @@ async function collectAccountStorageKeys(
  * time this runs the account is already gone, and the request must not report a
  * failure for work that cannot be undone. An orphaned object is recoverable from
  * the log; a deletion the caller believes failed is not.
+ *
+ * Keys a surviving row or stored asset URL still names are kept (a legacy `kb/`
+ * object can back documents in more than one workspace); see
+ * `createStorageReferenceGuard`.
  */
 async function purgeStorageObjects(userId: string, batches: StorageKeyBatch[]): Promise<void> {
-  for (const { context, keys } of batches) {
-    if (keys.length === 0) continue
+  const isReferenced = createStorageReferenceGuard()
+  for (const { context, keys: collected } of batches) {
+    if (collected.length === 0) continue
     try {
+      const referenced = await isReferenced(collected)
+      if (referenced.size > 0) {
+        logger.warn('Keeping objects a surviving reference still names', {
+          context,
+          kept: referenced.size,
+        })
+      }
+      const keys = collected.filter((key) => !referenced.has(key))
+      if (keys.length === 0) continue
       const { failed } = await StorageService.deleteFiles(keys, context)
       for (const { key, error } of failed) {
         logger.error('Failed to erase stored object during account deletion', {

apps/sim/lib/users/storage-sweep.integration.ts

@@ -151,6 +151,9 @@ async function seed() {
     privateAttachment: `workspace/${survivingId}/${generateId()}/private.txt`,
     othersChatAttachment: `workspace/${survivingId}/${generateId()}/others.txt`,
     personalCopilot: `copilot/${generateId()}/personal.txt`,
+    legacyWorkspacelessKb: `kb/${generateId()}-legacy.txt`,
+    profilePictureRowLogo: `profile-pictures/${generateId()}/org-logo.png`,
+    logoNamedByUrl: `workspace-logos/${doomedId}/${generateId()}-org-logo.png`,
   }
   await db.insert(workspaceFiles).values([
     fileRow({
@@ -197,7 +200,23 @@ async function seed() {
       workspaceId: null,
       context: 'copilot',
     }),
+    fileRow({
+      key: keys.legacyWorkspacelessKb,
+      userId: departingId,
+      workspaceId: null,
+      context: 'knowledge-base',
+    }),
+    fileRow({
+      key: keys.profilePictureRowLogo,
+      userId: departingId,
+      workspaceId: null,
+      context: 'profile-pictures',
+    }),
   ])
+  await db
+    .update(user)
+    .set({ image: `/api/files/serve/${encodeURIComponent(keys.logoNamedByUrl)}` })
+    .where(eq(user.id, ownerId))
   for (const key of Object.values(keys)) await put(key)
   return { ownerId, departingId, doomedId, survivingId, keys }
 }
@@ -259,6 +278,8 @@ describe('account deletion storage sweep', () => {
       )
       expect(queuedKeys.map((object) => object.key)).not.toContain(f.keys.othersChatAttachment)
       expect(queuedKeys.map((object) => object.key)).not.toContain(f.keys.handedOff)
+      expect(queuedKeys.map((object) => object.key)).not.toContain(f.keys.legacyWorkspacelessKb)
+      expect(queuedKeys.map((object) => object.key)).not.toContain(f.keys.profilePictureRowLogo)
 
       const lateUpload = `workspace/${f.doomedId}/late-${generateId()}.bin`
       await put(lateUpload)
@@ -270,7 +291,6 @@ describe('account deletion storage sweep', () => {
         f.keys.rowlessExecution,
         f.keys.rowlessSnapshot,
         f.keys.rowlessCompiled,
-        f.keys.privateAttachment,
         f.keys.personalCopilot,
         lateUpload,
       ])
@@ -280,6 +300,10 @@ describe('account deletion storage sweep', () => {
         f.keys.neighbor,
         f.keys.handedOff,
         f.keys.othersChatAttachment,
+        f.keys.privateAttachment,
+        f.keys.legacyWorkspacelessKb,
+        f.keys.profilePictureRowLogo,
+        f.keys.logoNamedByUrl,
       ])
         expect(await readFile(join(storageRoot, key), 'utf8')).toBe('x')
       expect(