fix(cleanup): purge chat inline images by prefix instead of by transcript

Inline chat images live at `chat-images/<chatId>/<requestId>/<sha>.webp`
with no `workspace_files` row, and chat cleanup found them by re-deriving
keys from the transcript it read before the delete. An image a turn published
after that read, which the admin removal hook and the account deletion
pre-read both leave a window for, was never collected and stayed in storage.

The `copilot.chat.backend-cleanup` handler now deletes every object under
`chat-images/<chatId>/` for each chat it confirms deleted, through the same
paged listing and deadline-aware step the Project prefix cleanup uses (moved
into `lib/uploads/core/sweep-prefix.ts` and shared), checkpointing how many
chats it finished and continuing in a later run near its deadline.
`collectChatFiles` no longer derives inline image keys and reads only each
message's `fileAttachments`.

So that every path still reaches the sweep, chat ids are queued even without
`COPILOT_API_KEY`, with a `copilotBackend` flag saying whether they also go to
the backend (absent on already-queued events, which keeps their meaning), and
the retention jobs behind `prepareChatCleanup` queue an image-only event for
the chats they confirm deleted. Events already queued with inline image file
entries still delete those keys normally. Comments on the account deletion
and admin removal paths now state the residual precisely: chats created after
the read, and attachments added to an already-read chat.

Autopsy: the tests seeded each chat's images together with the transcript
that names them, so collecting keys from the transcript always matched what
was stored; none wrote an object after the cleanup read, which is the window
the two removal paths leave open. The new integration case writes images
after the admin hook's read and fails on the old handler with the late images
still on disk; a bystander chat's images stay untouched.

Author: waleedlatif1

apps/sim/lib/auth/admin-removal-chat-cleanup.ts

@@ -19,9 +19,12 @@ const ADMIN_REMOVAL_CLEANUP_DELAY_MS = 60 * 60_000
  * queued now, before the delete, and delayed past it. The handler purges only chats whose rows
  * are gone, so a delete that fails or is refused purges nothing. A failed read or enqueue throws,
  * which aborts the removal rather than erasing chats whose cleanup nothing recorded. Better Auth's
- * delete runs outside any transaction this hook can join, so a chat or attachment written between
- * this read and that delete is not covered; account deletion, which owns its transaction, re-reads
- * under the account lock instead.
+ * delete runs outside any transaction this hook can join, so what is written between this read
+ * and that delete is covered only in part: a chat created in that window is not purged at all, and
+ * an attachment (a `workspace_files` row or message key) added to a chat already read stays in
+ * storage, while that chat's inline images still go, because the handler sweeps each read chat's
+ * whole image prefix. Account deletion, which owns its transaction, re-reads under the account
+ * lock instead.
  */
 export async function enqueueAdminRemovedChatCleanup(user: { id: string }): Promise<void> {
   const targets = await readAccountRemovalChats({

apps/sim/lib/cleanup/chat-backend-cleanup-outbox.ts

@@ -9,22 +9,67 @@ import {
   organizationAttachmentsStillReferenced,
 } from '@/lib/cleanup/chat-cleanup'
 import { env } from '@/lib/core/config/env'
-import type { OutboxHandlerRegistry } from '@/lib/core/outbox/service'
+import {
+  continueOutboxHandler,
+  deferOutboxHandler,
+  type OutboxEventContext,
+  type OutboxHandlerRegistry,
+} from '@/lib/core/outbox/service'
 import { mapWithConcurrency } from '@/lib/core/utils/concurrency'
+import {
+  INLINE_CHAT_IMAGE_PREFIX,
+  inlineImageRequestIdSchema,
+} from '@/lib/mothership/chat/inline-image-reference'
+import { outboxDeadlineNear } from '@/lib/uploads/core/prefix-sweep'
 import { deleteFile } from '@/lib/uploads/core/storage-service'
+import { sweepStoragePrefix } from '@/lib/uploads/core/sweep-prefix'
 
 const FILE_DELETE_CONCURRENCY = 10
 
+/**
+ * Deletes every inline image under each deleted chat's `chat-images/<chatId>/` prefix, from the
+ * `from`th chat on. `swept` is how many leading chats are done: all of them unless the deadline
+ * neared (`continue`) or a listing stopped advancing (`stalled`).
+ */
+async function sweepInlineImages(
+  chatIds: readonly string[],
+  deleted: ReadonlySet<string>,
+  from: number,
+  context: OutboxEventContext
+): Promise<{ swept: number; outcome: 'done' | 'continue' | 'stalled' }> {
+  for (let index = from; index < chatIds.length; index++) {
+    const chatId = chatIds[index]
+    // A chat id outside the image key grammar never had an image stored under it.
+    if (!deleted.has(chatId) || !inlineImageRequestIdSchema.safeParse(chatId).success) continue
+    for (;;) {
+      context.signal.throwIfAborted()
+      const outcome = await sweepStoragePrefix(
+        `${INLINE_CHAT_IMAGE_PREFIX}${chatId}/`,
+        'mothership',
+        context
+      )
+      if (outcome === 'done') break
+      if (outcome === 'stalled' || outboxDeadlineNear(context)) return { swept: index, outcome }
+    }
+    if (outboxDeadlineNear(context)) return { swept: index + 1, outcome: 'continue' }
+  }
+  return { swept: chatIds.length, outcome: 'done' }
+}
+
 export const chatBackendCleanupOutboxHandlers = {
   /**
-   * Purges the copilot backend data and stored files of chats whose rows are gone. A chat whose
-   * row survived (its delete was spared, rolled back, or never happened) keeps both. Any failure
-   * throws so the outbox retries it: a failed backend call, a failed object delete, or chat ids
-   * queued while `COPILOT_API_KEY` was set reaching a processor without it. An object already
-   * gone counts as deleted, so a retry repeats only what is left.
+   * Purges the inline images, copilot backend data and stored files of chats whose rows are gone.
+   * A chat whose row survived (its delete was spared, rolled back, or never happened) keeps all
+   * three. Inline images go by the chat's prefix rather than by key, so one a turn published after
+   * the delete read the transcript goes too; the sweep checkpoints how many chats it finished and
+   * continues in a later run once its deadline nears. Any failure throws so the outbox retries it:
+   * a failed backend call, a failed object delete, or chat ids queued for the backend reaching a
+   * processor without `COPILOT_API_KEY`. An object already gone counts as deleted, so a retry
+   * repeats only what is left.
    */
   [CHAT_BACKEND_CLEANUP_EVENT]: async (rawPayload, context) => {
-    const { chatIds, files } = chatBackendCleanupPayloadSchema.parse(rawPayload)
+    const { chatIds, files, copilotBackend, imagesSwept } =
+      chatBackendCleanupPayloadSchema.parse(rawPayload)
     context.signal.throwIfAborted()
     const named = [...new Set([...chatIds, ...files.map((file) => file.chatId)])]
     const surviving = await db
@@ -35,7 +80,16 @@ export const chatBackendCleanupOutboxHandlers = {
     const deletedChatIds = chatIds.filter((id) => !survivingIds.has(id))
     const deletedFiles = files.filter((file) => !survivingIds.has(file.chatId))
 
-    if (deletedChatIds.length > 0) {
+    if (imagesSwept < chatIds.length) {
+      const sweep = await sweepInlineImages(chatIds, new Set(deletedChatIds), imagesSwept, context)
+      if (sweep.swept > imagesSwept) await context.checkpointPayload({ imagesSwept: sweep.swept })
+      if (sweep.outcome === 'continue')
+        return continueOutboxHandler('Continuing chat inline image cleanup')
+      if (sweep.outcome === 'stalled')
+        return deferOutboxHandler('Chat inline image cleanup removed nothing from its listing')
+    }
+
+    if (copilotBackend && deletedChatIds.length > 0) {
       context.signal.throwIfAborted()
       if (!env.COPILOT_API_KEY) throw new Error('Copilot cleanup is not configured')
       const result = await cleanupCopilotBackend(

apps/sim/lib/cleanup/chat-backend-cleanup.integration.ts

@@ -81,6 +81,7 @@ import { chatBackendCleanupOutboxHandlers } from '@/lib/cleanup/chat-backend-cle
 import { env } from '@/lib/core/config/env'
 import { processOutboxEventById } from '@/lib/core/outbox/service'
 import { deleteCopilotChat } from '@/lib/mothership/chat/deletion'
+import { inlineChatImageKey } from '@/lib/mothership/chat/inline-image-key'
 import { deleteUserAccount } from '@/lib/users/account-deletion'
 
 setUploadDirServer(storageRoot)
@@ -434,6 +435,58 @@ describe('cleanup of the chats a removal cascades away', () => {
     expect(await objectExists(ownChat.keys.attachment)).toBe(false)
   })
 
+  /**
+   * Inline images have no row, and the hook reads the transcript before Better Auth deletes the
+   * user, so an image a turn publishes in between was never collected by key.
+   */
+  it('purges inline images a removed user’s chat published after the cleanup read it', async () => {
+    const removedId = await seedUser()
+    const bystanderId = await seedUser()
+    const workspaceId = await seedWorkspace(bystanderId, [removedId])
+    const chatId = await seedChat(removedId, workspaceId)
+    const bystandersChat = await seedChat(bystanderId, workspaceId)
+    const read = inlineChatImageKey(chatId, 'req-1', 'files/chart.png')
+    await db.insert(copilotMessages).values({
+      chatId,
+      messageId: generateId(),
+      role: 'assistant',
+      content: { role: 'assistant', requestId: 'req-1', content: '![chart](files/chart.png)' },
+    })
+    await putObject(read)
+    const bystanders = inlineChatImageKey(bystandersChat, 'req-1', 'files/chart.png')
+    await putObject(bystanders)
+
+    await enqueueAdminRemovedChatCleanup({ id: removedId })
+    const late = [
+      inlineChatImageKey(chatId, 'req-2', 'files/later.png'),
+      inlineChatImageKey(chatId, 'req-2', 'files/another.png'),
+    ]
+    for (const key of late) await putObject(key)
+    await db.delete(user).where(eq(user.id, removedId))
+
+    const delivered = await deliverCleanups()
+    expect(new Set(delivered.outcomes)).toEqual(new Set(['completed']))
+    for (const key of [read, ...late]) expect(await objectExists(key)).toBe(false)
+    expect(await objectExists(bystanders)).toBe(true)
+  })
+
+  it('still deletes the inline image keys an event queued before prefix sweeps names', async () => {
+    const ownerId = await seedUser()
+    const workspaceId = await seedWorkspace(ownerId)
+    const chatId = await seedChat(ownerId, workspaceId)
+    const key = inlineChatImageKey(chatId, 'req-1', 'files/chart.png')
+    await putObject(key)
+    await db.delete(copilotChats).where(eq(copilotChats.id, chatId))
+    await db.insert(outboxEvent).values({
+      id: generateId(),
+      eventType: CHAT_BACKEND_CLEANUP_EVENT,
+      payload: { chatIds: [], files: [{ key, context: 'mothership', chatId }] },
+    })
+
+    expect((await deliverCleanups()).outcomes).toEqual(['completed'])
+    expect(await objectExists(key)).toBe(false)
+  })
+
   it('purges nothing for a direct user delete that never happened', async () => {
     const keptId = await seedUser()
     const workspaceId = await seedWorkspace(keptId)

apps/sim/lib/cleanup/chat-backend-cleanup.ts

@@ -34,13 +34,17 @@ export const chatBackendCleanupPayloadSchema = z
   .object({
     chatIds: z.array(chatIdSchema).max(CHAT_BACKEND_CLEANUP_BATCH_SIZE).default([]),
     files: z.array(chatFileSchema).max(CHAT_BACKEND_CLEANUP_BATCH_SIZE).default([]),
+    /** Whether `chatIds` also reach the copilot backend; an event without it means they do. */
+    copilotBackend: z.boolean().default(true),
+    /** How many leading `chatIds` already had their inline image prefix swept; a checkpoint. */
+    imagesSwept: z.number().int().min(0).max(CHAT_BACKEND_CLEANUP_BATCH_SIZE).default(0),
   })
   .strict()
   .refine((payload) => payload.chatIds.length > 0 || payload.files.length > 0, {
     message: 'A chat cleanup names at least one chat or stored file',
   })
 
-type ChatBackendCleanupPayload = z.infer<typeof chatBackendCleanupPayloadSchema>
+type ChatBackendCleanupPayload = z.input<typeof chatBackendCleanupPayloadSchema>
 
 /** What a chat deletion leaves outside Sim's rows: the chats' copilot backend data and stored files. */
 export interface ChatCleanupTargets {
@@ -51,27 +55,31 @@ export interface ChatCleanupTargets {
 
 /**
  * Queues the purge of what the given chats leave outside Sim's rows once a delete cascades them
- * away: one bounded event per {@link CHAT_BACKEND_CLEANUP_BATCH_SIZE} chats for the copilot backend,
- * and one per as many stored files. Call it in the deleting transaction so the cleanup commits or
- * rolls back with the delete; the handler purges only chats whose rows are gone, so a chat the
- * delete ends up sparing keeps its data. A delete with no transaction to join queues it first,
- * with an `availableAt` past the moment the delete commits.
+ * away: one bounded event per {@link CHAT_BACKEND_CLEANUP_BATCH_SIZE} chats, which sweeps each
+ * chat's inline image prefix and purges its copilot backend data, and one per as many stored
+ * files. Call it in the deleting transaction so the cleanup commits or rolls back with the delete;
+ * the handler purges only chats whose rows are gone, so a chat the delete ends up sparing keeps its
+ * data. A delete with no transaction to join queues it first, with an `availableAt` past the
+ * moment the delete commits.
  *
- * Without `COPILOT_API_KEY` this Sim never reached a copilot backend, so no chat ids are queued for
- * it; the stored files are queued either way.
+ * Without `COPILOT_API_KEY` this Sim never reached a copilot backend, so the chats are queued for
+ * their image sweep only; `copilotBackend: false` does the same for a caller that already purged
+ * the backend itself.
  */
 export async function enqueueChatBackendCleanup(
   executor: Pick<typeof db, 'insert'>,
   targets: ChatCleanupTargets,
-  options: { availableAt?: Date } = {}
+  options: { availableAt?: Date; copilotBackend?: boolean } = {}
 ): Promise<void> {
-  const chatIds = env.COPILOT_API_KEY ? [...new Set(targets.chatIds)] : []
+  const chatIds = [...new Set(targets.chatIds)]
+  const copilotBackend = (options.copilotBackend ?? true) && Boolean(env.COPILOT_API_KEY)
   const filesByKey = new Map<string, ChatFileRef>()
   for (const file of targets.files) if (!filesByKey.has(file.key)) filesByKey.set(file.key, file)
   const payloads: ChatBackendCleanupPayload[] = [
     ...chunkArray(chatIds, CHAT_BACKEND_CLEANUP_BATCH_SIZE).map((batch) => ({
       chatIds: batch,
       files: [],
+      copilotBackend,
     })),
     ...chunkArray([...filesByKey.values()], CHAT_BACKEND_CLEANUP_BATCH_SIZE).map((batch) => ({
       chatIds: [],
@@ -84,7 +92,9 @@ export async function enqueueChatBackendCleanup(
     })),
   ]
   for (const events of chunkArray(payloads, MAX_BULK_ENQUEUE_EVENTS)) {
-    await enqueueOutboxEvents(executor, CHAT_BACKEND_CLEANUP_EVENT, events, options)
+    await enqueueOutboxEvents(executor, CHAT_BACKEND_CLEANUP_EVENT, events, {
+      availableAt: options.availableAt,
+    })
   }
 }
 
@@ -104,9 +114,11 @@ export async function readAccountRemovalChats(params: {
 /**
  * Queues the cleanup of every chat an account removal cascades away, inside its transaction:
  * the chat ids are re-read there, after the account row is locked, and the files of any chat
- * created since {@link readAccountRemovalChats} ran are read in the transaction. A key a turn adds
- * to an already-read chat between that read and the removal's run stops is not collected here;
- * when it names one of the account's uploads, the account storage sweep still reclaims it.
+ * created since {@link readAccountRemovalChats} ran are read in the transaction. Every chat's inline
+ * images go with its prefix sweep, however late they were written. An attachment a turn adds to
+ * an already-read chat between that read and the removal's run stops is not collected here; when
+ * it names one of the account's uploads, the account storage sweep still reclaims it, and
+ * otherwise it stays behind.
  */
 export async function enqueueAccountRemovalChatCleanupInTx(
   tx: DbTransaction,

apps/sim/lib/cleanup/chat-cleanup.integration.ts

@@ -6,7 +6,6 @@ import { compareStrings } from '@sim/utils/string'
 import { eq } from 'drizzle-orm'
 import { afterAll, beforeAll, describe, expect, it } from 'vitest'
 import { collectChatFiles } from '@/lib/cleanup/chat-files'
-import { inlineChatImageKey } from '@/lib/mothership/chat/inline-image-key'
 
 function attachment(key: string) {
   return { id: generateId(), key, filename: 'x.png', media_type: 'image/png', size: 1 }
@@ -51,7 +50,7 @@ describe('chat cleanup file collection in PostgreSQL', () => {
     return chat.id
   }
 
-  it('collects attachment and inline image keys from every row, never from tool payloads', async () => {
+  it('collects attachment keys from every row, never from tool payloads or inline images', async () => {
     const chatId = await seedChat(
       [
         {
@@ -110,12 +109,6 @@ describe('chat cleanup file collection in PostgreSQL', () => {
           chatId,
           organizationId: 'org-1',
         },
-        {
-          key: inlineChatImageKey(chatId, 'req-1', 'files/chart.png'),
-          context: 'mothership',
-          chatId,
-        },
-        { key: inlineChatImageKey(chatId, 'req-1', '/tmp/out.png'), context: 'mothership', chatId },
         { key: 'copilot/1234/legacy.png', context: 'copilot', chatId },
         { key: 'copilot/5/other.png', context: 'copilot', chatId: otherChatId },
         { key: 'copilot/9/removed.png', context: 'copilot', chatId },

apps/sim/lib/cleanup/chat-cleanup.test.ts

@@ -7,7 +7,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
 vi.mock('@/lib/uploads', () => uploadsMock)
 
 import { prepareChatCleanup } from '@/lib/cleanup/chat-cleanup'
-import { inlineChatImageKey } from '@/lib/mothership/chat/inline-image-key'
 
 const chatId = '3f0c2a52-8a43-4d4b-9b5f-0b3c7a1e2d10'
 
@@ -26,7 +25,7 @@ async function purge(
   queueTableRows(workspaceFiles, [])
   queueTableRows(
     copilotMessages,
-    messages.map((content) => ({ chatId, content }))
+    messages.map((content) => ({ chatId, fileAttachments: content.fileAttachments ?? null }))
   )
   const cleanup = await prepareChatCleanup([chatId], 'test')
   queueTableRows(copilotChats, [])
@@ -90,23 +89,4 @@ describe('chat purge storage', () => {
     ])
     expect(deleted).toEqual({ copilot: ['copilot/1234/kept.png'] })
   })
-
-  it('deletes the inline images an assistant message published', async () => {
-    const deleted = await purge([
-      {
-        role: 'assistant',
-        requestId: 'req-1',
-        content: 'Here: ![chart](files/chart.png) and `![code](files/no.png)`',
-        contentBlocks: [{ type: 'text', content: '![other](/tmp/out.png)' }],
-      },
-      { role: 'user', requestId: 'req-2', content: '![u](files/u.png)' },
-      { role: 'assistant', content: '![x](files/x.png)' },
-    ])
-    expect(deleted).toEqual({
-      mothership: [
-        inlineChatImageKey(chatId, 'req-1', 'files/chart.png'),
-        inlineChatImageKey(chatId, 'req-1', '/tmp/out.png'),
-      ],
-    })
-  })
 })

apps/sim/lib/cleanup/chat-cleanup.ts

@@ -4,6 +4,7 @@ import { createLogger } from '@sim/logger'
 import { chunkArray } from '@sim/utils/helpers'
 import { isRecordLike } from '@sim/utils/object'
 import { and, eq, inArray, or, sql } from 'drizzle-orm'
+import { enqueueChatBackendCleanup } from '@/lib/cleanup/chat-backend-cleanup'
 import {
   type ChatFileRef,
   type ChatScopedContext,
@@ -156,7 +157,7 @@ export async function cleanupCopilotBackend(
 
 /**
  * Full chat cleanup: collect file refs, then (after DB deletion by caller)
- * call copilot backend and delete storage files.
+ * call copilot backend, delete storage files, and queue the chats' inline image sweep.
  *
  * Usage:
  *   const cleanup = await prepareChatCleanup(chatIds, label)
@@ -207,6 +208,12 @@ export async function prepareChatCleanup(
         logger.info(
           `[${label}] Copilot backend: ${copilotResult.deleted} deleted, ${copilotResult.failed} failed`
         )
+        // Inline images have no row or key list to delete by; the outbox sweeps each chat's prefix.
+        await enqueueChatBackendCleanup(
+          cleanupDb,
+          { chatIds: confirmedChatIds, files: [] },
+          { copilotBackend: false }
+        )
       }
 
       // Delete storage files with correct context per file