fix(content): keep stored provenance consistent (#9097)

Author: icecrasher321

apps/sim/lib/internal/file/operations.test.ts

@@ -1278,55 +1278,6 @@ describe('file manage operations', () => {
     expect(mockUploadWorkspaceFile).toHaveBeenCalled()
   })
 
-  it('downgrades provenance when overwriting a file owned by another user', async () => {
-    const existing = workspaceFile('report', 'other-user')
-    mockResolveWorkspaceFileReference.mockResolvedValue(existing)
-    mockUpdateWorkspaceFileContent.mockResolvedValue(existing)
-
-    const response = await POST(
-      createMockRequest(
-        'POST',
-        {
-          operation: 'write',
-          workspaceId: 'workspace-1',
-          fileName: 'report.txt',
-          content: 'secret-value',
-          overwrite: true,
-          __privateSecretProvenance: {
-            version: 1,
-            complete: true,
-            selections: [
-              {
-                key: 'content',
-                provenance: {
-                  version: 1,
-                  complete: true,
-                  entries: [{ name: 'TOKEN', encryptedValue: 'encrypted-token' }],
-                  scope: { userId: 'user-1', workspaceId: 'workspace-1' },
-                },
-              },
-            ],
-          },
-        },
-        PRIVATE_SECRET_PROVENANCE_HEADER
-      )
-    )
-
-    expect(response.status).toBe(200)
-    expect(mockUpdateWorkspaceFileContent).toHaveBeenCalledWith(
-      'workspace-1',
-      'report',
-      'user-1',
-      Buffer.from('secret-value'),
-      'text/plain',
-      {
-        version: { source: 'workflow', authorUserId: 'user-1' },
-        expectedUpdatedAt: CONTENT_UPDATED_AT,
-        secretProvenancePolicy: { mode: 'replace', provenance: { status: 'unknown' } },
-      }
-    )
-  })
-
   it('atomically binds append provenance to the exact predecessor version', async () => {
     const existing = workspaceFile('file-1')
     mockResolveWorkspaceFileReference.mockResolvedValue(existing)

apps/sim/lib/internal/file/operations.ts

@@ -1412,14 +1412,6 @@ export async function executeFileManageOperation(
             leafName,
           })
           if (existing) {
-            // Writing into a file someone else owns must not hand its owner an
-            // exact, re-resolvable secret lineage, exactly as appending does.
-            const overwriteProvenance =
-              writeProvenance?.status === 'exact' &&
-              writeProvenance.entries.length > 0 &&
-              existing.uploadedBy !== userId
-                ? { status: 'unknown' as const }
-                : writeProvenance
             const { file: overwritten } = await updateWorkspaceFileContent.execute({
               principal,
               input: {
@@ -1431,7 +1423,7 @@ export async function executeFileManageOperation(
                 provenanceMode: 'replace_empty',
                 expectedUpdatedAt: existing.contentUpdatedAt ?? undefined,
                 expectedRevision,
-                ...(overwriteProvenance ? { secretProvenance: overwriteProvenance } : {}),
+                ...(writeProvenance ? { secretProvenance: writeProvenance } : {}),
               },
             })
 
@@ -1657,14 +1649,9 @@ export async function executeFileManageOperation(
             )
           }
           const appendedProvenance = appendedResolution.provenanceBySelection?.get('content')
-          const secretProvenance =
-            appendedProvenance?.status === 'exact' &&
-            appendedProvenance.entries.length > 0 &&
-            existing.uploadedBy !== userId
-              ? { status: 'unknown' as const }
-              : appendedProvenance
-                ? mergeWorkspaceFileSecretProvenance(existingProvenance, appendedProvenance)
-                : undefined
+          const secretProvenance = appendedProvenance
+            ? mergeWorkspaceFileSecretProvenance(existingProvenance, appendedProvenance)
+            : undefined
           const { content: existingBuffer } = await readWorkspaceFileContent.execute({
             principal,
             input: {

apps/sim/lib/knowledge/__integration__/connector-save-concurrency.integration.ts

@@ -15,6 +15,7 @@ import {
   workspace,
 } from '@sim/db/schema'
 import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures'
+import { sha256Hex } from '@sim/security/hash'
 import { generateId } from '@sim/utils/id'
 import { and, eq, inArray, isNotNull, isNull, sql } from 'drizzle-orm'
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'
@@ -113,6 +114,78 @@ describe('independent connector saves in one knowledge base', () => {
     await db.$client.end()
   })
 
+  it('keeps retained chunk tags consistent during metadata-only connector refresh', async () => {
+    const item = await source('retry hash')
+    const chunkId = generateId()
+    const refreshedTags = {
+      tag1: 'current',
+      tag2: null,
+      number1: 3.5,
+      boolean1: true,
+      date1: new Date('2026-10-01T00:00:00.000Z'),
+    }
+    await db
+      .update(document)
+      .set({ tag1: 'old', tag2: 'cleared', chunkCount: 1 })
+      .where(eq(document.id, item.documentId))
+    await db.insert(embedding).values({
+      id: chunkId,
+      knowledgeBaseId: ids.knowledgeBaseId,
+      documentId: item.documentId,
+      chunkIndex: 0,
+      chunkHash: sha256Hex('retained content'),
+      content: 'retained content',
+      contentLength: 16,
+      tokenCount: 4,
+      startOffset: 0,
+      endOffset: 16,
+      embedding: Array(1536).fill(0.2),
+      tag1: 'old',
+      tag2: 'cleared',
+    })
+    await persistHashOnlyUpdates(
+      ids.knowledgeBaseId,
+      ids.connectorId,
+      [
+        {
+          existingId: item.documentId,
+          externalId: item.extDoc.externalId,
+          contentHash: 'refreshed-hash',
+          sourceMetadata: { sourceUrl: null, sourceModifiedAt: null, ...refreshedTags },
+        },
+      ],
+      { stillHeld: () => stillHoldsSyncLock(ids.connectorId, ids.lockId) }
+    )
+    expect(
+      await db
+        .select({
+          tag1: document.tag1,
+          tag2: document.tag2,
+          number1: document.number1,
+          boolean1: document.boolean1,
+          date1: document.date1,
+          count: document.chunkCount,
+          status: document.processingStatus,
+        })
+        .from(document)
+        .where(eq(document.id, item.documentId))
+    ).toEqual([{ ...refreshedTags, count: 1, status: 'completed' }])
+    expect(
+      await db
+        .select({
+          tag1: embedding.tag1,
+          tag2: embedding.tag2,
+          number1: embedding.number1,
+          boolean1: embedding.boolean1,
+          date1: embedding.date1,
+          content: embedding.content,
+          id: embedding.id,
+        })
+        .from(embedding)
+        .where(eq(embedding.documentId, item.documentId))
+    ).toEqual([{ ...refreshedTags, content: 'retained content', id: chunkId }])
+  })
+
   async function source(kind: SaveKind) {
     const item = {
       documentId: generateId(),

apps/sim/lib/knowledge/__integration__/execution-archive-provenance.integration.ts

@@ -12,14 +12,15 @@ import {
   knowledgeBase,
   organization,
   outboxEvent,
+  permissions,
   user,
   workspace,
   workspaceFiles,
 } from '@sim/db/schema'
 import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures'
 import { remoteSandboxMock, remoteSandboxMockFns } from '@sim/testing/mocks/remote-sandbox.mock'
 import { generateId } from '@sim/utils/id'
-import { eq, inArray, sql } from 'drizzle-orm'
+import { and, eq, inArray, sql } from 'drizzle-orm'
 import JSZip from 'jszip'
 import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
 
@@ -64,7 +65,10 @@ vi.mock('@/lib/embeddings', async () => ({
 }))
 
 import { functionExecuteBodySchema } from '@/lib/api/contracts'
-import { fileManageDecompressBodySchema } from '@/lib/api/contracts/tools/file'
+import {
+  fileManageBodySchema,
+  fileManageDecompressBodySchema,
+} from '@/lib/api/contracts/tools/file'
 import { observeContentDelivery } from '@/lib/core/application'
 import { processOutboxEventById } from '@/lib/core/outbox/service'
 import { decryptSecret, encryptSecret } from '@/lib/core/security/encryption'
@@ -990,9 +994,121 @@ describe('execution archive durable provenance', () => {
     expect(await isOpaqueWorkspaceFileEgressSafe(ids.workspaceId, source.identity)).toBe(true)
   })
 
-  it.each([false, true])(
-    'refuses tracked unknown execution attachments with historical metadata (archivedOnly=%s)',
-    async (archivedOnly) => {
+  it.each(['append', 'write'] as const)(
+    'preserves encrypted origins when another member performs %s',
+    async (operation) => {
+      const ids = await seed()
+      await db
+        .update(permissions)
+        .set({ permissionType: 'write' })
+        .where(and(eq(permissions.userId, ids.bobId), eq(permissions.entityId, ids.workspaceId)))
+      const priorSecret = 'fixture-prior-origin-secret'
+      const previous = {
+        status: 'exact' as const,
+        entries: [
+          {
+            encryptedValue: (await encryptSecret(priorSecret)).encrypted,
+            sourceUserId: ids.aliceId,
+            sourceWorkspaceId: ids.workspaceId,
+            name: 'PRIOR_TOKEN',
+          },
+        ],
+      }
+      const source = await uploadWorkspaceFile(
+        ids.workspaceId,
+        ids.aliceId,
+        Buffer.from(priorSecret),
+        `${generateId()}.txt`,
+        'text/plain',
+        { secretProvenance: previous, notifyWorkspaceChange: false }
+      )
+      const response = await executeFileManageOperation(
+        fileManageBodySchema.parse({
+          operation,
+          workspaceId: ids.workspaceId,
+          fileName: source.name,
+          content: FIXTURE_SECRET,
+          ...(operation === 'write' ? { overwrite: true } : {}),
+          [PRIVATE_SECRET_PROVENANCE_FIELD]: {
+            version: 1,
+            complete: true,
+            selections: [
+              {
+                key: 'content',
+                provenance: {
+                  version: 1,
+                  complete: true,
+                  scope: { userId: ids.bobId, workspaceId: ids.workspaceId },
+                  entries: [
+                    {
+                      name: 'NEW_TOKEN',
+                      encryptedValue: (await encryptSecret(FIXTURE_SECRET)).encrypted,
+                    },
+                  ],
+                },
+              },
+            ],
+          },
+        }),
+        {
+          principal: createWorkspaceFileDelegatedPrincipal({
+            serviceId: 'executor',
+            subjectUserId: ids.bobId,
+            workspaceId: ids.workspaceId,
+            delegationId: generateId(),
+            executionId: ids.executionId,
+          }),
+          workspaceId: ids.workspaceId,
+          attributedUserId: ids.bobId,
+          fileAccessUserId: ids.bobId,
+          workflowId: ids.workflowId,
+          executionId: ids.executionId,
+          headers: new Headers({
+            [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1,
+          }),
+          requestId: generateId(),
+        }
+      )
+      expect(response.status, await response.text()).toBe(200)
+      const record = await getWorkspaceFile(ids.workspaceId, source.id)
+      if (!record) throw new Error('Edited file disappeared')
+      const bytes = await downloadFile({ key: record.key, context: 'workspace' })
+      expect(bytes.toString()).toBe(
+        operation === 'append' ? priorSecret + FIXTURE_SECRET : FIXTURE_SECRET
+      )
+      const registry = new ResolvedSecretTraceRegistry([], {
+        userId: ids.bobId,
+        workspaceId: ids.workspaceId,
+      })
+      expect(
+        await importWorkspaceFileSecretProvenanceForRuntime({
+          workspaceId: ids.workspaceId,
+          identity: {
+            fileId: record.id,
+            key: record.key,
+            context: 'workspace',
+            contentUpdatedAt: record.contentUpdatedAt ?? undefined,
+          },
+          registry,
+        })
+      ).toBe(true)
+      const shown = projectResolvedSecretModelContent(bytes.toString(), registry)
+      expect(shown.safe).toBe(true)
+      expect(JSON.stringify(shown)).not.toContain(FIXTURE_SECRET)
+      expect(JSON.stringify(shown)).not.toContain(priorSecret)
+      expect(JSON.stringify(shown)).toContain('{{NEW_TOKEN}}')
+      if (operation === 'append') expect(JSON.stringify(shown)).toContain('[REDACTED_SECRET]')
+    }
+  )
+
+  it.each([
+    ['execution', false],
+    ['execution', true],
+    ['workspace', true],
+    ['mothership', true],
+  ] as const)(
+    'retains refusal evidence for %s attachments (archivedOnly=%s)',
+    async (context, archivedOnly) => {
       const ids = await seed()
       const file = await uploadExecutionFile(
         ids,
@@ -1005,7 +1121,7 @@ describe('execution archive durable provenance', () => {
       if (archivedOnly) {
         await db
           .update(workspaceFiles)
-          .set({ deletedAt: new Date() })
+          .set({ deletedAt: new Date(), context })
           .where(eq(workspaceFiles.key, file.key))
       } else {
         await db.insert(workspaceFiles).values({