fix(search): bind live IDs to their reader through the checksum, not a visible field

Author: waleedlatif1

apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts

@@ -135,10 +135,10 @@ describe.skipIf(!fixtureFile)('federated self-hosted GitLab Search', () => {
 
   function documentForReader(documentId: string, person: FixturePerson) {
     const scope = resourceScopeKey(resourceScopeFromOwner({ organizationId: ids.organizationId }))
-    return encodeLiveReference({
-      ...decodeLiveReference(documentId),
-      owner: liveReferenceOwner(person.simId, scope),
-    })
+    return encodeLiveReference(
+      decodeLiveReference(documentId),
+      liveReferenceOwner(person.simId, scope)
+    )
   }
 
   beforeAll(async () => {

apps/sim/lib/knowledge/mcp/server.test.ts

@@ -55,7 +55,6 @@ vi.mock('@/lib/knowledge/application/chat', () => ({
 
 import { OrchestrationError } from '@/lib/core/orchestration/types'
 import { createKnowledgeMcpServer } from '@/lib/knowledge/mcp/server'
-import { encodeLiveReference, liveReferenceOwner } from '@/lib/sim-search/live/reference'
 import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 
 const mocks = hoisted
@@ -302,24 +301,6 @@ describe('batched document reads', () => {
     expect(mocks.liveRead).toHaveBeenCalledTimes(2)
   })
 
-  it('reads a document once when the batch names it in both ID forms', async () => {
-    create()
-    mocks.liveRead.mockImplementation(async ({ input }: { input: { documentId: string } }) =>
-      readResult(input.documentId)
-    )
-    const reference = { provider: 'google_drive', account: 'account', id: 'doc' } as const
-    const earlier = `live:${Buffer.from(
-      JSON.stringify({ v: 1, user: 'person-1', scope: 'organization:org-1', ...reference })
-    ).toString('base64url')}`
-    const current = encodeLiveReference({
-      ...reference,
-      owner: liveReferenceOwner('person-1', 'organization:org-1'),
-    })
-    await call('read_document', { documentIds: [earlier, current] })
-    expect(mocks.rateLimit).toHaveBeenCalledWith(request, auth, expect.anything(), 1)
-    expect(mocks.liveRead).toHaveBeenCalledTimes(1)
-  })
-
   it('reads at most five documents from providers at once, in request order', async () => {
     create()
     let inFlight = 0

apps/sim/lib/knowledge/mcp/server.ts

@@ -30,7 +30,6 @@ import { liveCitationId } from '@/lib/knowledge/search/citation'
 import { toolError } from '@/lib/mcp/tool-result'
 import { readLiveDocument, searchLiveKnowledge } from '@/lib/sim-search/live/application'
 import { LiveReadError } from '@/lib/sim-search/live/read-error'
-import { liveDocumentKey } from '@/lib/sim-search/live/reference'
 import { v2CaughtOrchestrationError } from '@/app/api/v2/lib/response'
 import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 
@@ -226,9 +225,7 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe
     async (raw: unknown, extra: { signal: AbortSignal }) => {
       const parsed = readLiveDocumentMcpSchema.safeParse(raw)
       const batch =
-        parsed.success && parsed.data.documentIds
-          ? [...new Map(parsed.data.documentIds.map((id) => [liveDocumentKey(id), id])).values()]
-          : []
+        parsed.success && parsed.data.documentIds ? [...new Set(parsed.data.documentIds)] : []
       return execute(
         'read_document',
         knowledgeOperations.readDocument,

apps/sim/lib/knowledge/search/filters.test.ts

@@ -3,18 +3,17 @@ import { liveCitationId } from '@/lib/knowledge/search/citation'
 import { intersectWorkspaceSearchFilters } from '@/lib/knowledge/search/filters'
 import { encodeLiveReference, liveReferenceOwner } from '@/lib/sim-search/live/reference'
 
+const owner = liveReferenceOwner('reader', 'workspace:w')
 const fields = {
-  owner: liveReferenceOwner('reader', 'workspace:w'),
   account: 'account',
   provider: 'slack' as const,
   id: '1730912345.123456',
   container: 'C1',
 }
 /** One Slack message in the current ID form and in the earlier form older conversations hold. */
-const currentId = encodeLiveReference(fields)
-const { owner: _owner, ...located } = fields
+const currentId = encodeLiveReference(fields, owner)
 const earlierId = `live:${Buffer.from(
-  JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...located })
+  JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...fields })
 ).toString('base64url')}`
 
 describe('Assistant search scope', () => {
@@ -59,7 +58,7 @@ describe('live citation IDs', () => {
   it('cite one document as one source whichever ID form named it', () => {
     expect(liveCitationId(earlierId)).toBe(liveCitationId(currentId))
     expect(liveCitationId(currentId)).not.toBe(
-      liveCitationId(encodeLiveReference({ ...fields, id: '1730912345.123457' }))
+      liveCitationId(encodeLiveReference({ ...fields, id: '1730912345.123457' }, owner))
     )
   })
 })

apps/sim/lib/sim-search/live/application.test.ts

@@ -1192,14 +1192,10 @@ describe('authorized live retrieval', () => {
   describe('live document IDs', () => {
     const workspaceScope = resourceScopeKey(resourceScopeFromOwner({ workspaceId: 'workspace' }))
     /** The earlier wire form, spelled out so these tests pin what older conversations hold. */
-    const legacyId = ({ owner: _owner, ...fields }: Record<string, string>) =>
+    const legacyId = (fields: Record<string, string>) =>
       `live:${Buffer.from(JSON.stringify({ v: 1, ...fields })).toString('base64url')}`
-    const fields = {
-      owner: liveReferenceOwner('reader', workspaceScope),
-      account: 'account',
-      provider: 'google_drive' as const,
-      id: 'doc',
-    }
+    const readerOwner = liveReferenceOwner('reader', workspaceScope)
+    const fields = { account: 'account', provider: 'google_drive' as const, id: 'doc' }
     const read = (documentId: string, extra: Record<string, unknown> = {}, as = principal) =>
       readLiveDocument.execute({
         principal: as,
@@ -1217,8 +1213,8 @@ describe('authorized live retrieval', () => {
       const [first] = search.results
       if (!first) throw new Error('Expected a search result')
       const { documentId } = first
-      expect(documentId).toBe(encodeLiveReference(fields))
-      expect(documentId).toMatch(/^live2:google_drive~[0-9a-f]{8}~account~doc~[0-9a-f]{6}$/)
+      expect(documentId).toBe(encodeLiveReference(fields, readerOwner))
+      expect(documentId).toMatch(/^live2:google_drive~account~doc~[0-9a-f]{6}$/)
       const result = await read(documentId)
       expect(result.documentId).toBe(documentId)
       expect(result.chunks[0]?.content).toBe('Evidence')
@@ -1235,7 +1231,7 @@ describe('authorized live retrieval', () => {
           throw new NativeSearchError('reconnect', 'This search connection is no longer available.')
         return { account, accessToken: 'secret' }
       })
-      const theirs = encodeLiveReference({ ...fields, account: 'their-account' })
+      const theirs = encodeLiveReference({ ...fields, account: 'their-account' }, readerOwner)
       await expect(read(theirs)).rejects.toMatchObject({ code: 'unauthorized' })
       expect(mocks.resolveAccount).toHaveBeenCalledWith(
         expect.anything(),
@@ -1255,7 +1251,7 @@ describe('authorized live retrieval', () => {
     ])(
       'refuses an ID minted for %s as not found before resolving any account',
       async (_case, user, scope) => {
-        const theirs = encodeLiveReference({ ...fields, owner: liveReferenceOwner(user, scope) })
+        const theirs = encodeLiveReference(fields, liveReferenceOwner(user, scope))
         await expect(read(theirs)).rejects.toThrow('Document not found in this search scope')
         expect(mocks.resolveAccount).not.toHaveBeenCalled()
         expect(mocks.read).not.toHaveBeenCalled()
@@ -1289,7 +1285,7 @@ describe('authorized live retrieval', () => {
       }
     )
     it('matches a selected document across the earlier and current forms', async () => {
-      const current = encodeLiveReference(fields)
+      const current = encodeLiveReference(fields, readerOwner)
       const earlier = legacyId({ user: 'reader', scope: workspaceScope, ...fields })
       await expect(read(current, { filters: { documentIds: [earlier] } })).resolves.toMatchObject({
         documentId: current,
@@ -1300,7 +1296,7 @@ describe('authorized live retrieval', () => {
       ).toBe(true)
       await expect(
         read(current, {
-          filters: { documentIds: [encodeLiveReference({ ...fields, id: 'other' })] },
+          filters: { documentIds: [encodeLiveReference({ ...fields, id: 'other' }, readerOwner)] },
         })
       ).rejects.toThrow('outside the selected search filters')
     })

apps/sim/lib/sim-search/live/application.ts

@@ -76,11 +76,11 @@ import { LIVE_SEARCH_PROVIDER_IDS } from '@/lib/sim-search/live/provider-catalog
 import { liveSearchGuidance } from '@/lib/sim-search/live/providers'
 import { LiveReadError } from '@/lib/sim-search/live/read-error'
 import {
-  decodeLiveReference,
   encodeLiveReference,
   includesLiveDocument,
   liveDocumentKey,
   liveReferenceOwner,
+  readLiveReference,
 } from '@/lib/sim-search/live/reference'
 import { queryRelevance } from '@/lib/sim-search/live/text-match'
 import type { LiveAccount, NativeDocument } from '@/lib/sim-search/live/types'
@@ -204,17 +204,19 @@ function candidateFor(
 ): LiveCandidate {
   return {
     document,
-    documentId: encodeLiveReference({
-      owner,
-      account: account.id,
-      provider: account.provider,
-      id: document.id,
-      ...(document.container ? { container: document.container } : {}),
-      ...(document.kind ? { kind: document.kind } : {}),
-      ...(document.revision ? { revision: document.revision } : {}),
-      ...(document.connectionBinding ? { connectionBinding: document.connectionBinding } : {}),
-      ...(document.threadId ? { threadId: document.threadId } : {}),
-    }),
+    documentId: encodeLiveReference(
+      {
+        account: account.id,
+        provider: account.provider,
+        id: document.id,
+        ...(document.container ? { container: document.container } : {}),
+        ...(document.kind ? { kind: document.kind } : {}),
+        ...(document.revision ? { revision: document.revision } : {}),
+        ...(document.connectionBinding ? { connectionBinding: document.connectionBinding } : {}),
+        ...(document.threadId ? { threadId: document.threadId } : {}),
+      },
+      owner
+    ),
   }
 }
 
@@ -1039,12 +1041,10 @@ export const readLiveDocument = defineAuthorizedKnowledgeUseCase({
     if (input.organizationId) await requireOrganizationSearchAvailable(input.organizationId)
     if (!Number.isInteger(input.limit) || input.limit < 1 || input.limit > 8)
       throw new OrchestrationError('validation', 'Invalid document read limit')
-    const reference = decodeLiveReference(input.documentId)
-    if (
-      reference.owner !==
+    const reference = readLiveReference(
+      input.documentId,
       liveReferenceOwner(userId, resourceScopeKey(resourceScopeFromOwner(input)))
     )
-      throw new OrchestrationError('not_found', 'Document not found in this search scope')
     if (
       (input.filters?.source && input.filters.source !== reference.provider) ||
       (input.filters?.documentIds &&

apps/sim/lib/sim-search/live/reference.test.ts

@@ -1,22 +1,22 @@
 import { describe, expect, it } from 'vitest'
 import {
-  decodeLiveReference,
   encodeLiveReference,
   type LiveReference,
+  liveReferenceOwner,
+  readLiveReference,
 } from '@/lib/sim-search/live/reference'
 
-const owner = 'a1b2c3d4'
+const owner = liveReferenceOwner('reader', 'workspace:w')
 
 const thread: LiveReference = {
   provider: 'slack',
-  owner,
   account: 'account',
   id: '1730912345.123456',
   container: 'C1',
   threadId: '1730900000.000100',
 }
 
-const earlierForm = ({ owner: _owner, ...reference }: LiveReference) =>
+const earlierForm = (reference: LiveReference) =>
   `live:${Buffer.from(
     JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...reference })
   ).toString('base64url')}`
@@ -27,7 +27,21 @@ describe('live document IDs', () => {
     ['dropping the thread it names', (id: string) => id.replace(/~1730900000\.000100(?=~)/, '')],
     ['changing the message it names', (id: string) => id.replace('123456', '123457')],
   ])('asks for a new search after %s rather than naming another document', (_case, damage) => {
-    expect(() => decodeLiveReference(damage(encodeLiveReference(thread)))).toThrow('Search again')
+    expect(() => readLiveReference(damage(encodeLiveReference(thread, owner)), owner)).toThrow(
+      'Search again'
+    )
+  })
+
+  it.each([
+    ['another member', liveReferenceOwner('someone-else', 'workspace:w')],
+    ['another scope', liveReferenceOwner('reader', 'workspace:other')],
+  ])('refuses an ID read by %s as not found in this scope', (_case, reader) => {
+    expect(() => readLiveReference(encodeLiveReference(thread, owner), reader)).toThrow(
+      'Document not found in this search scope'
+    )
+    expect(() => readLiveReference(earlierForm(thread), reader)).toThrow(
+      'Document not found in this search scope'
+    )
   })
 
   it.each([
@@ -36,17 +50,20 @@ describe('live document IDs', () => {
     ['Unicode text', 'docs/設計/仕様書.md'],
     ['an unpaired surrogate', 'name\ud800end'],
   ])('reads back an ID holding %s', (_case, id) => {
-    const reference = { provider: 'gitlab', owner, account: 'account', id } as const
-    expect(decodeLiveReference(encodeLiveReference(reference))).toEqual(reference)
+    const reference = { provider: 'gitlab', account: 'account', id } as const
+    expect(readLiveReference(encodeLiveReference(reference, owner), owner)).toEqual(reference)
+  })
+
+  it('reads an earlier-form ID minted for the same reader and scope', () => {
+    expect(readLiveReference(earlierForm(thread), owner)).toEqual(thread)
   })
 
   it('is never longer than the earlier form for the same document', () => {
     const reference = {
       provider: 'gitlab',
-      owner,
       account: 'account',
       id: `src/${'設計書'.repeat(150)}.md`,
     } as const
-    expect(encodeLiveReference(reference).length).toBeLessThan(earlierForm(reference).length)
+    expect(encodeLiveReference(reference, owner).length).toBeLessThan(earlierForm(reference).length)
   })
 })