improvement(search): shorter live document IDs

Live search now mints live2: IDs: readable, ~-separated, percent-escaped provider fields with no reader or scope, about a quarter the length and tokens of the base64 JSON form. Reads derive the reader and scope from the request; account resolution and provider verification remain the authorization. Earlier live: IDs still read with their reader and scope checks. Document ID filters, Assistant scope intersection, and citation IDs compare by the decoded document identity, so both forms of one document match.

Author: waleedlatif1

apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts

@@ -31,12 +31,8 @@ import {
   updateKnowledgeConnector,
 } from '@/lib/knowledge/application/connectors'
 import { refreshConnectorDirectory } from '@/lib/knowledge/connectors/external-group-sync'
-import {
-  decodeLiveReference,
-  encodeLiveReference,
-  readLiveDocument,
-  searchLiveKnowledge,
-} from '@/lib/sim-search/live/application'
+import { readLiveDocument, searchLiveKnowledge } from '@/lib/sim-search/live/application'
+import { decodeLiveReference } from '@/lib/sim-search/live/reference'
 import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 
 interface FixturePerson {
@@ -132,10 +128,6 @@ describe.skipIf(!fixtureFile)('federated self-hosted GitLab Search', () => {
     })
   }
 
-  function documentForReader(documentId: string, person: FixturePerson) {
-    return encodeLiveReference({ ...decodeLiveReference(documentId), user: person.simId })
-  }
-
   beforeAll(async () => {
     const fixture: unknown = JSON.parse(await readFile(fixtureFile!, 'utf8'))
     if (
@@ -318,13 +310,11 @@ describe.skipIf(!fixtureFile)('federated self-hosted GitLab Search', () => {
     120_000
   )
 
-  it('keeps references reader-bound and rechecks confidentiality before reads', async () => {
+  it('checks a shared source reference against its reader and rechecks confidentiality', async () => {
     const reporter = await search(people.reporter!, 'code')
     const file = reporter.results[0]!
+    /** The ID names no reader: the guest's own GitLab access refuses the reporter's code result. */
     await expect(read(people.guest!, file.documentId)).rejects.toMatchObject({ code: 'not_found' })
-    await expect(
-      read(people.guest!, documentForReader(file.documentId, people.guest!))
-    ).rejects.toBeDefined()
     const guest = await search(people.guest!, 'issues')
     const normal = guest.results.find(
       (row) => decodeLiveReference(row.documentId).id === String(issueIid)

apps/sim/lib/knowledge/mcp/server.test.ts

@@ -55,6 +55,7 @@ vi.mock('@/lib/knowledge/application/chat', () => ({
 
 import { OrchestrationError } from '@/lib/core/orchestration/types'
 import { createKnowledgeMcpServer } from '@/lib/knowledge/mcp/server'
+import { encodeLiveReference } from '@/lib/sim-search/live/reference'
 import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 
 const mocks = hoisted
@@ -301,6 +302,20 @@ describe('batched document reads', () => {
     expect(mocks.liveRead).toHaveBeenCalledTimes(2)
   })
 
+  it('reads a document once when the batch names it in both ID forms', async () => {
+    create()
+    mocks.liveRead.mockImplementation(async ({ input }: { input: { documentId: string } }) =>
+      readResult(input.documentId)
+    )
+    const reference = { provider: 'google_drive', account: 'account', id: 'doc' } as const
+    const earlier = `live:${Buffer.from(
+      JSON.stringify({ v: 1, user: 'person-1', scope: 'organization:org-1', ...reference })
+    ).toString('base64url')}`
+    await call('read_document', { documentIds: [earlier, encodeLiveReference(reference)] })
+    expect(mocks.rateLimit).toHaveBeenCalledWith(request, auth, expect.anything(), 1)
+    expect(mocks.liveRead).toHaveBeenCalledTimes(1)
+  })
+
   it('reads at most five documents from providers at once, in request order', async () => {
     create()
     let inFlight = 0

apps/sim/lib/knowledge/mcp/server.ts

@@ -30,6 +30,7 @@ import { liveCitationId } from '@/lib/knowledge/search/citation'
 import { toolError } from '@/lib/mcp/tool-result'
 import { readLiveDocument, searchLiveKnowledge } from '@/lib/sim-search/live/application'
 import { LiveReadError } from '@/lib/sim-search/live/read-error'
+import { liveDocumentKey } from '@/lib/sim-search/live/reference'
 import { v2CaughtOrchestrationError } from '@/app/api/v2/lib/response'
 import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 
@@ -225,7 +226,9 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe
     async (raw: unknown, extra: { signal: AbortSignal }) => {
       const parsed = readLiveDocumentMcpSchema.safeParse(raw)
       const batch =
-        parsed.success && parsed.data.documentIds ? [...new Set(parsed.data.documentIds)] : []
+        parsed.success && parsed.data.documentIds
+          ? [...new Map(parsed.data.documentIds.map((id) => [liveDocumentKey(id), id])).values()]
+          : []
       return execute(
         'read_document',
         knowledgeOperations.readDocument,

apps/sim/lib/knowledge/search/citation.ts

@@ -1,6 +1,12 @@
 import { sha256Hex } from '@sim/security/hash'
+import { liveDocumentKey } from '@/lib/sim-search/live/reference'
 
-/** Stable opaque citation IDs keep the model from rewriting long live references. */
+/**
+ * Stable opaque citation IDs keep the model from rewriting long live references. The hash covers
+ * the document's identity rather than the ID's spelling, so an earlier-form ID and a current one
+ * for the same document cite as one source. Saved chats keep the citation IDs their tool results
+ * stored, so changing what is hashed never strands an existing citation.
+ */
 export function liveCitationId(documentId: string): string {
-  return `live:${sha256Hex(documentId).slice(0, 32)}`
+  return `live:${sha256Hex(liveDocumentKey(documentId)).slice(0, 32)}`
 }

apps/sim/lib/knowledge/search/filters.test.ts

@@ -1,5 +1,19 @@
 import { describe, expect, it } from 'vitest'
+import { liveCitationId } from '@/lib/knowledge/search/citation'
 import { intersectWorkspaceSearchFilters } from '@/lib/knowledge/search/filters'
+import { encodeLiveReference } from '@/lib/sim-search/live/reference'
+
+const fields = {
+  account: 'account',
+  provider: 'slack' as const,
+  id: '1730912345.123456',
+  container: 'C1',
+}
+/** One Slack message in the current ID form and in the earlier form older conversations hold. */
+const currentId = encodeLiveReference(fields)
+const earlierId = `live:${Buffer.from(
+  JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...fields })
+).toString('base64url')}`
 
 describe('Assistant search scope', () => {
   it('keeps the narrower end of a date window and refuses an empty one', () => {
@@ -28,4 +42,22 @@ describe('Assistant search scope', () => {
       intersectWorkspaceSearchFilters({ documentIds: ['a'] }, { documentIds: ['b'] })
     ).toThrow('outside this search')
   })
+
+  it('keeps a requested document the user selected in the other ID form', () => {
+    expect(
+      intersectWorkspaceSearchFilters({ documentIds: [currentId] }, { documentIds: [earlierId] })
+    ).toEqual({ documentIds: [currentId] })
+    expect(
+      intersectWorkspaceSearchFilters({ documentIds: [earlierId] }, { documentIds: [currentId] })
+    ).toEqual({ documentIds: [earlierId] })
+  })
+})
+
+describe('live citation IDs', () => {
+  it('cite one document as one source whichever ID form named it', () => {
+    expect(liveCitationId(earlierId)).toBe(liveCitationId(currentId))
+    expect(liveCitationId(currentId)).not.toBe(
+      liveCitationId(encodeLiveReference({ ...fields, id: '1730912345.123457' }))
+    )
+  })
 })

apps/sim/lib/knowledge/search/filters.ts

@@ -1,4 +1,5 @@
 import { OrchestrationError } from '@/lib/core/orchestration/types'
+import { includesLiveDocument } from '@/lib/sim-search/live/reference'
 
 /** Document constraints shared by Search and Assistant, applied before ranking. */
 export interface WorkspaceSearchFilters {
@@ -19,10 +20,11 @@ export function intersectWorkspaceSearchFilters(
   if (scope.source && requested.source && scope.source !== requested.source) {
     throw new OrchestrationError('validation', 'The requested source is outside this search')
   }
-  const documentIds = scope.documentIds
+  const scopeDocumentIds = scope.documentIds
+  const documentIds = scopeDocumentIds
     ? requested.documentIds
-      ? requested.documentIds.filter((id) => scope.documentIds!.includes(id))
-      : scope.documentIds
+      ? requested.documentIds.filter((id) => includesLiveDocument(scopeDocumentIds, id))
+      : scopeDocumentIds
     : requested.documentIds
   if (documentIds?.length === 0) {
     throw new OrchestrationError('validation', 'The requested document is outside this search')

apps/sim/lib/sim-search/live/application.test.ts

@@ -76,10 +76,9 @@ vi.mock('@/executor/utils/resolved-secret-content-projection', () => ({
   projectResolvedSecretModelContent: (value: string) => ({ safe: true, value }),
 }))
 
+import { resourceScopeFromOwner, resourceScopeKey } from '@/lib/core/resource-scope'
 import { withSearchDiagnostics } from '@/lib/knowledge/search/diagnostics'
 import {
-  decodeLiveReference,
-  encodeLiveReference,
   matchesLiveFilters,
   readLiveDocument,
   searchLiveKnowledge,
@@ -89,6 +88,7 @@ import { NativeSearchError } from '@/lib/sim-search/live/http'
 import { createPolicyVerifier } from '@/lib/sim-search/live/policy'
 import { defaultLiveSearchPolicy } from '@/lib/sim-search/live/policy-schema'
 import { livePolicyFor } from '@/lib/sim-search/live/policy-store'
+import { decodeLiveReference, encodeLiveReference } from '@/lib/sim-search/live/reference'
 import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 
 workspaceAuthzMockFns.mockPermissionSatisfies.mockImplementation(
@@ -1185,22 +1185,110 @@ describe('authorized live retrieval', () => {
       })
     ).rejects.toBe('user_stop:test')
   })
-  it('rejects cross-user document references before token resolution', async () => {
-    const search = await searchLiveKnowledge.execute({ principal, input })
-    const reference = decodeLiveReference(search.results[0].documentId)
-    mocks.resolveAccount.mockClear()
-    await expect(
+  describe('live document IDs', () => {
+    const workspaceScope = resourceScopeKey(resourceScopeFromOwner({ workspaceId: 'workspace' }))
+    /** The earlier wire form, spelled out so these tests pin what older conversations hold. */
+    const legacyId = (fields: Record<string, string>) =>
+      `live:${Buffer.from(JSON.stringify({ v: 1, ...fields })).toString('base64url')}`
+    const fields = { account: 'account', provider: 'google_drive' as const, id: 'doc' }
+    const read = (documentId: string, extra: Record<string, unknown> = {}, as = principal) =>
       readLiveDocument.execute({
-        principal,
+        principal: as,
         input: {
           workspaceId: 'workspace',
-          documentId: encodeLiveReference({ ...reference, user: 'someone-else' }),
+          documentId,
           limit: 3,
           resultSecretRegistry: new ResolvedSecretTraceRegistry([]),
+          ...extra,
         },
       })
-    ).rejects.toThrow('Document not found')
-    expect(mocks.resolveAccount).not.toHaveBeenCalled()
+
+    it('mints a compact ID naming no reader or scope that reads back', async () => {
+      const search = await searchLiveKnowledge.execute({ principal, input })
+      const documentId = search.results[0]!.documentId
+      expect(documentId).toMatch(/^live2:google_drive~account~doc~[0-9a-f]{6}$/)
+      const result = await read(documentId)
+      expect(result.documentId).toBe(documentId)
+      expect(result.chunks[0]?.content).toBe('Evidence')
+      expect(mocks.resolveAccount).toHaveBeenLastCalledWith(
+        expect.objectContaining({ workspaceId: 'workspace' }),
+        'reader',
+        'account',
+        expect.anything()
+      )
+    })
+    it("refuses another member's personal account through the reader's own account list", async () => {
+      mocks.resolveAccount.mockImplementation(async (_owner, _userId, accountId: string) => {
+        if (accountId !== 'account')
+          throw new NativeSearchError('reconnect', 'This search connection is no longer available.')
+        return { account, accessToken: 'secret' }
+      })
+      const theirs = encodeLiveReference({ ...fields, account: 'their-account' })
+      await expect(read(theirs)).rejects.toMatchObject({ code: 'unauthorized' })
+      expect(mocks.resolveAccount).toHaveBeenCalledWith(
+        expect.anything(),
+        'reader',
+        'their-account',
+        expect.anything()
+      )
+      expect(mocks.read).not.toHaveBeenCalled()
+    })
+    it("gates a shared account's ID by the reader's own verification", async () => {
+      const search = await searchLiveKnowledge.execute({ principal, input })
+      mocks.service.mockResolvedValue({
+        policy: defaultLiveSearchPolicy(),
+        partial: false,
+        verify: async () => false,
+      })
+      const other = createSessionPrincipal({ userId: 'other-reader', sessionId: 's2' })
+      await expect(read(search.results[0]!.documentId, {}, other)).rejects.toThrow(
+        'outside your organization'
+      )
+      expect(mocks.read).not.toHaveBeenCalled()
+    })
+    it.each([
+      ['a missing field', 'live2:google_drive~account'],
+      ['an unknown provider', 'live2:nowhere~account~doc'],
+      ['a broken escape', 'live2:google_drive~account~%E0%A4'],
+      ['too many fields', 'live2:google_drive~a~b~c~d~e~f~g~h'],
+      ['an unreadable earlier form', 'live:not-a-reference'],
+      ['no prefix', 'google_drive~account~doc'],
+    ])('asks the caller to search again for %s', async (_case, documentId) => {
+      await expect(read(documentId)).rejects.toThrow('Search again')
+      expect(mocks.resolveAccount).not.toHaveBeenCalled()
+    })
+    it('still reads an earlier-form ID minted for this reader and scope', async () => {
+      const result = await read(legacyId({ user: 'reader', scope: workspaceScope, ...fields }))
+      expect(result.chunks[0]?.content).toBe('Evidence')
+    })
+    it.each([
+      ['reader', { user: 'someone-else', scope: workspaceScope }],
+      ['scope', { user: 'reader', scope: 'workspace:elsewhere' }],
+    ])(
+      'rejects an earlier-form ID minted for another %s before token resolution',
+      async (_case, binding) => {
+        await expect(read(legacyId({ ...binding, ...fields }))).rejects.toThrow(
+          'Document not found'
+        )
+        expect(mocks.resolveAccount).not.toHaveBeenCalled()
+      }
+    )
+    it('matches a selected document across the earlier and current forms', async () => {
+      const current = encodeLiveReference(fields)
+      const earlier = legacyId({ user: 'reader', scope: workspaceScope, ...fields })
+      await expect(read(current, { filters: { documentIds: [earlier] } })).resolves.toMatchObject({
+        documentId: current,
+      })
+      await expect(read(earlier, { filters: { documentIds: [current] } })).resolves.toBeDefined()
+      expect(
+        matchesLiveFilters(document, current, 'google_drive', { documentIds: [earlier] })
+      ).toBe(true)
+      await expect(
+        read(current, {
+          filters: { documentIds: [encodeLiveReference({ ...fields, id: 'other' })] },
+        })
+      ).rejects.toThrow('outside the selected search filters')
+    })
   })
   it('rejects a document moved out of scope while its content was being read', async () => {
     const search = await searchLiveKnowledge.execute({ principal, input })