improvement(search): shorter live document IDs

apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts

@@ -25,18 +25,19 @@ import { generateId } from '@sim/utils/id'
 import { isPlainRecord } from '@sim/utils/object'
 import { and, eq, inArray } from 'drizzle-orm'
 import { afterAll, beforeAll, describe, expect, it } from 'vitest'
+import { resourceScopeFromOwner, resourceScopeKey } from '@/lib/core/resource-scope'
 import { seedKnowledgeAclFixture } from '@/lib/knowledge/__integration__/seed-source-access-fixture'
 import {
   createKnowledgeConnector,
   updateKnowledgeConnector,
 } from '@/lib/knowledge/application/connectors'
 import { refreshConnectorDirectory } from '@/lib/knowledge/connectors/external-group-sync'
+import { readLiveDocument, searchLiveKnowledge } from '@/lib/sim-search/live/application'
 import {
   decodeLiveReference,
   encodeLiveReference,
-  readLiveDocument,
-  searchLiveKnowledge,
-} from '@/lib/sim-search/live/application'
+  liveReferenceOwner,
+} from '@/lib/sim-search/live/reference'
 import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 
 interface FixturePerson {
@@ -133,7 +134,11 @@ describe.skipIf(!fixtureFile)('federated self-hosted GitLab Search', () => {
   }
 
   function documentForReader(documentId: string, person: FixturePerson) {
-    return encodeLiveReference({ ...decodeLiveReference(documentId), user: person.simId })
+    const scope = resourceScopeKey(resourceScopeFromOwner({ organizationId: ids.organizationId }))
+    return encodeLiveReference(
+      decodeLiveReference(documentId),
+      liveReferenceOwner(person.simId, scope)
+    )
   }
 
   beforeAll(async () => {

apps/sim/lib/knowledge/search/citation.ts

@@ -1,6 +1,12 @@
 import { sha256Hex } from '@sim/security/hash'
+import { liveDocumentKey } from '@/lib/sim-search/live/reference'
 
-/** Stable opaque citation IDs keep the model from rewriting long live references. */
+/**
+ * Stable opaque citation IDs keep the model from rewriting long live references. The hash covers
+ * the document's identity rather than the ID's spelling, so an earlier-form ID and a current one
+ * for the same document cite as one source. Saved chats keep the citation IDs their tool results
+ * stored, so changing what is hashed never strands an existing citation.
+ */
 export function liveCitationId(documentId: string): string {
-  return `live:${sha256Hex(documentId).slice(0, 32)}`
+  return `live:${sha256Hex(liveDocumentKey(documentId)).slice(0, 32)}`
 }

apps/sim/lib/knowledge/search/filters.test.ts

@@ -1,5 +1,20 @@
 import { describe, expect, it } from 'vitest'
+import { liveCitationId } from '@/lib/knowledge/search/citation'
 import { intersectWorkspaceSearchFilters } from '@/lib/knowledge/search/filters'
+import { encodeLiveReference, liveReferenceOwner } from '@/lib/sim-search/live/reference'
+
+const owner = liveReferenceOwner('reader', 'workspace:w')
+const fields = {
+  account: 'account',
+  provider: 'slack' as const,
+  id: '1730912345.123456',
+  container: 'C1',
+}
+/** One Slack message in the current ID form and in the earlier form older conversations hold. */
+const currentId = encodeLiveReference(fields, owner)
+const earlierId = `live:${Buffer.from(
+  JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...fields })
+).toString('base64url')}`
 
 describe('Assistant search scope', () => {
   it('keeps the narrower end of a date window and refuses an empty one', () => {
@@ -28,4 +43,22 @@ describe('Assistant search scope', () => {
       intersectWorkspaceSearchFilters({ documentIds: ['a'] }, { documentIds: ['b'] })
     ).toThrow('outside this search')
   })
+
+  it('keeps a requested document the user selected in the other ID form', () => {
+    expect(
+      intersectWorkspaceSearchFilters({ documentIds: [currentId] }, { documentIds: [earlierId] })
+    ).toEqual({ documentIds: [currentId] })
+    expect(
+      intersectWorkspaceSearchFilters({ documentIds: [earlierId] }, { documentIds: [currentId] })
+    ).toEqual({ documentIds: [earlierId] })
+  })
+})
+
+describe('live citation IDs', () => {
+  it('cite one document as one source whichever ID form named it', () => {
+    expect(liveCitationId(earlierId)).toBe(liveCitationId(currentId))
+    expect(liveCitationId(currentId)).not.toBe(
+      liveCitationId(encodeLiveReference({ ...fields, id: '1730912345.123457' }, owner))
+    )
+  })
 })

apps/sim/lib/knowledge/search/filters.ts

@@ -1,4 +1,5 @@
 import { OrchestrationError } from '@/lib/core/orchestration/types'
+import { includesLiveDocument } from '@/lib/sim-search/live/reference'
 
 /** Document constraints shared by Search and Assistant, applied before ranking. */
 export interface WorkspaceSearchFilters {
@@ -19,10 +20,11 @@ export function intersectWorkspaceSearchFilters(
   if (scope.source && requested.source && scope.source !== requested.source) {
     throw new OrchestrationError('validation', 'The requested source is outside this search')
   }
-  const documentIds = scope.documentIds
+  const scopeDocumentIds = scope.documentIds
+  const documentIds = scopeDocumentIds
     ? requested.documentIds
-      ? requested.documentIds.filter((id) => scope.documentIds!.includes(id))
-      : scope.documentIds
+      ? requested.documentIds.filter((id) => includesLiveDocument(scopeDocumentIds, id))
+      : scopeDocumentIds
     : requested.documentIds
   if (documentIds?.length === 0) {
     throw new OrchestrationError('validation', 'The requested document is outside this search')

apps/sim/lib/sim-search/live/application.test.ts

@@ -76,10 +76,9 @@ vi.mock('@/executor/utils/resolved-secret-content-projection', () => ({
   projectResolvedSecretModelContent: (value: string) => ({ safe: true, value }),
 }))
 
+import { resourceScopeFromOwner, resourceScopeKey } from '@/lib/core/resource-scope'
 import { withSearchDiagnostics } from '@/lib/knowledge/search/diagnostics'
 import {
-  decodeLiveReference,
-  encodeLiveReference,
   matchesLiveFilters,
   readLiveDocument,
   searchLiveKnowledge,
@@ -89,6 +88,11 @@ import { NativeSearchError } from '@/lib/sim-search/live/http'
 import { createPolicyVerifier } from '@/lib/sim-search/live/policy'
 import { defaultLiveSearchPolicy } from '@/lib/sim-search/live/policy-schema'
 import { livePolicyFor } from '@/lib/sim-search/live/policy-store'
+import {
+  decodeLiveReference,
+  encodeLiveReference,
+  liveReferenceOwner,
+} from '@/lib/sim-search/live/reference'
 import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 
 workspaceAuthzMockFns.mockPermissionSatisfies.mockImplementation(
@@ -1185,22 +1189,117 @@ describe('authorized live retrieval', () => {
       })
     ).rejects.toBe('user_stop:test')
   })
-  it('rejects cross-user document references before token resolution', async () => {
-    const search = await searchLiveKnowledge.execute({ principal, input })
-    const reference = decodeLiveReference(search.results[0].documentId)
-    mocks.resolveAccount.mockClear()
-    await expect(
+  describe('live document IDs', () => {
+    const workspaceScope = resourceScopeKey(resourceScopeFromOwner({ workspaceId: 'workspace' }))
+    /** The earlier wire form, spelled out so these tests pin what older conversations hold. */
+    const legacyId = (fields: Record<string, string>) =>
+      `live:${Buffer.from(JSON.stringify({ v: 1, ...fields })).toString('base64url')}`
+    const readerOwner = liveReferenceOwner('reader', workspaceScope)
+    const fields = { account: 'account', provider: 'google_drive' as const, id: 'doc' }
+    const read = (documentId: string, extra: Record<string, unknown> = {}, as = principal) =>
       readLiveDocument.execute({
-        principal,
+        principal: as,
         input: {
           workspaceId: 'workspace',
-          documentId: encodeLiveReference({ ...reference, user: 'someone-else' }),
+          documentId,
           limit: 3,
           resultSecretRegistry: new ResolvedSecretTraceRegistry([]),
+          ...extra,
         },
       })
-    ).rejects.toThrow('Document not found')
-    expect(mocks.resolveAccount).not.toHaveBeenCalled()
+
+    it('mints a compact ID naming no reader or scope that reads back', async () => {
+      const search = await searchLiveKnowledge.execute({ principal, input })
+      const [first] = search.results
+      if (!first) throw new Error('Expected a search result')
+      const { documentId } = first
+      expect(documentId).toBe(encodeLiveReference(fields, readerOwner))
+      expect(documentId).toMatch(/^live2:google_drive~account~doc~[0-9a-f]{6}$/)
+      const result = await read(documentId)
+      expect(result.documentId).toBe(documentId)
+      expect(result.chunks[0]?.content).toBe('Evidence')
+      expect(mocks.resolveAccount).toHaveBeenLastCalledWith(
+        expect.objectContaining({ workspaceId: 'workspace' }),
+        'reader',
+        'account',
+        expect.anything()
+      )
+    })
+    it("refuses another member's personal account through the reader's own account list", async () => {
+      mocks.resolveAccount.mockImplementation(async (_owner, _userId, accountId: string) => {
+        if (accountId !== 'account')
+          throw new NativeSearchError('reconnect', 'This search connection is no longer available.')
+        return { account, accessToken: 'secret' }
+      })
+      const theirs = encodeLiveReference({ ...fields, account: 'their-account' }, readerOwner)
+      await expect(read(theirs)).rejects.toMatchObject({ code: 'unauthorized' })
+      expect(mocks.resolveAccount).toHaveBeenCalledWith(
+        expect.anything(),
+        'reader',
+        'their-account',
+        expect.anything()
+      )
+      expect(mocks.read).not.toHaveBeenCalled()
+    })
+    it.each([
+      ['another member', 'other-reader', workspaceScope],
+      [
+        'another scope',
+        'reader',
+        resourceScopeKey(resourceScopeFromOwner({ workspaceId: 'other' })),
+      ],
+    ])(
+      'refuses an ID minted for %s as not found before resolving any account',
+      async (_case, user, scope) => {
+        const theirs = encodeLiveReference(fields, liveReferenceOwner(user, scope))
+        await expect(read(theirs)).rejects.toThrow('Document not found in this search scope')
+        expect(mocks.resolveAccount).not.toHaveBeenCalled()
+        expect(mocks.read).not.toHaveBeenCalled()
+      }
+    )
+    it.each([
+      ['a missing field', 'live2:google_drive~account'],
+      ['an unknown provider', 'live2:nowhere~account~doc'],
+      ['a broken escape', 'live2:google_drive~account~%E0%A4'],
+      ['too many fields', 'live2:google_drive~a~b~c~d~e~f~g~h'],
+      ['an unreadable earlier form', 'live:not-a-reference'],
+      ['no prefix', 'google_drive~account~doc'],
+    ])('asks the caller to search again for %s', async (_case, documentId) => {
+      await expect(read(documentId)).rejects.toThrow('Search again')
+      expect(mocks.resolveAccount).not.toHaveBeenCalled()
+    })
+    it('still reads an earlier-form ID minted for this reader and scope', async () => {
+      const result = await read(legacyId({ user: 'reader', scope: workspaceScope, ...fields }))
+      expect(result.chunks[0]?.content).toBe('Evidence')
+    })
+    it.each([
+      ['reader', { user: 'someone-else', scope: workspaceScope }],
+      ['scope', { user: 'reader', scope: 'workspace:elsewhere' }],
+    ])(
+      'rejects an earlier-form ID minted for another %s before token resolution',
+      async (_case, binding) => {
+        await expect(read(legacyId({ ...binding, ...fields }))).rejects.toThrow(
+          'Document not found'
+        )
+        expect(mocks.resolveAccount).not.toHaveBeenCalled()
+      }
+    )
+    it('matches a selected document across the earlier and current forms', async () => {
+      const current = encodeLiveReference(fields, readerOwner)
+      const earlier = legacyId({ user: 'reader', scope: workspaceScope, ...fields })
+      await expect(read(current, { filters: { documentIds: [earlier] } })).resolves.toMatchObject({
+        documentId: current,
+      })
+      await expect(read(earlier, { filters: { documentIds: [current] } })).resolves.toBeDefined()
+      expect(
+        matchesLiveFilters(document, current, 'google_drive', { documentIds: [earlier] })
+      ).toBe(true)
+      await expect(
+        read(current, {
+          filters: { documentIds: [encodeLiveReference({ ...fields, id: 'other' }, readerOwner)] },
+        })
+      ).rejects.toThrow('outside the selected search filters')
+    })
   })
   it('rejects a document moved out of scope while its content was being read', async () => {
     const search = await searchLiveKnowledge.execute({ principal, input })