Skip to content

Commit 40d7474

Browse files
authored
improvement(search): shorter live document IDs (#9794)
* improvement(search): shorter live document IDs Live search now mints live2: IDs: readable, ~-separated, percent-escaped provider fields with no reader or scope, about a quarter the length and tokens of the base64 JSON form. Reads derive the reader and scope from the request; account resolution and provider verification remain the authorization. Earlier live: IDs still read with their reader and scope checks. Document ID filters, Assistant scope intersection, and citation IDs compare by the decoded document identity, so both forms of one document match. * fix(search): refuse another reader's live ID as not found in this scope * fix(search): bind live IDs to their reader through the checksum, not a visible field
1 parent e999cf5 commit 40d7474

9 files changed

Lines changed: 495 additions & 84 deletions

File tree

‎apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,19 @@ import { generateId } from '@sim/utils/id'
2525
import { isPlainRecord } from '@sim/utils/object'
2626
import { and, eq, inArray } from 'drizzle-orm'
2727
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
28+
import { resourceScopeFromOwner, resourceScopeKey } from '@/lib/core/resource-scope'
2829
import { seedKnowledgeAclFixture } from '@/lib/knowledge/__integration__/seed-source-access-fixture'
2930
import {
3031
createKnowledgeConnector,
3132
updateKnowledgeConnector,
3233
} from '@/lib/knowledge/application/connectors'
3334
import { refreshConnectorDirectory } from '@/lib/knowledge/connectors/external-group-sync'
35+
import { readLiveDocument, searchLiveKnowledge } from '@/lib/sim-search/live/application'
3436
import {
3537
decodeLiveReference,
3638
encodeLiveReference,
37-
readLiveDocument,
38-
searchLiveKnowledge,
39-
} from '@/lib/sim-search/live/application'
39+
liveReferenceOwner,
40+
} from '@/lib/sim-search/live/reference'
4041
import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
4142

4243
interface FixturePerson {
@@ -133,7 +134,11 @@ describe.skipIf(!fixtureFile)('federated self-hosted GitLab Search', () => {
133134
}
134135

135136
function documentForReader(documentId: string, person: FixturePerson) {
136-
return encodeLiveReference({ ...decodeLiveReference(documentId), user: person.simId })
137+
const scope = resourceScopeKey(resourceScopeFromOwner({ organizationId: ids.organizationId }))
138+
return encodeLiveReference(
139+
decodeLiveReference(documentId),
140+
liveReferenceOwner(person.simId, scope)
141+
)
137142
}
138143

139144
beforeAll(async () => {
Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
import { sha256Hex } from '@sim/security/hash'
2+
import { liveDocumentKey } from '@/lib/sim-search/live/reference'
23

3-
/** Stable opaque citation IDs keep the model from rewriting long live references. */
4+
/**
5+
* Stable opaque citation IDs keep the model from rewriting long live references. The hash covers
6+
* the document's identity rather than the ID's spelling, so an earlier-form ID and a current one
7+
* for the same document cite as one source. Saved chats keep the citation IDs their tool results
8+
* stored, so changing what is hashed never strands an existing citation.
9+
*/
410
export function liveCitationId(documentId: string): string {
5-
return `live:${sha256Hex(documentId).slice(0, 32)}`
11+
return `live:${sha256Hex(liveDocumentKey(documentId)).slice(0, 32)}`
612
}

‎apps/sim/lib/knowledge/search/filters.test.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,20 @@
11
import { describe, expect, it } from 'vitest'
2+
import { liveCitationId } from '@/lib/knowledge/search/citation'
23
import { intersectWorkspaceSearchFilters } from '@/lib/knowledge/search/filters'
4+
import { encodeLiveReference, liveReferenceOwner } from '@/lib/sim-search/live/reference'
5+
6+
const owner = liveReferenceOwner('reader', 'workspace:w')
7+
const fields = {
8+
account: 'account',
9+
provider: 'slack' as const,
10+
id: '1730912345.123456',
11+
container: 'C1',
12+
}
13+
/** One Slack message in the current ID form and in the earlier form older conversations hold. */
14+
const currentId = encodeLiveReference(fields, owner)
15+
const earlierId = `live:${Buffer.from(
16+
JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...fields })
17+
).toString('base64url')}`
318

419
describe('Assistant search scope', () => {
520
it('keeps the narrower end of a date window and refuses an empty one', () => {
@@ -28,4 +43,22 @@ describe('Assistant search scope', () => {
2843
intersectWorkspaceSearchFilters({ documentIds: ['a'] }, { documentIds: ['b'] })
2944
).toThrow('outside this search')
3045
})
46+
47+
it('keeps a requested document the user selected in the other ID form', () => {
48+
expect(
49+
intersectWorkspaceSearchFilters({ documentIds: [currentId] }, { documentIds: [earlierId] })
50+
).toEqual({ documentIds: [currentId] })
51+
expect(
52+
intersectWorkspaceSearchFilters({ documentIds: [earlierId] }, { documentIds: [currentId] })
53+
).toEqual({ documentIds: [earlierId] })
54+
})
55+
})
56+
57+
describe('live citation IDs', () => {
58+
it('cite one document as one source whichever ID form named it', () => {
59+
expect(liveCitationId(earlierId)).toBe(liveCitationId(currentId))
60+
expect(liveCitationId(currentId)).not.toBe(
61+
liveCitationId(encodeLiveReference({ ...fields, id: '1730912345.123457' }, owner))
62+
)
63+
})
3164
})

‎apps/sim/lib/knowledge/search/filters.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { OrchestrationError } from '@/lib/core/orchestration/types'
2+
import { includesLiveDocument } from '@/lib/sim-search/live/reference'
23

34
/** Document constraints shared by Search and Assistant, applied before ranking. */
45
export interface WorkspaceSearchFilters {
@@ -19,10 +20,11 @@ export function intersectWorkspaceSearchFilters(
1920
if (scope.source && requested.source && scope.source !== requested.source) {
2021
throw new OrchestrationError('validation', 'The requested source is outside this search')
2122
}
22-
const documentIds = scope.documentIds
23+
const scopeDocumentIds = scope.documentIds
24+
const documentIds = scopeDocumentIds
2325
? requested.documentIds
24-
? requested.documentIds.filter((id) => scope.documentIds!.includes(id))
25-
: scope.documentIds
26+
? requested.documentIds.filter((id) => includesLiveDocument(scopeDocumentIds, id))
27+
: scopeDocumentIds
2628
: requested.documentIds
2729
if (documentIds?.length === 0) {
2830
throw new OrchestrationError('validation', 'The requested document is outside this search')

‎apps/sim/lib/sim-search/live/application.test.ts‎

Lines changed: 110 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,9 @@ vi.mock('@/executor/utils/resolved-secret-content-projection', () => ({
7676
projectResolvedSecretModelContent: (value: string) => ({ safe: true, value }),
7777
}))
7878

79+
import { resourceScopeFromOwner, resourceScopeKey } from '@/lib/core/resource-scope'
7980
import { withSearchDiagnostics } from '@/lib/knowledge/search/diagnostics'
8081
import {
81-
decodeLiveReference,
82-
encodeLiveReference,
8382
matchesLiveFilters,
8483
readLiveDocument,
8584
searchLiveKnowledge,
@@ -89,6 +88,11 @@ import { NativeSearchError } from '@/lib/sim-search/live/http'
8988
import { createPolicyVerifier } from '@/lib/sim-search/live/policy'
9089
import { defaultLiveSearchPolicy } from '@/lib/sim-search/live/policy-schema'
9190
import { livePolicyFor } from '@/lib/sim-search/live/policy-store'
91+
import {
92+
decodeLiveReference,
93+
encodeLiveReference,
94+
liveReferenceOwner,
95+
} from '@/lib/sim-search/live/reference'
9296
import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
9397

9498
workspaceAuthzMockFns.mockPermissionSatisfies.mockImplementation(
@@ -1185,22 +1189,117 @@ describe('authorized live retrieval', () => {
11851189
})
11861190
).rejects.toBe('user_stop:test')
11871191
})
1188-
it('rejects cross-user document references before token resolution', async () => {
1189-
const search = await searchLiveKnowledge.execute({ principal, input })
1190-
const reference = decodeLiveReference(search.results[0].documentId)
1191-
mocks.resolveAccount.mockClear()
1192-
await expect(
1192+
describe('live document IDs', () => {
1193+
const workspaceScope = resourceScopeKey(resourceScopeFromOwner({ workspaceId: 'workspace' }))
1194+
/** The earlier wire form, spelled out so these tests pin what older conversations hold. */
1195+
const legacyId = (fields: Record<string, string>) =>
1196+
`live:${Buffer.from(JSON.stringify({ v: 1, ...fields })).toString('base64url')}`
1197+
const readerOwner = liveReferenceOwner('reader', workspaceScope)
1198+
const fields = { account: 'account', provider: 'google_drive' as const, id: 'doc' }
1199+
const read = (documentId: string, extra: Record<string, unknown> = {}, as = principal) =>
11931200
readLiveDocument.execute({
1194-
principal,
1201+
principal: as,
11951202
input: {
11961203
workspaceId: 'workspace',
1197-
documentId: encodeLiveReference({ ...reference, user: 'someone-else' }),
1204+
documentId,
11981205
limit: 3,
11991206
resultSecretRegistry: new ResolvedSecretTraceRegistry([]),
1207+
...extra,
12001208
},
12011209
})
1202-
).rejects.toThrow('Document not found')
1203-
expect(mocks.resolveAccount).not.toHaveBeenCalled()
1210+
1211+
it('mints a compact ID naming no reader or scope that reads back', async () => {
1212+
const search = await searchLiveKnowledge.execute({ principal, input })
1213+
const [first] = search.results
1214+
if (!first) throw new Error('Expected a search result')
1215+
const { documentId } = first
1216+
expect(documentId).toBe(encodeLiveReference(fields, readerOwner))
1217+
expect(documentId).toMatch(/^live2:google_drive~account~doc~[0-9a-f]{6}$/)
1218+
const result = await read(documentId)
1219+
expect(result.documentId).toBe(documentId)
1220+
expect(result.chunks[0]?.content).toBe('Evidence')
1221+
expect(mocks.resolveAccount).toHaveBeenLastCalledWith(
1222+
expect.objectContaining({ workspaceId: 'workspace' }),
1223+
'reader',
1224+
'account',
1225+
expect.anything()
1226+
)
1227+
})
1228+
it("refuses another member's personal account through the reader's own account list", async () => {
1229+
mocks.resolveAccount.mockImplementation(async (_owner, _userId, accountId: string) => {
1230+
if (accountId !== 'account')
1231+
throw new NativeSearchError('reconnect', 'This search connection is no longer available.')
1232+
return { account, accessToken: 'secret' }
1233+
})
1234+
const theirs = encodeLiveReference({ ...fields, account: 'their-account' }, readerOwner)
1235+
await expect(read(theirs)).rejects.toMatchObject({ code: 'unauthorized' })
1236+
expect(mocks.resolveAccount).toHaveBeenCalledWith(
1237+
expect.anything(),
1238+
'reader',
1239+
'their-account',
1240+
expect.anything()
1241+
)
1242+
expect(mocks.read).not.toHaveBeenCalled()
1243+
})
1244+
it.each([
1245+
['another member', 'other-reader', workspaceScope],
1246+
[
1247+
'another scope',
1248+
'reader',
1249+
resourceScopeKey(resourceScopeFromOwner({ workspaceId: 'other' })),
1250+
],
1251+
])(
1252+
'refuses an ID minted for %s as not found before resolving any account',
1253+
async (_case, user, scope) => {
1254+
const theirs = encodeLiveReference(fields, liveReferenceOwner(user, scope))
1255+
await expect(read(theirs)).rejects.toThrow('Document not found in this search scope')
1256+
expect(mocks.resolveAccount).not.toHaveBeenCalled()
1257+
expect(mocks.read).not.toHaveBeenCalled()
1258+
}
1259+
)
1260+
it.each([
1261+
['a missing field', 'live2:google_drive~account'],
1262+
['an unknown provider', 'live2:nowhere~account~doc'],
1263+
['a broken escape', 'live2:google_drive~account~%E0%A4'],
1264+
['too many fields', 'live2:google_drive~a~b~c~d~e~f~g~h'],
1265+
['an unreadable earlier form', 'live:not-a-reference'],
1266+
['no prefix', 'google_drive~account~doc'],
1267+
])('asks the caller to search again for %s', async (_case, documentId) => {
1268+
await expect(read(documentId)).rejects.toThrow('Search again')
1269+
expect(mocks.resolveAccount).not.toHaveBeenCalled()
1270+
})
1271+
it('still reads an earlier-form ID minted for this reader and scope', async () => {
1272+
const result = await read(legacyId({ user: 'reader', scope: workspaceScope, ...fields }))
1273+
expect(result.chunks[0]?.content).toBe('Evidence')
1274+
})
1275+
it.each([
1276+
['reader', { user: 'someone-else', scope: workspaceScope }],
1277+
['scope', { user: 'reader', scope: 'workspace:elsewhere' }],
1278+
])(
1279+
'rejects an earlier-form ID minted for another %s before token resolution',
1280+
async (_case, binding) => {
1281+
await expect(read(legacyId({ ...binding, ...fields }))).rejects.toThrow(
1282+
'Document not found'
1283+
)
1284+
expect(mocks.resolveAccount).not.toHaveBeenCalled()
1285+
}
1286+
)
1287+
it('matches a selected document across the earlier and current forms', async () => {
1288+
const current = encodeLiveReference(fields, readerOwner)
1289+
const earlier = legacyId({ user: 'reader', scope: workspaceScope, ...fields })
1290+
await expect(read(current, { filters: { documentIds: [earlier] } })).resolves.toMatchObject({
1291+
documentId: current,
1292+
})
1293+
await expect(read(earlier, { filters: { documentIds: [current] } })).resolves.toBeDefined()
1294+
expect(
1295+
matchesLiveFilters(document, current, 'google_drive', { documentIds: [earlier] })
1296+
).toBe(true)
1297+
await expect(
1298+
read(current, {
1299+
filters: { documentIds: [encodeLiveReference({ ...fields, id: 'other' }, readerOwner)] },
1300+
})
1301+
).rejects.toThrow('outside the selected search filters')
1302+
})
12041303
})
12051304
it('rejects a document moved out of scope while its content was being read', async () => {
12061305
const search = await searchLiveKnowledge.execute({ principal, input })

0 commit comments

Comments
 (0)