From 40215463abb16dda04fb1d77878b126c4933a95e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 7 Nov 2026 12:39:32 -0800 Subject: [PATCH 1/3] improvement(search): shorter live document IDs Live search now mints live2: IDs: readable, ~-separated, percent-escaped provider fields with no reader or scope, about a quarter the length and tokens of the base64 JSON form. Reads derive the reader and scope from the request; account resolution and provider verification remain the authorization. Earlier live: IDs still read with their reader and scope checks. Document ID filters, Assistant scope intersection, and citation IDs compare by the decoded document identity, so both forms of one document match. --- .../gitlab-federated.integration.ts | 18 +- apps/sim/lib/knowledge/mcp/server.test.ts | 15 ++ apps/sim/lib/knowledge/mcp/server.ts | 5 +- apps/sim/lib/knowledge/search/citation.ts | 10 +- apps/sim/lib/knowledge/search/filters.test.ts | 32 +++ apps/sim/lib/knowledge/search/filters.ts | 8 +- .../lib/sim-search/live/application.test.ts | 110 +++++++++- apps/sim/lib/sim-search/live/application.ts | 72 ++----- .../sim/lib/sim-search/live/reference.test.ts | 48 +++++ apps/sim/lib/sim-search/live/reference.ts | 193 ++++++++++++++++++ 10 files changed, 425 insertions(+), 86 deletions(-) create mode 100644 apps/sim/lib/sim-search/live/reference.test.ts create mode 100644 apps/sim/lib/sim-search/live/reference.ts diff --git a/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts b/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts index b217bc67d93..e100fb7d931 100644 --- a/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts @@ -31,12 +31,8 @@ import { updateKnowledgeConnector, } from '@/lib/knowledge/application/connectors' import { refreshConnectorDirectory } from '@/lib/knowledge/connectors/external-group-sync' -import { - decodeLiveReference, - encodeLiveReference, - readLiveDocument, - searchLiveKnowledge, -} from '@/lib/sim-search/live/application' +import { readLiveDocument, searchLiveKnowledge } from '@/lib/sim-search/live/application' +import { decodeLiveReference } from '@/lib/sim-search/live/reference' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' interface FixturePerson { @@ -132,10 +128,6 @@ describe.skipIf(!fixtureFile)('federated self-hosted GitLab Search', () => { }) } - function documentForReader(documentId: string, person: FixturePerson) { - return encodeLiveReference({ ...decodeLiveReference(documentId), user: person.simId }) - } - beforeAll(async () => { const fixture: unknown = JSON.parse(await readFile(fixtureFile!, 'utf8')) if ( @@ -318,13 +310,11 @@ describe.skipIf(!fixtureFile)('federated self-hosted GitLab Search', () => { 120_000 ) - it('keeps references reader-bound and rechecks confidentiality before reads', async () => { + it('checks a shared source reference against its reader and rechecks confidentiality', async () => { const reporter = await search(people.reporter!, 'code') const file = reporter.results[0]! + /** The ID names no reader: the guest's own GitLab access refuses the reporter's code result. */ await expect(read(people.guest!, file.documentId)).rejects.toMatchObject({ code: 'not_found' }) - await expect( - read(people.guest!, documentForReader(file.documentId, people.guest!)) - ).rejects.toBeDefined() const guest = await search(people.guest!, 'issues') const normal = guest.results.find( (row) => decodeLiveReference(row.documentId).id === String(issueIid) diff --git a/apps/sim/lib/knowledge/mcp/server.test.ts b/apps/sim/lib/knowledge/mcp/server.test.ts index 579a6aa317a..2ab34f4cff4 100644 --- a/apps/sim/lib/knowledge/mcp/server.test.ts +++ b/apps/sim/lib/knowledge/mcp/server.test.ts @@ -55,6 +55,7 @@ vi.mock('@/lib/knowledge/application/chat', () => ({ import { OrchestrationError } from '@/lib/core/orchestration/types' import { createKnowledgeMcpServer } from '@/lib/knowledge/mcp/server' +import { encodeLiveReference } from '@/lib/sim-search/live/reference' import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const mocks = hoisted @@ -301,6 +302,20 @@ describe('batched document reads', () => { expect(mocks.liveRead).toHaveBeenCalledTimes(2) }) + it('reads a document once when the batch names it in both ID forms', async () => { + create() + mocks.liveRead.mockImplementation(async ({ input }: { input: { documentId: string } }) => + readResult(input.documentId) + ) + const reference = { provider: 'google_drive', account: 'account', id: 'doc' } as const + const earlier = `live:${Buffer.from( + JSON.stringify({ v: 1, user: 'person-1', scope: 'organization:org-1', ...reference }) + ).toString('base64url')}` + await call('read_document', { documentIds: [earlier, encodeLiveReference(reference)] }) + expect(mocks.rateLimit).toHaveBeenCalledWith(request, auth, expect.anything(), 1) + expect(mocks.liveRead).toHaveBeenCalledTimes(1) + }) + it('reads at most five documents from providers at once, in request order', async () => { create() let inFlight = 0 diff --git a/apps/sim/lib/knowledge/mcp/server.ts b/apps/sim/lib/knowledge/mcp/server.ts index b965738afc9..e7e9899ed09 100644 --- a/apps/sim/lib/knowledge/mcp/server.ts +++ b/apps/sim/lib/knowledge/mcp/server.ts @@ -30,6 +30,7 @@ import { liveCitationId } from '@/lib/knowledge/search/citation' import { toolError } from '@/lib/mcp/tool-result' import { readLiveDocument, searchLiveKnowledge } from '@/lib/sim-search/live/application' import { LiveReadError } from '@/lib/sim-search/live/read-error' +import { liveDocumentKey } from '@/lib/sim-search/live/reference' import { v2CaughtOrchestrationError } from '@/app/api/v2/lib/response' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' @@ -225,7 +226,9 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe async (raw: unknown, extra: { signal: AbortSignal }) => { const parsed = readLiveDocumentMcpSchema.safeParse(raw) const batch = - parsed.success && parsed.data.documentIds ? [...new Set(parsed.data.documentIds)] : [] + parsed.success && parsed.data.documentIds + ? [...new Map(parsed.data.documentIds.map((id) => [liveDocumentKey(id), id])).values()] + : [] return execute( 'read_document', knowledgeOperations.readDocument, diff --git a/apps/sim/lib/knowledge/search/citation.ts b/apps/sim/lib/knowledge/search/citation.ts index c6eb6489991..e17bb6f8cf4 100644 --- a/apps/sim/lib/knowledge/search/citation.ts +++ b/apps/sim/lib/knowledge/search/citation.ts @@ -1,6 +1,12 @@ import { sha256Hex } from '@sim/security/hash' +import { liveDocumentKey } from '@/lib/sim-search/live/reference' -/** Stable opaque citation IDs keep the model from rewriting long live references. */ +/** + * Stable opaque citation IDs keep the model from rewriting long live references. The hash covers + * the document's identity rather than the ID's spelling, so an earlier-form ID and a current one + * for the same document cite as one source. Saved chats keep the citation IDs their tool results + * stored, so changing what is hashed never strands an existing citation. + */ export function liveCitationId(documentId: string): string { - return `live:${sha256Hex(documentId).slice(0, 32)}` + return `live:${sha256Hex(liveDocumentKey(documentId)).slice(0, 32)}` } diff --git a/apps/sim/lib/knowledge/search/filters.test.ts b/apps/sim/lib/knowledge/search/filters.test.ts index d4fb58dc48b..70c13589ec8 100644 --- a/apps/sim/lib/knowledge/search/filters.test.ts +++ b/apps/sim/lib/knowledge/search/filters.test.ts @@ -1,5 +1,19 @@ import { describe, expect, it } from 'vitest' +import { liveCitationId } from '@/lib/knowledge/search/citation' import { intersectWorkspaceSearchFilters } from '@/lib/knowledge/search/filters' +import { encodeLiveReference } from '@/lib/sim-search/live/reference' + +const fields = { + account: 'account', + provider: 'slack' as const, + id: '1730912345.123456', + container: 'C1', +} +/** One Slack message in the current ID form and in the earlier form older conversations hold. */ +const currentId = encodeLiveReference(fields) +const earlierId = `live:${Buffer.from( + JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...fields }) +).toString('base64url')}` describe('Assistant search scope', () => { it('keeps the narrower end of a date window and refuses an empty one', () => { @@ -28,4 +42,22 @@ describe('Assistant search scope', () => { intersectWorkspaceSearchFilters({ documentIds: ['a'] }, { documentIds: ['b'] }) ).toThrow('outside this search') }) + + it('keeps a requested document the user selected in the other ID form', () => { + expect( + intersectWorkspaceSearchFilters({ documentIds: [currentId] }, { documentIds: [earlierId] }) + ).toEqual({ documentIds: [currentId] }) + expect( + intersectWorkspaceSearchFilters({ documentIds: [earlierId] }, { documentIds: [currentId] }) + ).toEqual({ documentIds: [earlierId] }) + }) +}) + +describe('live citation IDs', () => { + it('cite one document as one source whichever ID form named it', () => { + expect(liveCitationId(earlierId)).toBe(liveCitationId(currentId)) + expect(liveCitationId(currentId)).not.toBe( + liveCitationId(encodeLiveReference({ ...fields, id: '1730912345.123457' })) + ) + }) }) diff --git a/apps/sim/lib/knowledge/search/filters.ts b/apps/sim/lib/knowledge/search/filters.ts index 9467df8a518..1d6e2abb6c3 100644 --- a/apps/sim/lib/knowledge/search/filters.ts +++ b/apps/sim/lib/knowledge/search/filters.ts @@ -1,4 +1,5 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' +import { includesLiveDocument } from '@/lib/sim-search/live/reference' /** Document constraints shared by Search and Assistant, applied before ranking. */ export interface WorkspaceSearchFilters { @@ -19,10 +20,11 @@ export function intersectWorkspaceSearchFilters( if (scope.source && requested.source && scope.source !== requested.source) { throw new OrchestrationError('validation', 'The requested source is outside this search') } - const documentIds = scope.documentIds + const scopeDocumentIds = scope.documentIds + const documentIds = scopeDocumentIds ? requested.documentIds - ? requested.documentIds.filter((id) => scope.documentIds!.includes(id)) - : scope.documentIds + ? requested.documentIds.filter((id) => includesLiveDocument(scopeDocumentIds, id)) + : scopeDocumentIds : requested.documentIds if (documentIds?.length === 0) { throw new OrchestrationError('validation', 'The requested document is outside this search') diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index 5bed7b39f05..f5855b2c37b 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -76,10 +76,9 @@ vi.mock('@/executor/utils/resolved-secret-content-projection', () => ({ projectResolvedSecretModelContent: (value: string) => ({ safe: true, value }), })) +import { resourceScopeFromOwner, resourceScopeKey } from '@/lib/core/resource-scope' import { withSearchDiagnostics } from '@/lib/knowledge/search/diagnostics' import { - decodeLiveReference, - encodeLiveReference, matchesLiveFilters, readLiveDocument, searchLiveKnowledge, @@ -89,6 +88,7 @@ import { NativeSearchError } from '@/lib/sim-search/live/http' import { createPolicyVerifier } from '@/lib/sim-search/live/policy' import { defaultLiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' import { livePolicyFor } from '@/lib/sim-search/live/policy-store' +import { decodeLiveReference, encodeLiveReference } from '@/lib/sim-search/live/reference' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' workspaceAuthzMockFns.mockPermissionSatisfies.mockImplementation( @@ -1185,22 +1185,110 @@ describe('authorized live retrieval', () => { }) ).rejects.toBe('user_stop:test') }) - it('rejects cross-user document references before token resolution', async () => { - const search = await searchLiveKnowledge.execute({ principal, input }) - const reference = decodeLiveReference(search.results[0].documentId) - mocks.resolveAccount.mockClear() - await expect( + describe('live document IDs', () => { + const workspaceScope = resourceScopeKey(resourceScopeFromOwner({ workspaceId: 'workspace' })) + /** The earlier wire form, spelled out so these tests pin what older conversations hold. */ + const legacyId = (fields: Record) => + `live:${Buffer.from(JSON.stringify({ v: 1, ...fields })).toString('base64url')}` + const fields = { account: 'account', provider: 'google_drive' as const, id: 'doc' } + const read = (documentId: string, extra: Record = {}, as = principal) => readLiveDocument.execute({ - principal, + principal: as, input: { workspaceId: 'workspace', - documentId: encodeLiveReference({ ...reference, user: 'someone-else' }), + documentId, limit: 3, resultSecretRegistry: new ResolvedSecretTraceRegistry([]), + ...extra, }, }) - ).rejects.toThrow('Document not found') - expect(mocks.resolveAccount).not.toHaveBeenCalled() + + it('mints a compact ID naming no reader or scope that reads back', async () => { + const search = await searchLiveKnowledge.execute({ principal, input }) + const documentId = search.results[0]!.documentId + expect(documentId).toMatch(/^live2:google_drive~account~doc~[0-9a-f]{6}$/) + const result = await read(documentId) + expect(result.documentId).toBe(documentId) + expect(result.chunks[0]?.content).toBe('Evidence') + expect(mocks.resolveAccount).toHaveBeenLastCalledWith( + expect.objectContaining({ workspaceId: 'workspace' }), + 'reader', + 'account', + expect.anything() + ) + }) + it("refuses another member's personal account through the reader's own account list", async () => { + mocks.resolveAccount.mockImplementation(async (_owner, _userId, accountId: string) => { + if (accountId !== 'account') + throw new NativeSearchError('reconnect', 'This search connection is no longer available.') + return { account, accessToken: 'secret' } + }) + const theirs = encodeLiveReference({ ...fields, account: 'their-account' }) + await expect(read(theirs)).rejects.toMatchObject({ code: 'unauthorized' }) + expect(mocks.resolveAccount).toHaveBeenCalledWith( + expect.anything(), + 'reader', + 'their-account', + expect.anything() + ) + expect(mocks.read).not.toHaveBeenCalled() + }) + it("gates a shared account's ID by the reader's own verification", async () => { + const search = await searchLiveKnowledge.execute({ principal, input }) + mocks.service.mockResolvedValue({ + policy: defaultLiveSearchPolicy(), + partial: false, + verify: async () => false, + }) + const other = createSessionPrincipal({ userId: 'other-reader', sessionId: 's2' }) + await expect(read(search.results[0]!.documentId, {}, other)).rejects.toThrow( + 'outside your organization' + ) + expect(mocks.read).not.toHaveBeenCalled() + }) + it.each([ + ['a missing field', 'live2:google_drive~account'], + ['an unknown provider', 'live2:nowhere~account~doc'], + ['a broken escape', 'live2:google_drive~account~%E0%A4'], + ['too many fields', 'live2:google_drive~a~b~c~d~e~f~g~h'], + ['an unreadable earlier form', 'live:not-a-reference'], + ['no prefix', 'google_drive~account~doc'], + ])('asks the caller to search again for %s', async (_case, documentId) => { + await expect(read(documentId)).rejects.toThrow('Search again') + expect(mocks.resolveAccount).not.toHaveBeenCalled() + }) + it('still reads an earlier-form ID minted for this reader and scope', async () => { + const result = await read(legacyId({ user: 'reader', scope: workspaceScope, ...fields })) + expect(result.chunks[0]?.content).toBe('Evidence') + }) + it.each([ + ['reader', { user: 'someone-else', scope: workspaceScope }], + ['scope', { user: 'reader', scope: 'workspace:elsewhere' }], + ])( + 'rejects an earlier-form ID minted for another %s before token resolution', + async (_case, binding) => { + await expect(read(legacyId({ ...binding, ...fields }))).rejects.toThrow( + 'Document not found' + ) + expect(mocks.resolveAccount).not.toHaveBeenCalled() + } + ) + it('matches a selected document across the earlier and current forms', async () => { + const current = encodeLiveReference(fields) + const earlier = legacyId({ user: 'reader', scope: workspaceScope, ...fields }) + await expect(read(current, { filters: { documentIds: [earlier] } })).resolves.toMatchObject({ + documentId: current, + }) + await expect(read(earlier, { filters: { documentIds: [current] } })).resolves.toBeDefined() + expect( + matchesLiveFilters(document, current, 'google_drive', { documentIds: [earlier] }) + ).toBe(true) + await expect( + read(current, { + filters: { documentIds: [encodeLiveReference({ ...fields, id: 'other' })] }, + }) + ).rejects.toThrow('outside the selected search filters') + }) }) it('rejects a document moved out of scope while its content was being read', async () => { const search = await searchLiveKnowledge.execute({ principal, input }) diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 596cbe372fa..6252c77726e 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -11,7 +11,6 @@ import type { } from '@/lib/api/contracts/mothership-assistant-tools' import { type LiveSearchAccountStatus, - liveSearchProviderSchema, type NativeSearchQuery, nativeSearchQueriesSchema, workspaceSearchFiltersSchema, @@ -76,6 +75,12 @@ import { loadLiveSearchPolicies } from '@/lib/sim-search/live/policy-store' import { LIVE_SEARCH_PROVIDER_IDS } from '@/lib/sim-search/live/provider-catalog' import { liveSearchGuidance } from '@/lib/sim-search/live/providers' import { LiveReadError } from '@/lib/sim-search/live/read-error' +import { + decodeLiveReference, + encodeLiveReference, + includesLiveDocument, + liveDocumentKey, +} from '@/lib/sim-search/live/reference' import { queryRelevance } from '@/lib/sim-search/live/text-match' import type { LiveAccount, NativeDocument } from '@/lib/sim-search/live/types' import { projectResolvedSecretModelContent } from '@/executor/utils/resolved-secret-content-projection' @@ -140,43 +145,6 @@ function writeBoundContinuation(provider: BoundProvider, value: BoundContinuatio return cursor } -const referenceSchema = z - .object({ - v: z.literal(1), - user: z.string().min(1), - scope: z.string().min(1), - account: z.string().min(1), - provider: liveSearchProviderSchema, - id: z.string().min(1).max(1000), - container: z.string().max(500).optional(), - kind: z.string().max(200).optional(), - revision: z.string().max(200).optional(), - connectionBinding: z.string().min(1).max(128).optional(), - threadId: z - .string() - .regex(/^\d+\.\d+$/) - .optional(), - }) - .strict() -type Reference = z.output - -export function encodeLiveReference(reference: Reference): string { - const result = `live:${Buffer.from(JSON.stringify(referenceSchema.parse(reference))).toString('base64url')}` - if (result.length > 4000) throw new Error('Provider document reference is too long') - return result -} -export function decodeLiveReference(value: string): Reference { - if (!value.startsWith('live:') || value.length > 4000) - throw new OrchestrationError('not_found', 'Search again to select a live document') - try { - return referenceSchema.parse( - JSON.parse(Buffer.from(value.slice(5), 'base64url').toString('utf8')) - ) - } catch { - throw new OrchestrationError('not_found', 'Invalid live document reference') - } -} - interface LiveSearchOptions { surface?: SearchKnowledgeInput['surface'] query: string @@ -203,7 +171,7 @@ export function matchesLiveFilters( ): boolean { if (!matchesSourceDates(document, provider, filters)) return false if (filters?.source && filters.source !== provider) return false - if (filters?.documentIds && !filters.documentIds.includes(documentId)) return false + if (filters?.documentIds && !includesLiveDocument(filters.documentIds, documentId)) return false if (filters?.modifiedAfter || filters?.modifiedBefore) { const modified = Date.parse(document.modifiedAt ?? '') if (!Number.isFinite(modified)) return false @@ -222,24 +190,16 @@ export function matchesLiveFilters( } return true } -/** A provider result with the reference that binds it to this member, scope, and account. */ +/** A provider result with the ID that locates it within its account. */ interface LiveCandidate { document: NativeDocument documentId: string } -function candidateFor( - document: NativeDocument, - account: LiveAccount, - owner: ResourceOwner, - userId: string -): LiveCandidate { +function candidateFor(document: NativeDocument, account: LiveAccount): LiveCandidate { return { document, documentId: encodeLiveReference({ - v: 1, - user: userId, - scope: resourceScopeKey(resourceScopeFromOwner(owner)), account: account.id, provider: account.provider, id: document.id, @@ -571,7 +531,7 @@ const liveKnowledgeSearch = defineAuthorizedKnowledgeUseCase({ modifiedAfter: instantScopePart(requestedFilters?.modifiedAfter), modifiedBefore: instantScopePart(requestedFilters?.modifiedBefore), documentIds: requestedFilters?.documentIds - ? [...new Set(requestedFilters.documentIds)].sort(compareStrings) + ? [...new Set(requestedFilters.documentIds.map(liveDocumentKey))].sort(compareStrings) : undefined, }) ) @@ -609,7 +569,7 @@ const liveKnowledgeSearch = defineAuthorizedKnowledgeUseCase({ ) const candidates = page.documents .filter((document) => document.id) - .map((document) => candidateFor(document, account, input, userId)) + .map((document) => candidateFor(document, account)) const matchingOf = (verified: LiveCandidate[]) => firstOfEachDocument( verified.filter(({ document, documentId }) => @@ -1072,15 +1032,17 @@ export const readLiveDocument = defineAuthorizedKnowledgeUseCase({ if (input.organizationId) await requireOrganizationSearchAvailable(input.organizationId) if (!Number.isInteger(input.limit) || input.limit < 1 || input.limit > 8) throw new OrchestrationError('validation', 'Invalid document read limit') - const reference = decodeLiveReference(input.documentId) + const { legacy, ...reference } = decodeLiveReference(input.documentId) + // A v2 ID names no reader or scope: the account resolution below decides both. if ( - reference.user !== userId || - reference.scope !== resourceScopeKey(resourceScopeFromOwner(input)) + legacy && + (legacy.user !== userId || legacy.scope !== resourceScopeKey(resourceScopeFromOwner(input))) ) throw new OrchestrationError('not_found', 'Document not found in this search scope') if ( (input.filters?.source && input.filters.source !== reference.provider) || - (input.filters?.documentIds && !input.filters.documentIds.includes(input.documentId)) + (input.filters?.documentIds && + !includesLiveDocument(input.filters.documentIds, input.documentId)) ) throw new OrchestrationError('not_found', 'Document is outside the selected search filters') /** Cancellation by the caller passes through unclassified. */ diff --git a/apps/sim/lib/sim-search/live/reference.test.ts b/apps/sim/lib/sim-search/live/reference.test.ts new file mode 100644 index 00000000000..59ded31e00b --- /dev/null +++ b/apps/sim/lib/sim-search/live/reference.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' +import { + decodeLiveReference, + encodeLiveReference, + type LiveReference, +} from '@/lib/sim-search/live/reference' + +const thread: LiveReference = { + provider: 'slack', + account: 'account', + id: '1730912345.123456', + container: 'C1', + threadId: '1730900000.000100', +} + +const earlierForm = (reference: LiveReference) => + `live:${Buffer.from( + JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...reference }) + ).toString('base64url')}` + +describe('live document IDs', () => { + it.each([ + ['dropping its last character', (id: string) => id.slice(0, -1)], + ['dropping the thread it names', (id: string) => id.replace(/~1730900000\.000100(?=~)/, '')], + ['changing the message it names', (id: string) => id.replace('123456', '123457')], + ])('asks for a new search after %s rather than naming another document', (_case, damage) => { + expect(() => decodeLiveReference(damage(encodeLiveReference(thread)))).toThrow('Search again') + }) + + it.each([ + ['the separator and escape character', 'a~b%c%41'], + ['control characters', 'line\none\ttab'], + ['Unicode text', 'docs/設計/仕様書.md'], + ['an unpaired surrogate', 'name\ud800end'], + ])('reads back an ID holding %s', (_case, id) => { + const reference = { provider: 'gitlab', account: 'account', id } as const + expect(decodeLiveReference(encodeLiveReference(reference))).toEqual(reference) + }) + + it('is never longer than the earlier form for the same document', () => { + const reference = { + provider: 'gitlab', + account: 'account', + id: `src/${'設計書'.repeat(150)}.md`, + } as const + expect(encodeLiveReference(reference).length).toBeLessThan(earlierForm(reference).length) + }) +}) diff --git a/apps/sim/lib/sim-search/live/reference.ts b/apps/sim/lib/sim-search/live/reference.ts new file mode 100644 index 00000000000..18b73c00a25 --- /dev/null +++ b/apps/sim/lib/sim-search/live/reference.ts @@ -0,0 +1,193 @@ +import { sha256Hex } from '@sim/security/hash' +import { z } from 'zod' +import { liveSearchProviderSchema } from '@/lib/api/contracts/mothership-assistant-tools' +import { OrchestrationError } from '@/lib/core/orchestration/types' + +/** + * Live document IDs name a provider document within one of the reader's connected accounts. + * + * They are not an authorization boundary: every read re-resolves the account among the reader's + * own accounts in the request's scope, then re-verifies the document with the provider and the + * admin's source policy. So the ID carries only what locates the document, and nothing the + * request already knows (the reader, the scope). + * + * The current form is `live2:` followed by `~`-separated fields, trailing empty fields dropped, + * then a checksum: `live2:slack~~~~~~`. Clients copy + * IDs verbatim into later calls, so the form stays readable (provider IDs and UUIDs tokenize far + * better than base64 of them) and uses only characters a model reproduces reliably. Earlier IDs + * (`live:` + base64url JSON naming the reader and scope) still read, with their reader and scope + * checked as before. + */ +const LIVE_V1_PREFIX = 'live:' +const LIVE_V2_PREFIX = 'live2:' +const MAX_REFERENCE_LENGTH = 4000 +const FIELD_SEPARATOR = '~' +const CHECKSUM_LENGTH = 6 + +const fieldsSchema = z + .object({ + provider: liveSearchProviderSchema, + account: z.string().min(1).max(500), + id: z.string().min(1).max(1000), + container: z.string().min(1).max(500).optional(), + kind: z.string().min(1).max(200).optional(), + threadId: z + .string() + .regex(/^\d+\.\d+$/) + .optional(), + revision: z.string().min(1).max(200).optional(), + connectionBinding: z.string().min(1).max(128).optional(), + }) + .strict() + +/** Field order of the v2 form; optional fields last so absent ones cost nothing. */ +const FIELD_ORDER = [ + 'provider', + 'account', + 'id', + 'container', + 'kind', + 'threadId', + 'revision', + 'connectionBinding', +] as const satisfies readonly (keyof z.input)[] + +const legacySchema = z + .object({ + v: z.literal(1), + user: z.string().min(1), + scope: z.string().min(1), + account: z.string().min(1), + provider: liveSearchProviderSchema, + id: z.string().min(1).max(1000), + container: z.string().max(500).optional(), + kind: z.string().max(200).optional(), + revision: z.string().max(200).optional(), + connectionBinding: z.string().min(1).max(128).optional(), + threadId: z + .string() + .regex(/^\d+\.\d+$/) + .optional(), + }) + .strict() + +export type LiveReference = z.output + +/** A decoded ID; `legacy` holds the reader and scope a v1 ID was minted for. */ +export type DecodedLiveReference = LiveReference & { + legacy?: { user: string; scope: string } +} + +/** + * Only the separator, the escape character and control characters are escaped, each as `%XX`; + * every other character, Unicode included, is kept as written, so an ID is never longer than the + * text it names and any string encodes. + */ +function escapeField(value: string): string { + let escaped = '' + for (const character of value) { + const code = character.charCodeAt(0) + escaped += + character === '%' || character === FIELD_SEPARATOR || code < 0x20 || code === 0x7f + ? `%${code.toString(16).toUpperCase().padStart(2, '0')}` + : character + } + return escaped +} + +function unescapeField(value: string): string { + return value.replace(/%([0-9A-F]{2})/g, (_, code: string) => + String.fromCharCode(Number.parseInt(code, 16)) + ) +} + +/** + * The last field checks the others, so a copy the client truncated or altered fails to decode and + * asks for a new search instead of naming a different document (a dropped thread or a shortened path + * would otherwise still parse). + */ +function checksum(body: string): string { + return sha256Hex(body).slice(0, CHECKSUM_LENGTH) +} + +/** Mints the current, compact ID for a provider document. */ +export function encodeLiveReference(reference: LiveReference): string { + const fields = fieldsSchema.parse(reference) + const parts = FIELD_ORDER.map((name) => { + const value = fields[name] + return value ? escapeField(value) : '' + }) + while (parts.length > 0 && parts[parts.length - 1] === '') parts.pop() + const body = parts.join(FIELD_SEPARATOR) + const result = `${LIVE_V2_PREFIX}${body}${FIELD_SEPARATOR}${checksum(body)}` + if (result.length > MAX_REFERENCE_LENGTH) + throw new Error('Provider document reference is too long') + return result +} + +function decodeV2(value: string): LiveReference { + const separator = value.lastIndexOf(FIELD_SEPARATOR) + const body = value.slice(0, separator) + if (separator < 0 || value.slice(separator + 1) !== checksum(body)) + throw new Error('Checksum mismatch') + const parts = body.split(FIELD_SEPARATOR) + if (parts.length < 3 || parts.length > FIELD_ORDER.length) throw new Error('Invalid field count') + const fields: Record = {} + FIELD_ORDER.forEach((name, index) => { + const part = parts[index] + if (part) fields[name] = unescapeField(part) + }) + return fieldsSchema.parse(fields) +} + +function decodeV1(body: string): DecodedLiveReference { + const { v, user, scope, ...rest } = legacySchema.parse( + JSON.parse(Buffer.from(body, 'base64url').toString('utf8')) + ) + const fields = Object.fromEntries( + Object.entries(rest).filter(([, value]) => value !== undefined && value !== '') + ) + return { ...fieldsSchema.parse(fields), legacy: { user, scope } } +} + +const SEARCH_AGAIN = 'Invalid live document reference. Search again to select a live document' + +/** Decodes a current or earlier live document ID; anything else asks the caller to search again. */ +export function decodeLiveReference(value: string): DecodedLiveReference { + if (value.length > MAX_REFERENCE_LENGTH) throw new OrchestrationError('not_found', SEARCH_AGAIN) + try { + if (value.startsWith(LIVE_V2_PREFIX)) return decodeV2(value.slice(LIVE_V2_PREFIX.length)) + if (value.startsWith(LIVE_V1_PREFIX)) return decodeV1(value.slice(LIVE_V1_PREFIX.length)) + } catch {} + throw new OrchestrationError('not_found', SEARCH_AGAIN) +} + +/** + * The identity two document IDs share when they name the same document: the current encoding of + * a live ID, whichever form it arrived in, or the value itself for any other ID. Compare document + * IDs through this, never as raw strings, so a v1 ID from an earlier conversation matches the v2 + * ID a new search returns for the same document. + */ +export function liveDocumentKey(documentId: string): string { + if (!documentId.startsWith(LIVE_V1_PREFIX) && !documentId.startsWith(LIVE_V2_PREFIX)) + return documentId + try { + const { legacy, ...reference } = decodeLiveReference(documentId) + return encodeLiveReference(reference) + } catch { + return documentId + } +} + +/** Each filter list's keys, decoded once however many results a search checks against it. */ +const listKeys = new WeakMap>() + +/** Whether `documentId` names one of `documentIds`, comparing by {@link liveDocumentKey}. */ +export function includesLiveDocument(documentIds: readonly string[], documentId: string): boolean { + let keys = listKeys.get(documentIds) + if (!keys) { + keys = new Set(documentIds.map(liveDocumentKey)) + listKeys.set(documentIds, keys) + } + return keys.has(liveDocumentKey(documentId)) +} From 293d58ceeabd9c9531e56b977cde5af98255d39b Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 7 Nov 2026 15:32:59 -0800 Subject: [PATCH 2/3] fix(search): refuse another reader's live ID as not found in this scope --- .../gitlab-federated.integration.ts | 21 ++++++-- apps/sim/lib/knowledge/mcp/server.test.ts | 8 ++- apps/sim/lib/knowledge/search/filters.test.ts | 6 ++- .../lib/sim-search/live/application.test.ts | 51 ++++++++++++------- apps/sim/lib/sim-search/live/application.ts | 18 ++++--- .../sim/lib/sim-search/live/reference.test.ts | 8 ++- apps/sim/lib/sim-search/live/reference.ts | 35 +++++++------ scripts/check-explicit-any.baseline.json | 1 - 8 files changed, 98 insertions(+), 50 deletions(-) diff --git a/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts b/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts index e100fb7d931..eb7a6cf9a4b 100644 --- a/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts @@ -25,6 +25,7 @@ import { generateId } from '@sim/utils/id' import { isPlainRecord } from '@sim/utils/object' import { and, eq, inArray } from 'drizzle-orm' import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { resourceScopeFromOwner, resourceScopeKey } from '@/lib/core/resource-scope' import { seedKnowledgeAclFixture } from '@/lib/knowledge/__integration__/seed-source-access-fixture' import { createKnowledgeConnector, @@ -32,7 +33,11 @@ import { } from '@/lib/knowledge/application/connectors' import { refreshConnectorDirectory } from '@/lib/knowledge/connectors/external-group-sync' import { readLiveDocument, searchLiveKnowledge } from '@/lib/sim-search/live/application' -import { decodeLiveReference } from '@/lib/sim-search/live/reference' +import { + decodeLiveReference, + encodeLiveReference, + liveReferenceOwner, +} from '@/lib/sim-search/live/reference' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' interface FixturePerson { @@ -128,6 +133,14 @@ describe.skipIf(!fixtureFile)('federated self-hosted GitLab Search', () => { }) } + function documentForReader(documentId: string, person: FixturePerson) { + const scope = resourceScopeKey(resourceScopeFromOwner({ organizationId: ids.organizationId })) + return encodeLiveReference({ + ...decodeLiveReference(documentId), + owner: liveReferenceOwner(person.simId, scope), + }) + } + beforeAll(async () => { const fixture: unknown = JSON.parse(await readFile(fixtureFile!, 'utf8')) if ( @@ -310,11 +323,13 @@ describe.skipIf(!fixtureFile)('federated self-hosted GitLab Search', () => { 120_000 ) - it('checks a shared source reference against its reader and rechecks confidentiality', async () => { + it('keeps references reader-bound and rechecks confidentiality before reads', async () => { const reporter = await search(people.reporter!, 'code') const file = reporter.results[0]! - /** The ID names no reader: the guest's own GitLab access refuses the reporter's code result. */ await expect(read(people.guest!, file.documentId)).rejects.toMatchObject({ code: 'not_found' }) + await expect( + read(people.guest!, documentForReader(file.documentId, people.guest!)) + ).rejects.toBeDefined() const guest = await search(people.guest!, 'issues') const normal = guest.results.find( (row) => decodeLiveReference(row.documentId).id === String(issueIid) diff --git a/apps/sim/lib/knowledge/mcp/server.test.ts b/apps/sim/lib/knowledge/mcp/server.test.ts index 2ab34f4cff4..d6ec8d20147 100644 --- a/apps/sim/lib/knowledge/mcp/server.test.ts +++ b/apps/sim/lib/knowledge/mcp/server.test.ts @@ -55,7 +55,7 @@ vi.mock('@/lib/knowledge/application/chat', () => ({ import { OrchestrationError } from '@/lib/core/orchestration/types' import { createKnowledgeMcpServer } from '@/lib/knowledge/mcp/server' -import { encodeLiveReference } from '@/lib/sim-search/live/reference' +import { encodeLiveReference, liveReferenceOwner } from '@/lib/sim-search/live/reference' import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const mocks = hoisted @@ -311,7 +311,11 @@ describe('batched document reads', () => { const earlier = `live:${Buffer.from( JSON.stringify({ v: 1, user: 'person-1', scope: 'organization:org-1', ...reference }) ).toString('base64url')}` - await call('read_document', { documentIds: [earlier, encodeLiveReference(reference)] }) + const current = encodeLiveReference({ + ...reference, + owner: liveReferenceOwner('person-1', 'organization:org-1'), + }) + await call('read_document', { documentIds: [earlier, current] }) expect(mocks.rateLimit).toHaveBeenCalledWith(request, auth, expect.anything(), 1) expect(mocks.liveRead).toHaveBeenCalledTimes(1) }) diff --git a/apps/sim/lib/knowledge/search/filters.test.ts b/apps/sim/lib/knowledge/search/filters.test.ts index 70c13589ec8..07890949785 100644 --- a/apps/sim/lib/knowledge/search/filters.test.ts +++ b/apps/sim/lib/knowledge/search/filters.test.ts @@ -1,9 +1,10 @@ import { describe, expect, it } from 'vitest' import { liveCitationId } from '@/lib/knowledge/search/citation' import { intersectWorkspaceSearchFilters } from '@/lib/knowledge/search/filters' -import { encodeLiveReference } from '@/lib/sim-search/live/reference' +import { encodeLiveReference, liveReferenceOwner } from '@/lib/sim-search/live/reference' const fields = { + owner: liveReferenceOwner('reader', 'workspace:w'), account: 'account', provider: 'slack' as const, id: '1730912345.123456', @@ -11,8 +12,9 @@ const fields = { } /** One Slack message in the current ID form and in the earlier form older conversations hold. */ const currentId = encodeLiveReference(fields) +const { owner: _owner, ...located } = fields const earlierId = `live:${Buffer.from( - JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...fields }) + JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...located }) ).toString('base64url')}` describe('Assistant search scope', () => { diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index f5855b2c37b..528f838b9d0 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -88,7 +88,11 @@ import { NativeSearchError } from '@/lib/sim-search/live/http' import { createPolicyVerifier } from '@/lib/sim-search/live/policy' import { defaultLiveSearchPolicy } from '@/lib/sim-search/live/policy-schema' import { livePolicyFor } from '@/lib/sim-search/live/policy-store' -import { decodeLiveReference, encodeLiveReference } from '@/lib/sim-search/live/reference' +import { + decodeLiveReference, + encodeLiveReference, + liveReferenceOwner, +} from '@/lib/sim-search/live/reference' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' workspaceAuthzMockFns.mockPermissionSatisfies.mockImplementation( @@ -1188,9 +1192,14 @@ describe('authorized live retrieval', () => { describe('live document IDs', () => { const workspaceScope = resourceScopeKey(resourceScopeFromOwner({ workspaceId: 'workspace' })) /** The earlier wire form, spelled out so these tests pin what older conversations hold. */ - const legacyId = (fields: Record) => + const legacyId = ({ owner: _owner, ...fields }: Record) => `live:${Buffer.from(JSON.stringify({ v: 1, ...fields })).toString('base64url')}` - const fields = { account: 'account', provider: 'google_drive' as const, id: 'doc' } + const fields = { + owner: liveReferenceOwner('reader', workspaceScope), + account: 'account', + provider: 'google_drive' as const, + id: 'doc', + } const read = (documentId: string, extra: Record = {}, as = principal) => readLiveDocument.execute({ principal: as, @@ -1205,8 +1214,11 @@ describe('authorized live retrieval', () => { it('mints a compact ID naming no reader or scope that reads back', async () => { const search = await searchLiveKnowledge.execute({ principal, input }) - const documentId = search.results[0]!.documentId - expect(documentId).toMatch(/^live2:google_drive~account~doc~[0-9a-f]{6}$/) + const [first] = search.results + if (!first) throw new Error('Expected a search result') + const { documentId } = first + expect(documentId).toBe(encodeLiveReference(fields)) + expect(documentId).toMatch(/^live2:google_drive~[0-9a-f]{8}~account~doc~[0-9a-f]{6}$/) const result = await read(documentId) expect(result.documentId).toBe(documentId) expect(result.chunks[0]?.content).toBe('Evidence') @@ -1233,19 +1245,22 @@ describe('authorized live retrieval', () => { ) expect(mocks.read).not.toHaveBeenCalled() }) - it("gates a shared account's ID by the reader's own verification", async () => { - const search = await searchLiveKnowledge.execute({ principal, input }) - mocks.service.mockResolvedValue({ - policy: defaultLiveSearchPolicy(), - partial: false, - verify: async () => false, - }) - const other = createSessionPrincipal({ userId: 'other-reader', sessionId: 's2' }) - await expect(read(search.results[0]!.documentId, {}, other)).rejects.toThrow( - 'outside your organization' - ) - expect(mocks.read).not.toHaveBeenCalled() - }) + it.each([ + ['another member', 'other-reader', workspaceScope], + [ + 'another scope', + 'reader', + resourceScopeKey(resourceScopeFromOwner({ workspaceId: 'other' })), + ], + ])( + 'refuses an ID minted for %s as not found before resolving any account', + async (_case, user, scope) => { + const theirs = encodeLiveReference({ ...fields, owner: liveReferenceOwner(user, scope) }) + await expect(read(theirs)).rejects.toThrow('Document not found in this search scope') + expect(mocks.resolveAccount).not.toHaveBeenCalled() + expect(mocks.read).not.toHaveBeenCalled() + } + ) it.each([ ['a missing field', 'live2:google_drive~account'], ['an unknown provider', 'live2:nowhere~account~doc'], diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index 6252c77726e..d5cb69ccede 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -80,6 +80,7 @@ import { encodeLiveReference, includesLiveDocument, liveDocumentKey, + liveReferenceOwner, } from '@/lib/sim-search/live/reference' import { queryRelevance } from '@/lib/sim-search/live/text-match' import type { LiveAccount, NativeDocument } from '@/lib/sim-search/live/types' @@ -196,10 +197,15 @@ interface LiveCandidate { documentId: string } -function candidateFor(document: NativeDocument, account: LiveAccount): LiveCandidate { +function candidateFor( + document: NativeDocument, + account: LiveAccount, + owner: string +): LiveCandidate { return { document, documentId: encodeLiveReference({ + owner, account: account.id, provider: account.provider, id: document.id, @@ -431,6 +437,7 @@ const liveKnowledgeSearch = defineAuthorizedKnowledgeUseCase({ const userId = requirePrincipalSubjectUserId(principal) if (input.organizationId) await requireOrganizationSearchAvailable(input.organizationId) input.signal?.throwIfAborted() + const owner = liveReferenceOwner(userId, resourceScopeKey(resourceScopeFromOwner(input))) const queries = input.nativeQueries ? nativeSearchQueriesSchema.parse(input.nativeQueries) : undefined @@ -569,7 +576,7 @@ const liveKnowledgeSearch = defineAuthorizedKnowledgeUseCase({ ) const candidates = page.documents .filter((document) => document.id) - .map((document) => candidateFor(document, account)) + .map((document) => candidateFor(document, account, owner)) const matchingOf = (verified: LiveCandidate[]) => firstOfEachDocument( verified.filter(({ document, documentId }) => @@ -1032,11 +1039,10 @@ export const readLiveDocument = defineAuthorizedKnowledgeUseCase({ if (input.organizationId) await requireOrganizationSearchAvailable(input.organizationId) if (!Number.isInteger(input.limit) || input.limit < 1 || input.limit > 8) throw new OrchestrationError('validation', 'Invalid document read limit') - const { legacy, ...reference } = decodeLiveReference(input.documentId) - // A v2 ID names no reader or scope: the account resolution below decides both. + const reference = decodeLiveReference(input.documentId) if ( - legacy && - (legacy.user !== userId || legacy.scope !== resourceScopeKey(resourceScopeFromOwner(input))) + reference.owner !== + liveReferenceOwner(userId, resourceScopeKey(resourceScopeFromOwner(input))) ) throw new OrchestrationError('not_found', 'Document not found in this search scope') if ( diff --git a/apps/sim/lib/sim-search/live/reference.test.ts b/apps/sim/lib/sim-search/live/reference.test.ts index 59ded31e00b..3ff1e5f6355 100644 --- a/apps/sim/lib/sim-search/live/reference.test.ts +++ b/apps/sim/lib/sim-search/live/reference.test.ts @@ -5,15 +5,18 @@ import { type LiveReference, } from '@/lib/sim-search/live/reference' +const owner = 'a1b2c3d4' + const thread: LiveReference = { provider: 'slack', + owner, account: 'account', id: '1730912345.123456', container: 'C1', threadId: '1730900000.000100', } -const earlierForm = (reference: LiveReference) => +const earlierForm = ({ owner: _owner, ...reference }: LiveReference) => `live:${Buffer.from( JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...reference }) ).toString('base64url')}` @@ -33,13 +36,14 @@ describe('live document IDs', () => { ['Unicode text', 'docs/設計/仕様書.md'], ['an unpaired surrogate', 'name\ud800end'], ])('reads back an ID holding %s', (_case, id) => { - const reference = { provider: 'gitlab', account: 'account', id } as const + const reference = { provider: 'gitlab', owner, account: 'account', id } as const expect(decodeLiveReference(encodeLiveReference(reference))).toEqual(reference) }) it('is never longer than the earlier form for the same document', () => { const reference = { provider: 'gitlab', + owner, account: 'account', id: `src/${'設計書'.repeat(150)}.md`, } as const diff --git a/apps/sim/lib/sim-search/live/reference.ts b/apps/sim/lib/sim-search/live/reference.ts index 18b73c00a25..439343852d2 100644 --- a/apps/sim/lib/sim-search/live/reference.ts +++ b/apps/sim/lib/sim-search/live/reference.ts @@ -8,25 +8,28 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' * * They are not an authorization boundary: every read re-resolves the account among the reader's * own accounts in the request's scope, then re-verifies the document with the provider and the - * admin's source policy. So the ID carries only what locates the document, and nothing the - * request already knows (the reader, the scope). + * admin's source policy. An ID names its reader and scope only by a short {@link liveReferenceOwner} + * fingerprint, so a read in another scope or by another member is refused as not found in this + * scope rather than reaching account resolution. * * The current form is `live2:` followed by `~`-separated fields, trailing empty fields dropped, - * then a checksum: `live2:slack~~~~~~`. Clients copy - * IDs verbatim into later calls, so the form stays readable (provider IDs and UUIDs tokenize far - * better than base64 of them) and uses only characters a model reproduces reliably. Earlier IDs - * (`live:` + base64url JSON naming the reader and scope) still read, with their reader and scope - * checked as before. + * then a checksum: `live2:slack~~~~~~~`. + * Clients copy IDs verbatim into later calls, so the form stays readable (provider IDs and UUIDs + * tokenize far better than base64 of them) and uses only characters a model reproduces reliably. + * Earlier IDs (`live:` + base64url JSON naming the reader and scope) still read, checked against + * the same fingerprint. */ const LIVE_V1_PREFIX = 'live:' const LIVE_V2_PREFIX = 'live2:' const MAX_REFERENCE_LENGTH = 4000 const FIELD_SEPARATOR = '~' const CHECKSUM_LENGTH = 6 +const OWNER_LENGTH = 8 const fieldsSchema = z .object({ provider: liveSearchProviderSchema, + owner: z.string().regex(new RegExp(`^[0-9a-f]{${OWNER_LENGTH}}$`)), account: z.string().min(1).max(500), id: z.string().min(1).max(1000), container: z.string().min(1).max(500).optional(), @@ -43,6 +46,7 @@ const fieldsSchema = z /** Field order of the v2 form; optional fields last so absent ones cost nothing. */ const FIELD_ORDER = [ 'provider', + 'owner', 'account', 'id', 'container', @@ -73,9 +77,9 @@ const legacySchema = z export type LiveReference = z.output -/** A decoded ID; `legacy` holds the reader and scope a v1 ID was minted for. */ -export type DecodedLiveReference = LiveReference & { - legacy?: { user: string; scope: string } +/** The reader and scope an ID was minted for, as the short fingerprint the ID carries. */ +export function liveReferenceOwner(userId: string, scope: string): string { + return sha256Hex(`${userId}\u0000${scope}`).slice(0, OWNER_LENGTH) } /** @@ -131,7 +135,7 @@ function decodeV2(value: string): LiveReference { if (separator < 0 || value.slice(separator + 1) !== checksum(body)) throw new Error('Checksum mismatch') const parts = body.split(FIELD_SEPARATOR) - if (parts.length < 3 || parts.length > FIELD_ORDER.length) throw new Error('Invalid field count') + if (parts.length < 4 || parts.length > FIELD_ORDER.length) throw new Error('Invalid field count') const fields: Record = {} FIELD_ORDER.forEach((name, index) => { const part = parts[index] @@ -140,20 +144,20 @@ function decodeV2(value: string): LiveReference { return fieldsSchema.parse(fields) } -function decodeV1(body: string): DecodedLiveReference { +function decodeV1(body: string): LiveReference { const { v, user, scope, ...rest } = legacySchema.parse( JSON.parse(Buffer.from(body, 'base64url').toString('utf8')) ) const fields = Object.fromEntries( Object.entries(rest).filter(([, value]) => value !== undefined && value !== '') ) - return { ...fieldsSchema.parse(fields), legacy: { user, scope } } + return fieldsSchema.parse({ ...fields, owner: liveReferenceOwner(user, scope) }) } const SEARCH_AGAIN = 'Invalid live document reference. Search again to select a live document' /** Decodes a current or earlier live document ID; anything else asks the caller to search again. */ -export function decodeLiveReference(value: string): DecodedLiveReference { +export function decodeLiveReference(value: string): LiveReference { if (value.length > MAX_REFERENCE_LENGTH) throw new OrchestrationError('not_found', SEARCH_AGAIN) try { if (value.startsWith(LIVE_V2_PREFIX)) return decodeV2(value.slice(LIVE_V2_PREFIX.length)) @@ -172,8 +176,7 @@ export function liveDocumentKey(documentId: string): string { if (!documentId.startsWith(LIVE_V1_PREFIX) && !documentId.startsWith(LIVE_V2_PREFIX)) return documentId try { - const { legacy, ...reference } = decodeLiveReference(documentId) - return encodeLiveReference(reference) + return encodeLiveReference(decodeLiveReference(documentId)) } catch { return documentId } diff --git a/scripts/check-explicit-any.baseline.json b/scripts/check-explicit-any.baseline.json index 8c3029c65c2..a4fb7f4bbf0 100644 --- a/scripts/check-explicit-any.baseline.json +++ b/scripts/check-explicit-any.baseline.json @@ -1511,7 +1511,6 @@ "apps/sim/lib/knowledge/read-access.test.ts": 1, "apps/sim/lib/knowledge/read-access.ts": 1, "apps/sim/lib/knowledge/search/budget.test.ts": 1, - "apps/sim/lib/knowledge/search/filters.ts": 1, "apps/sim/lib/knowledge/search/queries.test.ts": 10, "apps/sim/lib/knowledge/search/queries.ts": 3, "apps/sim/lib/knowledge/tags/filter-resolution.ts": 1, From 49c06dd1f2e097e6e784da35225afdb77907b88d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 7 Nov 2026 16:10:42 -0800 Subject: [PATCH 3/3] fix(search): bind live IDs to their reader through the checksum, not a visible field --- .../gitlab-federated.integration.ts | 8 +- apps/sim/lib/knowledge/mcp/server.test.ts | 19 --- apps/sim/lib/knowledge/mcp/server.ts | 5 +- apps/sim/lib/knowledge/search/filters.test.ts | 9 +- .../lib/sim-search/live/application.test.ts | 22 ++- apps/sim/lib/sim-search/live/application.ts | 32 ++--- .../sim/lib/sim-search/live/reference.test.ts | 35 +++-- apps/sim/lib/sim-search/live/reference.ts | 128 +++++++++++------- 8 files changed, 141 insertions(+), 117 deletions(-) diff --git a/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts b/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts index eb7a6cf9a4b..183e4328089 100644 --- a/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/gitlab-federated.integration.ts @@ -135,10 +135,10 @@ describe.skipIf(!fixtureFile)('federated self-hosted GitLab Search', () => { function documentForReader(documentId: string, person: FixturePerson) { const scope = resourceScopeKey(resourceScopeFromOwner({ organizationId: ids.organizationId })) - return encodeLiveReference({ - ...decodeLiveReference(documentId), - owner: liveReferenceOwner(person.simId, scope), - }) + return encodeLiveReference( + decodeLiveReference(documentId), + liveReferenceOwner(person.simId, scope) + ) } beforeAll(async () => { diff --git a/apps/sim/lib/knowledge/mcp/server.test.ts b/apps/sim/lib/knowledge/mcp/server.test.ts index d6ec8d20147..579a6aa317a 100644 --- a/apps/sim/lib/knowledge/mcp/server.test.ts +++ b/apps/sim/lib/knowledge/mcp/server.test.ts @@ -55,7 +55,6 @@ vi.mock('@/lib/knowledge/application/chat', () => ({ import { OrchestrationError } from '@/lib/core/orchestration/types' import { createKnowledgeMcpServer } from '@/lib/knowledge/mcp/server' -import { encodeLiveReference, liveReferenceOwner } from '@/lib/sim-search/live/reference' import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' const mocks = hoisted @@ -302,24 +301,6 @@ describe('batched document reads', () => { expect(mocks.liveRead).toHaveBeenCalledTimes(2) }) - it('reads a document once when the batch names it in both ID forms', async () => { - create() - mocks.liveRead.mockImplementation(async ({ input }: { input: { documentId: string } }) => - readResult(input.documentId) - ) - const reference = { provider: 'google_drive', account: 'account', id: 'doc' } as const - const earlier = `live:${Buffer.from( - JSON.stringify({ v: 1, user: 'person-1', scope: 'organization:org-1', ...reference }) - ).toString('base64url')}` - const current = encodeLiveReference({ - ...reference, - owner: liveReferenceOwner('person-1', 'organization:org-1'), - }) - await call('read_document', { documentIds: [earlier, current] }) - expect(mocks.rateLimit).toHaveBeenCalledWith(request, auth, expect.anything(), 1) - expect(mocks.liveRead).toHaveBeenCalledTimes(1) - }) - it('reads at most five documents from providers at once, in request order', async () => { create() let inFlight = 0 diff --git a/apps/sim/lib/knowledge/mcp/server.ts b/apps/sim/lib/knowledge/mcp/server.ts index e7e9899ed09..b965738afc9 100644 --- a/apps/sim/lib/knowledge/mcp/server.ts +++ b/apps/sim/lib/knowledge/mcp/server.ts @@ -30,7 +30,6 @@ import { liveCitationId } from '@/lib/knowledge/search/citation' import { toolError } from '@/lib/mcp/tool-result' import { readLiveDocument, searchLiveKnowledge } from '@/lib/sim-search/live/application' import { LiveReadError } from '@/lib/sim-search/live/read-error' -import { liveDocumentKey } from '@/lib/sim-search/live/reference' import { v2CaughtOrchestrationError } from '@/app/api/v2/lib/response' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' @@ -226,9 +225,7 @@ export function createKnowledgeMcpServer(context: KnowledgeMcpContext): McpServe async (raw: unknown, extra: { signal: AbortSignal }) => { const parsed = readLiveDocumentMcpSchema.safeParse(raw) const batch = - parsed.success && parsed.data.documentIds - ? [...new Map(parsed.data.documentIds.map((id) => [liveDocumentKey(id), id])).values()] - : [] + parsed.success && parsed.data.documentIds ? [...new Set(parsed.data.documentIds)] : [] return execute( 'read_document', knowledgeOperations.readDocument, diff --git a/apps/sim/lib/knowledge/search/filters.test.ts b/apps/sim/lib/knowledge/search/filters.test.ts index 07890949785..4aa4c73de0e 100644 --- a/apps/sim/lib/knowledge/search/filters.test.ts +++ b/apps/sim/lib/knowledge/search/filters.test.ts @@ -3,18 +3,17 @@ import { liveCitationId } from '@/lib/knowledge/search/citation' import { intersectWorkspaceSearchFilters } from '@/lib/knowledge/search/filters' import { encodeLiveReference, liveReferenceOwner } from '@/lib/sim-search/live/reference' +const owner = liveReferenceOwner('reader', 'workspace:w') const fields = { - owner: liveReferenceOwner('reader', 'workspace:w'), account: 'account', provider: 'slack' as const, id: '1730912345.123456', container: 'C1', } /** One Slack message in the current ID form and in the earlier form older conversations hold. */ -const currentId = encodeLiveReference(fields) -const { owner: _owner, ...located } = fields +const currentId = encodeLiveReference(fields, owner) const earlierId = `live:${Buffer.from( - JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...located }) + JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...fields }) ).toString('base64url')}` describe('Assistant search scope', () => { @@ -59,7 +58,7 @@ describe('live citation IDs', () => { it('cite one document as one source whichever ID form named it', () => { expect(liveCitationId(earlierId)).toBe(liveCitationId(currentId)) expect(liveCitationId(currentId)).not.toBe( - liveCitationId(encodeLiveReference({ ...fields, id: '1730912345.123457' })) + liveCitationId(encodeLiveReference({ ...fields, id: '1730912345.123457' }, owner)) ) }) }) diff --git a/apps/sim/lib/sim-search/live/application.test.ts b/apps/sim/lib/sim-search/live/application.test.ts index 528f838b9d0..ca6f8a3d0bf 100644 --- a/apps/sim/lib/sim-search/live/application.test.ts +++ b/apps/sim/lib/sim-search/live/application.test.ts @@ -1192,14 +1192,10 @@ describe('authorized live retrieval', () => { describe('live document IDs', () => { const workspaceScope = resourceScopeKey(resourceScopeFromOwner({ workspaceId: 'workspace' })) /** The earlier wire form, spelled out so these tests pin what older conversations hold. */ - const legacyId = ({ owner: _owner, ...fields }: Record) => + const legacyId = (fields: Record) => `live:${Buffer.from(JSON.stringify({ v: 1, ...fields })).toString('base64url')}` - const fields = { - owner: liveReferenceOwner('reader', workspaceScope), - account: 'account', - provider: 'google_drive' as const, - id: 'doc', - } + const readerOwner = liveReferenceOwner('reader', workspaceScope) + const fields = { account: 'account', provider: 'google_drive' as const, id: 'doc' } const read = (documentId: string, extra: Record = {}, as = principal) => readLiveDocument.execute({ principal: as, @@ -1217,8 +1213,8 @@ describe('authorized live retrieval', () => { const [first] = search.results if (!first) throw new Error('Expected a search result') const { documentId } = first - expect(documentId).toBe(encodeLiveReference(fields)) - expect(documentId).toMatch(/^live2:google_drive~[0-9a-f]{8}~account~doc~[0-9a-f]{6}$/) + expect(documentId).toBe(encodeLiveReference(fields, readerOwner)) + expect(documentId).toMatch(/^live2:google_drive~account~doc~[0-9a-f]{6}$/) const result = await read(documentId) expect(result.documentId).toBe(documentId) expect(result.chunks[0]?.content).toBe('Evidence') @@ -1235,7 +1231,7 @@ describe('authorized live retrieval', () => { throw new NativeSearchError('reconnect', 'This search connection is no longer available.') return { account, accessToken: 'secret' } }) - const theirs = encodeLiveReference({ ...fields, account: 'their-account' }) + const theirs = encodeLiveReference({ ...fields, account: 'their-account' }, readerOwner) await expect(read(theirs)).rejects.toMatchObject({ code: 'unauthorized' }) expect(mocks.resolveAccount).toHaveBeenCalledWith( expect.anything(), @@ -1255,7 +1251,7 @@ describe('authorized live retrieval', () => { ])( 'refuses an ID minted for %s as not found before resolving any account', async (_case, user, scope) => { - const theirs = encodeLiveReference({ ...fields, owner: liveReferenceOwner(user, scope) }) + const theirs = encodeLiveReference(fields, liveReferenceOwner(user, scope)) await expect(read(theirs)).rejects.toThrow('Document not found in this search scope') expect(mocks.resolveAccount).not.toHaveBeenCalled() expect(mocks.read).not.toHaveBeenCalled() @@ -1289,7 +1285,7 @@ describe('authorized live retrieval', () => { } ) it('matches a selected document across the earlier and current forms', async () => { - const current = encodeLiveReference(fields) + const current = encodeLiveReference(fields, readerOwner) const earlier = legacyId({ user: 'reader', scope: workspaceScope, ...fields }) await expect(read(current, { filters: { documentIds: [earlier] } })).resolves.toMatchObject({ documentId: current, @@ -1300,7 +1296,7 @@ describe('authorized live retrieval', () => { ).toBe(true) await expect( read(current, { - filters: { documentIds: [encodeLiveReference({ ...fields, id: 'other' })] }, + filters: { documentIds: [encodeLiveReference({ ...fields, id: 'other' }, readerOwner)] }, }) ).rejects.toThrow('outside the selected search filters') }) diff --git a/apps/sim/lib/sim-search/live/application.ts b/apps/sim/lib/sim-search/live/application.ts index d5cb69ccede..66bcbfc3433 100644 --- a/apps/sim/lib/sim-search/live/application.ts +++ b/apps/sim/lib/sim-search/live/application.ts @@ -76,11 +76,11 @@ import { LIVE_SEARCH_PROVIDER_IDS } from '@/lib/sim-search/live/provider-catalog import { liveSearchGuidance } from '@/lib/sim-search/live/providers' import { LiveReadError } from '@/lib/sim-search/live/read-error' import { - decodeLiveReference, encodeLiveReference, includesLiveDocument, liveDocumentKey, liveReferenceOwner, + readLiveReference, } from '@/lib/sim-search/live/reference' import { queryRelevance } from '@/lib/sim-search/live/text-match' import type { LiveAccount, NativeDocument } from '@/lib/sim-search/live/types' @@ -204,17 +204,19 @@ function candidateFor( ): LiveCandidate { return { document, - documentId: encodeLiveReference({ - owner, - account: account.id, - provider: account.provider, - id: document.id, - ...(document.container ? { container: document.container } : {}), - ...(document.kind ? { kind: document.kind } : {}), - ...(document.revision ? { revision: document.revision } : {}), - ...(document.connectionBinding ? { connectionBinding: document.connectionBinding } : {}), - ...(document.threadId ? { threadId: document.threadId } : {}), - }), + documentId: encodeLiveReference( + { + account: account.id, + provider: account.provider, + id: document.id, + ...(document.container ? { container: document.container } : {}), + ...(document.kind ? { kind: document.kind } : {}), + ...(document.revision ? { revision: document.revision } : {}), + ...(document.connectionBinding ? { connectionBinding: document.connectionBinding } : {}), + ...(document.threadId ? { threadId: document.threadId } : {}), + }, + owner + ), } } @@ -1039,12 +1041,10 @@ export const readLiveDocument = defineAuthorizedKnowledgeUseCase({ if (input.organizationId) await requireOrganizationSearchAvailable(input.organizationId) if (!Number.isInteger(input.limit) || input.limit < 1 || input.limit > 8) throw new OrchestrationError('validation', 'Invalid document read limit') - const reference = decodeLiveReference(input.documentId) - if ( - reference.owner !== + const reference = readLiveReference( + input.documentId, liveReferenceOwner(userId, resourceScopeKey(resourceScopeFromOwner(input))) ) - throw new OrchestrationError('not_found', 'Document not found in this search scope') if ( (input.filters?.source && input.filters.source !== reference.provider) || (input.filters?.documentIds && diff --git a/apps/sim/lib/sim-search/live/reference.test.ts b/apps/sim/lib/sim-search/live/reference.test.ts index 3ff1e5f6355..86c36a8f6b8 100644 --- a/apps/sim/lib/sim-search/live/reference.test.ts +++ b/apps/sim/lib/sim-search/live/reference.test.ts @@ -1,22 +1,22 @@ import { describe, expect, it } from 'vitest' import { - decodeLiveReference, encodeLiveReference, type LiveReference, + liveReferenceOwner, + readLiveReference, } from '@/lib/sim-search/live/reference' -const owner = 'a1b2c3d4' +const owner = liveReferenceOwner('reader', 'workspace:w') const thread: LiveReference = { provider: 'slack', - owner, account: 'account', id: '1730912345.123456', container: 'C1', threadId: '1730900000.000100', } -const earlierForm = ({ owner: _owner, ...reference }: LiveReference) => +const earlierForm = (reference: LiveReference) => `live:${Buffer.from( JSON.stringify({ v: 1, user: 'reader', scope: 'workspace:w', ...reference }) ).toString('base64url')}` @@ -27,7 +27,21 @@ describe('live document IDs', () => { ['dropping the thread it names', (id: string) => id.replace(/~1730900000\.000100(?=~)/, '')], ['changing the message it names', (id: string) => id.replace('123456', '123457')], ])('asks for a new search after %s rather than naming another document', (_case, damage) => { - expect(() => decodeLiveReference(damage(encodeLiveReference(thread)))).toThrow('Search again') + expect(() => readLiveReference(damage(encodeLiveReference(thread, owner)), owner)).toThrow( + 'Search again' + ) + }) + + it.each([ + ['another member', liveReferenceOwner('someone-else', 'workspace:w')], + ['another scope', liveReferenceOwner('reader', 'workspace:other')], + ])('refuses an ID read by %s as not found in this scope', (_case, reader) => { + expect(() => readLiveReference(encodeLiveReference(thread, owner), reader)).toThrow( + 'Document not found in this search scope' + ) + expect(() => readLiveReference(earlierForm(thread), reader)).toThrow( + 'Document not found in this search scope' + ) }) it.each([ @@ -36,17 +50,20 @@ describe('live document IDs', () => { ['Unicode text', 'docs/設計/仕様書.md'], ['an unpaired surrogate', 'name\ud800end'], ])('reads back an ID holding %s', (_case, id) => { - const reference = { provider: 'gitlab', owner, account: 'account', id } as const - expect(decodeLiveReference(encodeLiveReference(reference))).toEqual(reference) + const reference = { provider: 'gitlab', account: 'account', id } as const + expect(readLiveReference(encodeLiveReference(reference, owner), owner)).toEqual(reference) + }) + + it('reads an earlier-form ID minted for the same reader and scope', () => { + expect(readLiveReference(earlierForm(thread), owner)).toEqual(thread) }) it('is never longer than the earlier form for the same document', () => { const reference = { provider: 'gitlab', - owner, account: 'account', id: `src/${'設計書'.repeat(150)}.md`, } as const - expect(encodeLiveReference(reference).length).toBeLessThan(earlierForm(reference).length) + expect(encodeLiveReference(reference, owner).length).toBeLessThan(earlierForm(reference).length) }) }) diff --git a/apps/sim/lib/sim-search/live/reference.ts b/apps/sim/lib/sim-search/live/reference.ts index 439343852d2..2266c55ec3f 100644 --- a/apps/sim/lib/sim-search/live/reference.ts +++ b/apps/sim/lib/sim-search/live/reference.ts @@ -8,28 +8,27 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' * * They are not an authorization boundary: every read re-resolves the account among the reader's * own accounts in the request's scope, then re-verifies the document with the provider and the - * admin's source policy. An ID names its reader and scope only by a short {@link liveReferenceOwner} - * fingerprint, so a read in another scope or by another member is refused as not found in this - * scope rather than reaching account resolution. + * admin's source policy. * - * The current form is `live2:` followed by `~`-separated fields, trailing empty fields dropped, - * then a checksum: `live2:slack~~~~~~~`. + * The current form is `live2:` followed by the document's `~`-separated location fields, trailing + * empty fields dropped, then a checksum: `live2:slack~~~~~~`. * Clients copy IDs verbatim into later calls, so the form stays readable (provider IDs and UUIDs - * tokenize far better than base64 of them) and uses only characters a model reproduces reliably. - * Earlier IDs (`live:` + base64url JSON naming the reader and scope) still read, checked against - * the same fingerprint. + * tokenize far better than base64 of them) and carries nothing a model must copy that does not + * locate the document. The checksum covers the location and the reader and scope the ID was minted + * for, so a copy that was truncated or altered, or one read by another member or in another scope, + * is refused as not found here rather than naming a different document or reaching account + * resolution. Earlier IDs (`live:` + base64url JSON naming the reader and scope) still read, with + * their reader and scope checked as before. */ const LIVE_V1_PREFIX = 'live:' const LIVE_V2_PREFIX = 'live2:' const MAX_REFERENCE_LENGTH = 4000 const FIELD_SEPARATOR = '~' const CHECKSUM_LENGTH = 6 -const OWNER_LENGTH = 8 const fieldsSchema = z .object({ provider: liveSearchProviderSchema, - owner: z.string().regex(new RegExp(`^[0-9a-f]{${OWNER_LENGTH}}$`)), account: z.string().min(1).max(500), id: z.string().min(1).max(1000), container: z.string().min(1).max(500).optional(), @@ -46,7 +45,6 @@ const fieldsSchema = z /** Field order of the v2 form; optional fields last so absent ones cost nothing. */ const FIELD_ORDER = [ 'provider', - 'owner', 'account', 'id', 'container', @@ -77,9 +75,9 @@ const legacySchema = z export type LiveReference = z.output -/** The reader and scope an ID was minted for, as the short fingerprint the ID carries. */ +/** The reader and scope an ID is minted for and read in. */ export function liveReferenceOwner(userId: string, scope: string): string { - return sha256Hex(`${userId}\u0000${scope}`).slice(0, OWNER_LENGTH) + return `${userId}\u0000${scope}` } /** @@ -105,78 +103,114 @@ function unescapeField(value: string): string { ) } -/** - * The last field checks the others, so a copy the client truncated or altered fails to decode and - * asks for a new search instead of naming a different document (a dropped thread or a shortened path - * would otherwise still parse). - */ -function checksum(body: string): string { - return sha256Hex(body).slice(0, CHECKSUM_LENGTH) -} - -/** Mints the current, compact ID for a provider document. */ -export function encodeLiveReference(reference: LiveReference): string { +function locationOf(reference: LiveReference): string { const fields = fieldsSchema.parse(reference) const parts = FIELD_ORDER.map((name) => { const value = fields[name] return value ? escapeField(value) : '' }) while (parts.length > 0 && parts[parts.length - 1] === '') parts.pop() - const body = parts.join(FIELD_SEPARATOR) - const result = `${LIVE_V2_PREFIX}${body}${FIELD_SEPARATOR}${checksum(body)}` + return parts.join(FIELD_SEPARATOR) +} + +function checksum(owner: string, location: string): string { + return sha256Hex(`${owner}\u0000${location}`).slice(0, CHECKSUM_LENGTH) +} + +/** Mints the current, compact ID for a provider document, for the reader and scope `owner` names. */ +export function encodeLiveReference(reference: LiveReference, owner: string): string { + const location = locationOf(reference) + const result = `${LIVE_V2_PREFIX}${location}${FIELD_SEPARATOR}${checksum(owner, location)}` if (result.length > MAX_REFERENCE_LENGTH) throw new Error('Provider document reference is too long') return result } -function decodeV2(value: string): LiveReference { +interface ParsedReference { + reference: LiveReference + /** Whether the ID was minted for the reader and scope `owner` names. */ + mintedFor: (owner: string) => boolean +} + +function parseV2(value: string): ParsedReference { const separator = value.lastIndexOf(FIELD_SEPARATOR) - const body = value.slice(0, separator) - if (separator < 0 || value.slice(separator + 1) !== checksum(body)) - throw new Error('Checksum mismatch') - const parts = body.split(FIELD_SEPARATOR) - if (parts.length < 4 || parts.length > FIELD_ORDER.length) throw new Error('Invalid field count') + if (separator < 0) throw new Error('Missing checksum') + const location = value.slice(0, separator) + const check = value.slice(separator + 1) + const parts = location.split(FIELD_SEPARATOR) + if (parts.length < 3 || parts.length > FIELD_ORDER.length) throw new Error('Invalid field count') const fields: Record = {} FIELD_ORDER.forEach((name, index) => { const part = parts[index] if (part) fields[name] = unescapeField(part) }) - return fieldsSchema.parse(fields) + return { + reference: fieldsSchema.parse(fields), + mintedFor: (owner) => check === checksum(owner, location), + } } -function decodeV1(body: string): LiveReference { +function parseV1(body: string): ParsedReference { const { v, user, scope, ...rest } = legacySchema.parse( JSON.parse(Buffer.from(body, 'base64url').toString('utf8')) ) const fields = Object.fromEntries( Object.entries(rest).filter(([, value]) => value !== undefined && value !== '') ) - return fieldsSchema.parse({ ...fields, owner: liveReferenceOwner(user, scope) }) + return { + reference: fieldsSchema.parse(fields), + mintedFor: (owner) => owner === liveReferenceOwner(user, scope), + } } -const SEARCH_AGAIN = 'Invalid live document reference. Search again to select a live document' +const SEARCH_AGAIN = 'Search again to select a live document' -/** Decodes a current or earlier live document ID; anything else asks the caller to search again. */ -export function decodeLiveReference(value: string): LiveReference { - if (value.length > MAX_REFERENCE_LENGTH) throw new OrchestrationError('not_found', SEARCH_AGAIN) +function parse(value: string): ParsedReference { try { - if (value.startsWith(LIVE_V2_PREFIX)) return decodeV2(value.slice(LIVE_V2_PREFIX.length)) - if (value.startsWith(LIVE_V1_PREFIX)) return decodeV1(value.slice(LIVE_V1_PREFIX.length)) + if (value.length <= MAX_REFERENCE_LENGTH) { + if (value.startsWith(LIVE_V2_PREFIX)) return parseV2(value.slice(LIVE_V2_PREFIX.length)) + if (value.startsWith(LIVE_V1_PREFIX)) return parseV1(value.slice(LIVE_V1_PREFIX.length)) + } } catch {} - throw new OrchestrationError('not_found', SEARCH_AGAIN) + throw new OrchestrationError('not_found', `Invalid live document reference. ${SEARCH_AGAIN}`) +} + +/** + * The document a current or earlier live ID names, without checking whom it was minted for. Reads + * use {@link readLiveReference}; anything that is not a live ID asks the caller to search again. + */ +export function decodeLiveReference(value: string): LiveReference { + return parse(value).reference +} + +/** + * The document an ID names, when it was minted for the reader and scope `owner` names. A current ID + * whose checksum does not hold for them, because it was minted for someone or somewhere else or + * was altered, is not found in this scope. + */ +export function readLiveReference(value: string, owner: string): LiveReference { + const { reference, mintedFor } = parse(value) + if (!mintedFor(owner)) + throw new OrchestrationError( + 'not_found', + value.startsWith(LIVE_V2_PREFIX) + ? `Document not found in this search scope. ${SEARCH_AGAIN}` + : 'Document not found in this search scope' + ) + return reference } /** - * The identity two document IDs share when they name the same document: the current encoding of - * a live ID, whichever form it arrived in, or the value itself for any other ID. Compare document - * IDs through this, never as raw strings, so a v1 ID from an earlier conversation matches the v2 - * ID a new search returns for the same document. + * The identity two document IDs share when they name the same document: its location, whichever + * form the ID arrived in, or the value itself for any other ID. Compare document IDs through this, + * never as raw strings, so a v1 ID from an earlier conversation matches the v2 ID a new search + * returns for the same document. */ export function liveDocumentKey(documentId: string): string { if (!documentId.startsWith(LIVE_V1_PREFIX) && !documentId.startsWith(LIVE_V2_PREFIX)) return documentId try { - return encodeLiveReference(decodeLiveReference(documentId)) + return `${LIVE_V2_PREFIX}${locationOf(decodeLiveReference(documentId))}` } catch { return documentId }