Skip to content

Commit ff0469e

Browse files
committed
fix(knowledge): grant member access as each batch is indexed instead of at the end of the run
1 parent a9245b4 commit ff0469e

3 files changed

Lines changed: 89 additions & 0 deletions

File tree

‎apps/sim/lib/knowledge/connectors/member-sync-engine.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import {
2626
memberFailureBackoffMs,
2727
memberNextAttemptAt,
2828
nextMemberSyncTime,
29+
persistedDocumentsByObserver,
2930
shouldListFully,
3031
} from '@/lib/knowledge/connectors/member-sync-engine'
3132
import {
@@ -272,6 +273,26 @@ describe('member sync engine decisions', () => {
272273
expect(second.retainedBytes).toBeGreaterThan(first.retainedBytes)
273274
})
274275

276+
it('grants a persisted batch to every member who listed each document, as it lands', () => {
277+
const union = new Map()
278+
admitMemberListing(union, 'm-1', [doc('a'), doc('b')], 'c-1', 0)
279+
admitMemberListing(union, 'm-2', [doc('a'), doc('c')], 'c-1', 0)
280+
281+
const byMember = persistedDocumentsByObserver(
282+
[
283+
{ externalId: 'a', documentId: 'd-a' },
284+
{ externalId: 'b', documentId: 'd-b' },
285+
{ externalId: 'zzz', documentId: 'd-z' },
286+
],
287+
union
288+
)
289+
290+
expect([...byMember.entries()]).toEqual([
291+
['m-1', ['d-a', 'd-b']],
292+
['m-2', ['d-a']],
293+
])
294+
})
295+
275296
it('counts a member once per external id even when their listing repeats it', () => {
276297
const union = new Map()
277298
const admitted = admitMemberListing(union, 'm-1', [doc('a'), doc('a')], 'c-1', 0)

‎apps/sim/lib/knowledge/connectors/member-sync-engine.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ import {
6969
classifySuspectListing,
7070
createSyncRunState,
7171
loadOwnedCorpus,
72+
type PersistedDocument,
7273
processDocOps,
7374
RETRY_WINDOW_DAYS,
7475
runChangeFeedPass,
@@ -169,6 +170,27 @@ interface MemberListingOutcome {
169170
changeCursor: string | null | undefined
170171
}
171172

173+
/**
174+
* The documents a batch wrote, grouped by each member who listed them, so a
175+
* grant can be recorded per observer the moment the row exists. A document
176+
* nobody in the union listed (it cannot happen for a persisted one, but the
177+
* map is the source of truth) grants nothing.
178+
*/
179+
export function persistedDocumentsByObserver(
180+
persisted: readonly PersistedDocument[],
181+
union: ReadonlyMap<string, Pick<UnionEntry, 'observers'>>
182+
): Map<string, string[]> {
183+
const byMember = new Map<string, string[]>()
184+
for (const { externalId, documentId } of persisted) {
185+
for (const memberId of union.get(externalId)?.observers ?? []) {
186+
const documentIds = byMember.get(memberId)
187+
if (documentIds) documentIds.push(documentId)
188+
else byMember.set(memberId, [documentId])
189+
}
190+
}
191+
return byMember
192+
}
193+
172194
interface UnionEntry {
173195
document: ExternalDocument
174196
/** Member ids whose listings returned the document, in listing order. */
@@ -1550,6 +1572,33 @@ export async function executeMemberSync(
15501572
},
15511573
lease: run.lease,
15521574
documentAccess: 'members',
1575+
/**
1576+
* Grants surface as each batch lands: every member who listed a document
1577+
* observes it the moment its row exists, and its ACL is materialised in
1578+
* the same lease-proved transaction. The listing's own pass below records
1579+
* the same observations again, idempotently, and is still what decides
1580+
* removals; this only brings the additions forward from the end of the
1581+
* run to the moment they are indexed.
1582+
*/
1583+
onBatchPersisted: async (persisted) => {
1584+
const byMember = persistedDocumentsByObserver(persisted, union)
1585+
if (byMember.size === 0) return
1586+
await withMemberLease(run, async (tx) => {
1587+
for (const [memberId, documentIds] of byMember) {
1588+
result.observationsAdded += await recordMemberObservations(
1589+
tx,
1590+
memberId,
1591+
documentIds,
1592+
runId
1593+
)
1594+
}
1595+
await materializeDocumentAcls(
1596+
connectorId,
1597+
persisted.map(({ documentId }) => documentId),
1598+
tx
1599+
)
1600+
})
1601+
},
15531602
})
15541603

15551604
const documentIdByExternalId = await loadDocumentIdsByExternalId(connectorId)

‎apps/sim/lib/knowledge/connectors/sync-primitives.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1496,6 +1496,12 @@ export interface ProcessDocOpsInput {
14961496
state: SyncRunState
14971497
hydration: DocOpHydration
14981498
lease: Pick<SyncRunLease, 'beatIfDue' | 'beatLive' | 'stillHeld'>
1499+
/**
1500+
* Runs after each batch's rows are written, with the documents that landed.
1501+
* A members-mode run grants access here, batch by batch, so a long first
1502+
* crawl becomes searchable as it goes rather than all at once at the end.
1503+
*/
1504+
onBatchPersisted?: (persisted: readonly PersistedDocument[]) => Promise<void>
14991505
/** Who may read the documents this pass writes. */
15001506
documentAccess: SyncDocumentAccess
15011507
}
@@ -1507,6 +1513,12 @@ export interface ProcessDocOpsInput {
15071513
* run so the connector backs off, and a lost lease, which ends it so no
15081514
* further write lands beside the replacement run's.
15091515
*/
1516+
/** A document a batch wrote, by the id the source knows it by and the id the row has. */
1517+
export interface PersistedDocument {
1518+
externalId: string
1519+
documentId: string
1520+
}
1521+
15101522
export async function processDocOps(input: ProcessDocOpsInput): Promise<void> {
15111523
const {
15121524
connectorId,
@@ -1744,10 +1756,15 @@ export async function processDocOps(input: ProcessDocOpsInput): Promise<void> {
17441756
if (leaseLost) throw leaseLost.reason
17451757

17461758
const batchDocs: DocumentData[] = []
1759+
const persisted: PersistedDocument[] = []
17471760
for (let j = 0; j < settled.length; j++) {
17481761
const outcome = settled[j]
17491762
if (outcome.status === 'fulfilled') {
17501763
batchDocs.push(outcome.value)
1764+
persisted.push({
1765+
externalId: batch[j].extDoc.externalId,
1766+
documentId: outcome.value.documentId,
1767+
})
17511768
if (batch[j].type === 'add') result.docsAdded++
17521769
else result.docsUpdated++
17531770
} else {
@@ -1761,6 +1778,8 @@ export async function processDocOps(input: ProcessDocOpsInput): Promise<void> {
17611778
}
17621779
}
17631780

1781+
if (persisted.length > 0) await input.onBatchPersisted?.(persisted)
1782+
17641783
if (batchDocs.length > 0) {
17651784
result.processingDispatch.requested += batchDocs.length
17661785
try {

0 commit comments

Comments
 (0)