@@ -21,6 +21,7 @@ import { deleteWorkspaceFixture } from '@sim/db/testing/workspace-fixtures'
2121import { sha256Hex } from '@sim/security/hash'
2222import { generateId } from '@sim/utils/id'
2323import { and , asc , eq , inArray , sql } from 'drizzle-orm'
24+ import JSZip from 'jszip'
2425import { afterAll , afterEach , beforeAll , describe , expect , it , vi } from 'vitest'
2526
2627const fixture = vi . hoisted ( ( ) => ( {
@@ -71,6 +72,7 @@ import {
7172 findKnowledgeImport ,
7273 insertKnowledgeImport ,
7374 KNOWLEDGE_IMPORT_STORAGE_CONTEXT ,
75+ stopKnowledgeImport ,
7476} from '@/lib/knowledge/transfer/import-jobs'
7577import { runKnowledgeImport } from '@/lib/knowledge/transfer/import-runner'
7678import { embeddingVectorColumn , embeddingVectorValues } from '@/lib/knowledge/vector-columns'
@@ -243,10 +245,11 @@ afterEach(() => {
243245 fixture . target = { model : 'text-embedding-3-small' , dimensions : 1536 }
244246} )
245247afterAll ( async ( ) => {
248+ const workspaceBases = sql `SELECT ${ knowledgeBase . id } FROM ${ knowledgeBase } WHERE ${ knowledgeBase . workspaceId } = ${ ids . workspaceId } `
246249 await db
247250 . delete ( outboxEvent )
248251 . where (
249- sql `${ outboxEvent . payload } ::jsonb ->> 'knowledgeBaseId' IN (SELECT ${ knowledgeBase . id } FROM ${ knowledgeBase } WHERE ${ knowledgeBase . workspaceId } = ${ ids . workspaceId } )`
252+ sql `${ outboxEvent . payload } ::jsonb ->> 'knowledgeBaseId' IN (${ workspaceBases } ) OR ${ outboxEvent . payload } ::jsonb ->> 'documentId' IN ( SELECT ${ document . id } FROM ${ document } WHERE ${ document . knowledgeBaseId } IN ( ${ workspaceBases } ) )`
250253 )
251254 await db . delete ( auditLog ) . where ( eq ( auditLog . workspaceId , ids . workspaceId ) )
252255 await db . delete ( knowledgeBase ) . where ( eq ( knowledgeBase . workspaceId , ids . workspaceId ) )
@@ -408,6 +411,77 @@ describe('knowledge base bundle import', () => {
408411 ) . rejects . toThrow ( )
409412 } )
410413
414+ it ( 'fails a bundle with a damaged chunk entry before creating anything' , async ( ) => {
415+ const zip = new JSZip ( )
416+ zip . file ( 'chunks/doc-1.ndjson' , '{"index": 0, "content": "half a line"\n' )
417+ zip . file (
418+ 'manifest.json' ,
419+ JSON . stringify ( {
420+ version : 1 ,
421+ exportedAt : '2026-10-01T00:00:00.000Z' ,
422+ embedding : { model : 'text-embedding-3-small' , dimension : 1536 , vectorsIncluded : false } ,
423+ knowledgeBase : {
424+ name : 'Damaged fixture' ,
425+ description : null ,
426+ chunkingConfig : { maxSize : 1024 , minSize : 1 , overlap : 20 } ,
427+ } ,
428+ tags : [ ] ,
429+ documents : [
430+ {
431+ id : 'doc-1' ,
432+ filename : 'notes.txt' ,
433+ mimeType : 'text/plain' ,
434+ fileSize : 0 ,
435+ enabled : true ,
436+ tags : { } ,
437+ file : null ,
438+ chunks : 'chunks/doc-1.ndjson' ,
439+ chunkCount : 1 ,
440+ tokenCount : 1 ,
441+ characterCount : 11 ,
442+ } ,
443+ ] ,
444+ } )
445+ )
446+ const importId = generateId ( )
447+ const sourceKey = await stageBundleBytes (
448+ importId ,
449+ await zip . generateAsync ( { type : 'nodebuffer' } )
450+ )
451+
452+ const record = await runImport ( importId , sourceKey , 'Damaged fixture' )
453+
454+ expect ( record ) . toMatchObject ( { status : 'failed' , knowledgeBaseId : null , documentsTotal : 0 } )
455+ expect ( record . error ) . toContain ( 'chunks/doc-1.ndjson line 1 is not valid JSON' )
456+ expect (
457+ await db
458+ . select ( { id : knowledgeBase . id } )
459+ . from ( knowledgeBase )
460+ . where ( eq ( knowledgeBase . name , 'Damaged fixture' ) )
461+ ) . toEqual ( [ ] )
462+ } )
463+
464+ it ( 'creates no base for an import canceled before its worker reached it' , async ( ) => {
465+ const importId = generateId ( )
466+ const sourceKey = await stageExportedBundle ( importId )
467+ const staged = await findKnowledgeImport ( { importId } )
468+ if ( ! staged ) throw new Error ( 'Import was not staged' )
469+ await stopKnowledgeImport ( staged )
470+
471+ const record = await runImport ( importId , sourceKey , 'Canceled fixture' )
472+
473+ expect ( record ) . toMatchObject ( { status : 'canceled' , knowledgeBaseId : null } )
474+ expect (
475+ await db
476+ . select ( { id : knowledgeBase . id } )
477+ . from ( knowledgeBase )
478+ . where ( eq ( knowledgeBase . name , 'Canceled fixture' ) )
479+ ) . toEqual ( [ ] )
480+ await expect (
481+ downloadFile ( { key : sourceKey , context : KNOWLEDGE_IMPORT_STORAGE_CONTEXT } )
482+ ) . rejects . toThrow ( )
483+ } )
484+
411485 it ( 'fails without a partial base when the name is already taken' , async ( ) => {
412486 const importId = generateId ( )
413487 const sourceKey = await stageExportedBundle ( importId )
@@ -459,7 +533,7 @@ describe('knowledge base import requests', () => {
459533 await writeLocalPutObject ( {
460534 uploadId : created . upload . id ,
461535 key : created . upload . finalKey ,
462- body : new Blob ( [ bytes ] ) . stream ( ) ,
536+ body : new Blob ( [ new Uint8Array ( bytes ) ] ) . stream ( ) ,
463537 expectedSize : bytes . length ,
464538 contentType : created . upload . contentType ,
465539 metadata : uploadSessionObjectMetadata ( created . upload ) ,
@@ -499,7 +573,12 @@ describe('knowledge base import requests', () => {
499573 db
500574 . select ( { resourceId : auditLog . resourceId } )
501575 . from ( auditLog )
502- . where ( eq ( auditLog . action , 'knowledge_base.imported' ) )
576+ . where (
577+ and (
578+ eq ( auditLog . workspaceId , ids . workspaceId ) ,
579+ eq ( auditLog . action , 'knowledge_base.imported' )
580+ )
581+ )
503582 )
504583 . toEqual ( [ { resourceId : finished . knowledgeBaseId } ] )
505584 } )
@@ -538,34 +617,62 @@ describe('knowledge base import requests', () => {
538617} )
539618
540619describe ( 'stale knowledge import cleanup' , ( ) => {
541- it ( 'fails an import whose worker went silent, deletes its bundle, and prunes long-settled imports' , async ( ) => {
620+ async function seedProcessingDocument ( processingQueueToken : string | null ) {
621+ const id = generateId ( )
622+ await db . insert ( document ) . values ( {
623+ id,
624+ knowledgeBaseId : ids . knowledgeBaseId ,
625+ filename : 'stranded.txt' ,
626+ fileUrl : '' ,
627+ fileSize : 0 ,
628+ mimeType : 'text/plain' ,
629+ processingStatus : 'processing' ,
630+ processingStartedAt : new Date ( Date . now ( ) - 2 * 60 * 60 * 1000 ) ,
631+ processingQueueToken,
632+ } )
633+ return id
634+ }
635+
636+ async function processingStatus ( documentId : string ) {
637+ const [ row ] = await db
638+ . select ( { status : document . processingStatus } )
639+ . from ( document )
640+ . where ( eq ( document . id , documentId ) )
641+ return row ?. status
642+ }
643+
644+ it ( 'fails an import whose worker went silent, its stranded documents, and its bundle, and prunes settled imports with theirs' , async ( ) => {
542645 const staleId = generateId ( )
543- const sourceKey = await stageBundleBytes ( staleId , Buffer . from ( 'bundle bytes' ) )
646+ const staleBundle = await stageBundleBytes ( staleId , Buffer . from ( 'bundle bytes' ) )
544647 const settledId = generateId ( )
545- await insertKnowledgeImport ( {
546- id : settledId ,
547- workspaceId : ids . workspaceId ,
548- userId : ids . aliceId ,
549- sourceKey : `knowledge-import/${ ids . workspaceId } /${ settledId } /bundle.simkb.zip` ,
550- } )
648+ const settledBundle = await stageBundleBytes ( settledId , Buffer . from ( 'bundle bytes' ) )
649+ const stranded = await seedProcessingDocument ( null )
650+ const queued = await seedProcessingDocument ( 'queue-token' )
551651 const longAgo = new Date ( Date . now ( ) - 30 * 24 * 60 * 60 * 1000 )
552652 await db
553653 . update ( knowledgeBaseImport )
554- . set ( { updatedAt : new Date ( Date . now ( ) - 2 * 60 * 60 * 1000 ) } )
654+ . set ( {
655+ knowledgeBaseId : ids . knowledgeBaseId ,
656+ updatedAt : new Date ( Date . now ( ) - 2 * 60 * 60 * 1000 ) ,
657+ } )
555658 . where ( eq ( knowledgeBaseImport . id , staleId ) )
556659 await db
557660 . update ( knowledgeBaseImport )
558- . set ( { status : 'completed ' , updatedAt : longAgo , completedAt : longAgo } )
661+ . set ( { status : 'canceled ' , updatedAt : longAgo , completedAt : longAgo } )
559662 . where ( eq ( knowledgeBaseImport . id , settledId ) )
560663
561664 await runCleanupStaleExecutions ( )
562665
563666 const stale = await findKnowledgeImport ( { importId : staleId } )
564667 expect ( stale ) . toMatchObject ( { status : 'failed' } )
565668 expect ( stale ?. error ) . toContain ( 'no progress' )
566- await expect (
567- downloadFile ( { key : sourceKey , context : KNOWLEDGE_IMPORT_STORAGE_CONTEXT } )
568- ) . rejects . toThrow ( )
669+ expect ( await processingStatus ( stranded ) ) . toBe ( 'failed' )
670+ expect ( await processingStatus ( queued ) ) . toBe ( 'processing' )
569671 expect ( await findKnowledgeImport ( { importId : settledId } ) ) . toBeNull ( )
672+ for ( const key of [ staleBundle , settledBundle ] ) {
673+ await expect (
674+ downloadFile ( { key, context : KNOWLEDGE_IMPORT_STORAGE_CONTEXT } )
675+ ) . rejects . toThrow ( )
676+ }
570677 } )
571678} )
0 commit comments