diff --git a/CHANGELOG.md b/CHANGELOG.md index 697723c..39355b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Interrupted indexing resume**: Interrupted indexing runs now persist incremental checkpoints (database, vectors, BM25, failed batches, and file hashes), so a subsequent run resumes incrementally instead of re-embedding the whole project. Interrupted global clears and force-indexes are completed on recovery (scoped to the current project when foreign data is present), and checkpointed failed batches and pending retries survive for later retry runs. Checkpoint persistence orders the BM25 index before the vector store so a crash between the two cannot orphan chunks from keyword search. Project-scope dead-lease recovery preserves checkpointed artifacts for incremental resume instead of resetting the local index. Pending retry chunks are no longer duplicated across multiple checkpoints, empty failed-batch state is published before new file hashes, and finalization deduplicates failed-batch records by chunk ID (keeping the highest attempt count). +- **Interrupted recovery safety**: Recovery of an interrupted global clear or clearing-phase force-index now replays the clear against the originating project scope persisted with the lease, so a project reclaiming a dead lease can no longer delete another project's indexed data. The destructive phase and effective embedding configuration are bound to the lease owner; legacy or configuration-mismatched clears fail closed and retain their recovery marker instead of applying the reclaiming project's settings. Retry runs restore SQLite chunk rows that a recovery health check may have collected as orphans, so a branch reference can never point at a chunk without metadata. +- **Forced re-embed migration safety**: While a project-scoped forced re-embed is pending, checkpoints no longer stamp new embedding metadata or a compatibility certificate, unchanged scope files are re-embedded instead of skipped, and `retryFailedBatches` only clears the pending migration after the main run has committed its new metadata. This prevents an interrupted migration from resuming into a silent mix of old and new vector spaces. - **Conceptual context retrieval**: Prefer implementation paths over tests and documentation for code-focused `codebase_context` searches, without applying a hard source-only filter or changing documentation and test queries. - **Scoped definition lookup**: Allow explicit definition searches constrained to a file type or directory to return matching declarations in fixtures and test paths without weakening source-first ranking for ordinary searches. diff --git a/src/indexer/index-lock.ts b/src/indexer/index-lock.ts index 8363035..e5e27ae 100644 --- a/src/indexer/index-lock.ts +++ b/src/indexer/index-lock.ts @@ -28,6 +28,23 @@ export interface IndexLockOwner { startedAt: string; operation: IndexMutationOperation; token: string; + /** Recovery protocol written by owners that persist destructive phase state. */ + recoveryProtocolVersion?: 1; + /** Originating project root for interrupted global clears (recovery scope). */ + projectRoot?: string; + /** Originating scoped roots for interrupted global clears (recovery scope). */ + scopedRoots?: string[]; + /** Destructive clear state, present only while a clear is in progress. */ + clearRecovery?: IndexLockClearRecoveryState; +} + +export interface IndexLockClearRecoveryState { + phase: "clearing"; + embeddingProvider: string; + embeddingModel: string; + embeddingDimensions: number; + embeddingStrategyVersion: string; + compatibilityDecision: "compatible" | "embedding-strategy-mismatch" | "incompatible"; } export interface IndexLockRecovery { @@ -100,6 +117,37 @@ function parseOwner(value: unknown): IndexLockOwner | null { if (typeof candidate.startedAt !== "string" || Number.isNaN(Date.parse(candidate.startedAt))) return null; if (typeof candidate.operation !== "string" || !VALID_OPERATIONS.has(candidate.operation as IndexMutationOperation)) return null; if (typeof candidate.token !== "string" || !UUID_PATTERN.test(candidate.token)) return null; + if (candidate.recoveryProtocolVersion !== undefined && candidate.recoveryProtocolVersion !== 1) return null; + if (candidate.projectRoot !== undefined && typeof candidate.projectRoot !== "string") return null; + if (candidate.scopedRoots !== undefined) { + if (!Array.isArray(candidate.scopedRoots) || candidate.scopedRoots.some((root) => typeof root !== "string")) { + return null; + } + } + if (candidate.clearRecovery !== undefined) { + const recovery = candidate.clearRecovery as Partial; + if ( + typeof recovery !== "object" + || recovery === null + || recovery.phase !== "clearing" + || typeof recovery.embeddingProvider !== "string" + || recovery.embeddingProvider.length === 0 + || typeof recovery.embeddingModel !== "string" + || recovery.embeddingModel.length === 0 + || !Number.isInteger(recovery.embeddingDimensions) + || (recovery.embeddingDimensions ?? 0) <= 0 + || typeof recovery.embeddingStrategyVersion !== "string" + || recovery.embeddingStrategyVersion.length === 0 + || ( + recovery.compatibilityDecision !== "compatible" + && recovery.compatibilityDecision !== "embedding-strategy-mismatch" + && recovery.compatibilityDecision !== "incompatible" + ) + || (candidate.operation !== "clear" && candidate.operation !== "force-index") + ) { + return null; + } + } return candidate as IndexLockOwner; } @@ -224,6 +272,11 @@ function createOwner(operation: IndexMutationOperation): IndexLockOwner { }; } +export interface IndexLockRecoveryScope { + projectRoot: string; + scopedRoots: string[]; +} + function recoveryMarkerPath(indexPath: string, owner: IndexLockOwner): string { return path.join(indexPath, `${RECOVERY_MARKER_PREFIX}${owner.token}`); } @@ -395,14 +448,25 @@ export function isTransientIndexLockContention(error: unknown): boolean { return error.reason === "active" || error.reason === "reclaiming"; } -export function acquireIndexLock(indexPath: string, operation: IndexMutationOperation): IndexLockLease { +export function acquireIndexLock( + indexPath: string, + operation: IndexMutationOperation, + recoveryScope?: IndexLockRecoveryScope, +): IndexLockLease { mkdirSync(indexPath, { recursive: true }); const canonicalIndexPath = realpathSync.native(indexPath); const lockPath = path.join(canonicalIndexPath, "indexing.lock"); cleanupDeadPublicationCandidates(canonicalIndexPath); for (let attempt = 0; attempt < 6; attempt += 1) { - const owner = createOwner(operation); + const owner = recoveryScope === undefined + ? createOwner(operation) + : { + ...createOwner(operation), + recoveryProtocolVersion: 1 as const, + projectRoot: recoveryScope.projectRoot, + scopedRoots: recoveryScope.scopedRoots, + }; if (publishJsonDirectory(lockPath, owner)) { const lease: IndexLockLease = { canonicalIndexPath, @@ -478,13 +542,46 @@ export function releaseIndexLock(lease: IndexLockLease): boolean { return true; } +export function setIndexLockClearRecoveryState( + lease: IndexLockLease, + clearRecovery: IndexLockClearRecoveryState | null, +): void { + const currentOwner = readDirectoryOwner(lease.lockPath); + if (!currentOwner || !sameOwner(currentOwner, lease.owner)) { + throw new Error(`Lost ownership of index mutation lease ${lease.owner.token}`); + } + + const nextOwner: IndexLockOwner = { ...currentOwner }; + if (clearRecovery === null) { + delete nextOwner.clearRecovery; + } else { + nextOwner.clearRecovery = clearRecovery; + } + + const ownerPath = path.join(lease.lockPath, OWNER_FILE_NAME); + const temporaryPath = path.join( + lease.lockPath, + `${OWNER_FILE_NAME}.tmp.${lease.owner.pid}.${lease.owner.token}.${randomUUID()}`, + ); + try { + writeFileSync(temporaryPath, JSON.stringify(nextOwner), { + encoding: "utf-8", + flag: "wx", + mode: 0o600, + }); + retryTransientFilesystemOperation(() => renameSync(temporaryPath, ownerPath)); + } finally { + if (existsSync(temporaryPath)) rmSync(temporaryPath, { force: true }); + } +} + export async function withIndexLock( indexPath: string, operation: IndexMutationOperation, callback: (lease: IndexLockLease) => Promise | T, - options: { completeRecoveries?: boolean } = {}, + options: { completeRecoveries?: boolean; recoveryScope?: IndexLockRecoveryScope } = {}, ): Promise { - const lease = acquireIndexLock(indexPath, operation); + const lease = acquireIndexLock(indexPath, operation, options.recoveryScope); let result: T | undefined; let callbackError: unknown; let callbackFailed = false; diff --git a/src/indexer/index.ts b/src/indexer/index.ts index 112d9ff..1f165be 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -84,6 +84,8 @@ import { recoverLeaseArtifacts, releaseIndexLock, removeLeaseTemporaryPath, + setIndexLockClearRecoveryState, + type IndexLockClearRecoveryState, type IndexLockLease, type IndexLockOwner, type IndexMutationOperation, @@ -102,6 +104,7 @@ import { import { createFailedBatchWriter, readFailedBatchRecords, + writeFailedBatchRecords, type FailedBatchRecordInput, type FailedBatchWriter, } from "./failed-state-persistence.js"; @@ -345,6 +348,8 @@ export interface IndexerRuntimeOptions { indexPath?: string; /** Internal test and benchmark override. Production uses the fixed limits. */ fileBatchLimits?: FileBatchLimits; + /** Internal test and benchmark override. Production uses a fixed default. */ + checkpointIntervalChunks?: number; } export interface BranchIndexResult { @@ -491,6 +496,7 @@ interface FailedChunkRecordMetadata { attemptCount: number; error: string; lastAttempt: string; + chunks: unknown[]; } interface FailedBatchWriteState { @@ -498,6 +504,13 @@ interface FailedBatchWriteState { recordsWritten: number; } +interface FailedBatchProcessingState { + state: FailedBatchWriteState; + latestById: Map; + materializedRetryIds: Set; + discardedExistingRecords: boolean; +} + interface EmbeddingRateLimitState { backoffMs: number; } @@ -1172,6 +1185,7 @@ export class Indexer { private writerArtifactFingerprint: ReaderArtifactFingerprint | null = null; private readerArtifactRetryAfter = new Map(); private readonly fileBatchLimits?: FileBatchLimits; + private readonly checkpointIntervalChunks?: number; constructor( projectRoot: string, @@ -1180,7 +1194,7 @@ export class Indexer { runtimeOptions: IndexerRuntimeOptions = {}, ) { this.projectRoot = projectRoot; - this.projectIdentityHash = hashContent(this.getCanonicalPath(projectRoot)).slice(0, 16); + this.projectIdentityHash = this.getProjectIdentityHash(projectRoot); this.materializedProjectRoot = runtimeOptions.materializedProjectRoot ?? projectRoot; this.branchNameOverride = runtimeOptions.branchName; this.catalogIdentityOverride = runtimeOptions.catalogIdentity; @@ -1190,6 +1204,7 @@ export class Indexer { this.expectedCommitOverride = runtimeOptions.expectedCommit?.toLowerCase(); this.indexPathOverride = runtimeOptions.indexPath; this.fileBatchLimits = runtimeOptions.fileBatchLimits; + this.checkpointIntervalChunks = runtimeOptions.checkpointIntervalChunks; this.config = config; this.host = host; if (isGitRepo(this.materializedProjectRoot)) { @@ -1327,6 +1342,10 @@ export class Indexer { } } + private getProjectIdentityHash(projectRoot: string): string { + return hashContent(this.getCanonicalPath(projectRoot)).slice(0, 16); + } + private isProjectOwnedIndexPath(): boolean { return isProjectIndexPathOwnedByProject(this.projectRoot, this.indexPath, this.host); } @@ -1369,7 +1388,10 @@ export class Indexer { callback: (recoveredOwners: readonly IndexLockOwner[]) => Promise, ): Promise { this.refreshBranchInfo(); - const lease = acquireIndexLock(this.indexPath, operation); + const lease = acquireIndexLock(this.indexPath, operation, { + projectRoot: this.projectRoot, + scopedRoots: this.getScopedRoots(), + }); this.indexPath = lease.canonicalIndexPath; this.refreshRuntimeArtifactPaths(); this.activeIndexLease = lease; @@ -1427,6 +1449,7 @@ export class Indexer { private loadFileHashCache(): void { if (!existsSync(this.fileHashCachePath)) { + this.fileHashCache = new Map(); return; } @@ -1471,11 +1494,11 @@ export class Indexer { ); } - private getScopedRoots(): string[] { - const roots = new Set([this.getCanonicalPath(this.projectRoot)]); + private getScopedRoots(projectRoot = this.projectRoot): string[] { + const roots = new Set([this.getCanonicalPath(projectRoot)]); for (const kbRoot of this.config.knowledgeBases) { - roots.add(this.getCanonicalPath(path.resolve(this.projectRoot, kbRoot))); + roots.add(this.getCanonicalPath(path.resolve(projectRoot, kbRoot))); } return Array.from(roots); @@ -1577,16 +1600,20 @@ export class Indexer { return this.currentBranch || "default"; } - private getLegacyMigrationMetadataKey(): string { - return `index.globalBranchMigration.${this.projectIdentityHash}`; + private getLegacyMigrationMetadataKey(projectIdentityHash = this.projectIdentityHash): string { + return `index.globalBranchMigration.${projectIdentityHash}`; } - private getProjectEmbeddingStrategyMetadataKey(): string { - return `index.embeddingStrategyVersion.${this.projectIdentityHash}`; + private getProjectEmbeddingStrategyMetadataKey(projectIdentityHash = this.projectIdentityHash): string { + return `index.embeddingStrategyVersion.${projectIdentityHash}`; } - private getProjectForceReembedMetadataKey(): string { - return `index.forceReembed.${this.projectIdentityHash}`; + private getProjectForceReembedMetadataKey(projectIdentityHash = this.projectIdentityHash): string { + return `index.forceReembed.${projectIdentityHash}`; + } + + private getProjectMigrationFinalizedMetadataKey(projectIdentityHash = this.projectIdentityHash): string { + return `index.migrationFinalized.${projectIdentityHash}`; } private getBranchMigrationMetadataKey( @@ -1743,7 +1770,7 @@ export class Indexer { return primary === legacy ? [primary] : [primary, legacy]; } - private getProjectLocalScopedOwnershipIds(roots: string[]): { + private getProjectLocalScopedOwnershipIds(roots: string[], projectRoot = this.projectRoot): { chunkIds: Set; symbolIds: Set; } { @@ -1755,12 +1782,12 @@ export class Indexer { const projectLocalFilePaths = new Set([ ...Array.from(this.fileHashCache.keys()).filter( - (filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath) + (filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot) ), ...(this.store?.getAllMetadata() ?? []) .map(({ metadata }) => metadata.filePath) .filter( - (filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath) + (filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot) ), ]); @@ -1777,7 +1804,11 @@ export class Indexer { return { chunkIds, symbolIds }; } - private getProjectScopedBranchCatalogCleanupKeys(projectChunkIds: string[], projectSymbolIds: string[]): string[] { + private getProjectScopedBranchCatalogCleanupKeys( + projectChunkIds: string[], + projectSymbolIds: string[], + projectRoot = this.projectRoot, + ): string[] { if (this.config.scope !== "global") { return this.getBranchCatalogCleanupKeys(); } @@ -1785,9 +1816,10 @@ export class Indexer { const keys = new Set(); const projectChunkIdSet = new Set(projectChunkIds); const projectSymbolIdSet = new Set(projectSymbolIds); + const projectIdentityHash = this.getProjectIdentityHash(projectRoot); for (const branchKey of this.database?.getAllBranches() ?? []) { - if (branchKey.startsWith(`${this.projectIdentityHash}:`)) { + if (branchKey.startsWith(`${projectIdentityHash}:`)) { keys.add(branchKey); continue; } @@ -1799,8 +1831,10 @@ export class Indexer { } } - for (const branchKey of this.getBranchCatalogCleanupKeys()) { - keys.add(branchKey); + if (projectRoot === this.projectRoot) { + for (const branchKey of this.getBranchCatalogCleanupKeys()) { + keys.add(branchKey); + } } return Array.from(keys); @@ -1811,10 +1845,10 @@ export class Indexer { return roots.some((root) => isPathWithinRoot(canonicalFilePath, root)); } - private isFileInProjectRoot(filePath: string): boolean { + private isFileInProjectRoot(filePath: string, projectRoot = this.projectRoot): boolean { return isPathWithinRoot( this.getCanonicalStoredFilePath(filePath), - this.getCanonicalPath(this.projectRoot), + this.getCanonicalPath(projectRoot), ); } @@ -1864,13 +1898,16 @@ export class Indexer { return false; } - private hasForeignScopedBranchData(): boolean { + private hasForeignScopedBranchData( + projectRoot = this.projectRoot, + roots = this.getScopedRoots(projectRoot), + ): boolean { if (!this.database || this.config.scope !== "global") { return false; } - const roots = this.getScopedRoots(); - const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots); + const projectIdentityHash = this.getProjectIdentityHash(projectRoot); + const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots, projectRoot); return this.database.getAllBranches().some( (branchKey) => { @@ -1881,7 +1918,7 @@ export class Indexer { return false; } - if (branchKey.startsWith(`${this.projectIdentityHash}:`)) { + if (branchKey.startsWith(`${projectIdentityHash}:`)) { return false; } @@ -1896,7 +1933,8 @@ export class Indexer { store: VectorStore, invertedIndex: InvertedIndex, database: Database, - roots: string[] + roots: string[], + projectRoot = this.projectRoot, ): { removedChunkIds: string[]; hasForeignData: boolean } { const allMetadata = store.getAllMetadata(); const scopedEntries = allMetadata.filter(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots)); @@ -1906,7 +1944,7 @@ export class Indexer { ]); const projectLocalFilePaths = new Set( - Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath)) + Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath, projectRoot)) ); const removedChunkIds = new Set(scopedEntries.map(({ key }) => key)); @@ -1919,7 +1957,7 @@ export class Indexer { const projectLocalChunkIds = new Set( scopedEntries - .filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath)) + .filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath, projectRoot)) .map(({ key }) => key) ); for (const filePath of projectLocalFilePaths) { @@ -1942,6 +1980,7 @@ export class Indexer { const branchCleanupKeys = this.getProjectScopedBranchCatalogCleanupKeys( Array.from(projectLocalChunkIds), Array.from(projectLocalSymbolIds), + projectRoot, ); for (const branchKey of branchCleanupKeys) { database.deleteBranchChunksForBranch(branchKey, removedChunkIdList); @@ -1984,8 +2023,12 @@ export class Indexer { database.gcOrphanEmbeddings(); database.gcOrphanChunks(); - store.save(); + // Persist the keyword index before the vector store: a crash between the + // two leaves the store as the conservative resume authority, so the next + // run re-embeds and repopulates BM25 instead of skipping addChunk for + // chunks whose vectors are already durable. this.saveInvertedIndex(invertedIndex); + store.save(); return { removedChunkIds: removedChunkIdList, @@ -1993,6 +2036,52 @@ export class Indexer { }; } + private getCurrentClearRecoveryState(): IndexLockClearRecoveryState { + if (!this.configuredProviderInfo) { + throw new Error("Cannot persist clear recovery state before the embedding provider is initialized"); + } + const compatibility = this.checkCompatibility(); + const compatibilityDecision = compatibility.compatible + ? "compatible" + : compatibility.code === IncompatibilityCode.EMBEDDING_STRATEGY_MISMATCH + ? "embedding-strategy-mismatch" + : "incompatible"; + return { + phase: "clearing", + embeddingProvider: this.configuredProviderInfo.provider, + embeddingModel: this.configuredProviderInfo.modelInfo.model, + embeddingDimensions: this.configuredProviderInfo.modelInfo.dimensions, + embeddingStrategyVersion: EMBEDDING_STRATEGY_VERSION, + compatibilityDecision, + }; + } + + private beginClearRecoveryState(): IndexLockClearRecoveryState { + const recovery = this.getCurrentClearRecoveryState(); + setIndexLockClearRecoveryState(this.requireActiveLease(), recovery); + return recovery; + } + + private finishClearRecoveryState(): void { + setIndexLockClearRecoveryState(this.requireActiveLease(), null); + } + + private matchesCurrentClearRecoveryConfiguration(recovery: IndexLockClearRecoveryState): boolean { + const configuredProviderInfo = this.configuredProviderInfo; + return configuredProviderInfo !== null + && recovery.embeddingProvider === configuredProviderInfo.provider + && recovery.embeddingModel === configuredProviderInfo.modelInfo.model + && recovery.embeddingDimensions === configuredProviderInfo.modelInfo.dimensions + && recovery.embeddingStrategyVersion === EMBEDDING_STRATEGY_VERSION; + } + + private hasUnknownLegacyForceIndexClear(owner: IndexLockOwner): boolean { + return owner.operation === "force-index" + && owner.clearRecovery === undefined + && owner.recoveryProtocolVersion !== 1 + && existsSync(path.join(this.indexPath, "force-index-phase")); + } + private async recoverFromInterruptedIndexingUnlocked(owners: readonly IndexLockOwner[]): Promise { for (const owner of owners) { this.logger.warn("Detected interrupted indexing session, recovering...", { @@ -2000,18 +2089,76 @@ export class Indexer { hostname: owner.hostname, operation: owner.operation, startedAt: owner.startedAt, + projectRoot: owner.projectRoot, }); } if (this.config.scope === "global") { - if (existsSync(this.fileHashCachePath)) { - unlinkSync(this.fileHashCachePath); + const clearScopes: Array<{ + projectRoot: string; + scopedRoots: string[]; + compatibilityDecision: IndexLockClearRecoveryState["compatibilityDecision"]; + }> = []; + for (const owner of owners) { + if (this.hasUnknownLegacyForceIndexClear(owner)) { + throw new Error( + `Cannot automatically recover interrupted force-index ${owner.token}: ` + + "the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection." + ); + } + if ( + owner.operation === "clear" + && owner.clearRecovery === undefined + && owner.recoveryProtocolVersion !== 1 + ) { + throw new Error( + `Cannot automatically recover interrupted global clear ${owner.token}: ` + + "the originating recovery state is unknown. The recovery marker was retained for manual inspection." + ); + } + if (owner.clearRecovery === undefined) continue; + if (!owner.projectRoot || !owner.scopedRoots || owner.scopedRoots.length === 0) { + throw new Error( + `Cannot automatically recover interrupted global clear ${owner.token}: ` + + "the originating project scope is unknown. The recovery marker was retained for manual inspection." + ); + } + if (!this.matchesCurrentClearRecoveryConfiguration(owner.clearRecovery)) { + throw new Error( + `Cannot automatically recover interrupted global clear ${owner.token}: ` + + "the current embedding configuration does not match the originating lease. " + + "The recovery marker was retained; retry from the originating project with matching settings." + ); + } + clearScopes.push({ + projectRoot: owner.projectRoot, + scopedRoots: owner.scopedRoots, + compatibilityDecision: owner.clearRecovery.compatibilityDecision, + }); + } + if (clearScopes.length > 0) { + // The scoped clear purges the file-hash cache entries of the + // originating project: load the persisted cache first so the purge + // is written back instead of being lost on an empty in-memory map. + this.loadFileHashCache(); + } + for (const { projectRoot, scopedRoots, compatibilityDecision } of clearScopes) { + // Re-apply the clear decision against the originating project scope: + // a global clear only wipes the whole shared index when no foreign + // project data is present, otherwise it stays scoped to the project + // that started the clear. + this.clearGlobalIndexUnlocked(projectRoot, scopedRoots, compatibilityDecision); } - await this.healthCheckUnlocked(); + this.logger.info( + clearScopes.length > 0 + ? "Recovery complete, next index will rebuild all files" + : "Recovery complete, next index will resume from the last checkpoint", + ); + return; } - this.logger.info("Recovery complete, next index will re-process all files"); + this.logger.info("Recovery complete, next index will resume from the last checkpoint"); } private *loadSerializedFailedBatches(): Generator { @@ -2057,16 +2204,132 @@ export class Indexer { state.recordsWritten += record.chunks.length; } - private finalizeFailedBatchWriteState(state: FailedBatchWriteState): void { + private finalizeFailedBatchWriteState( + state: FailedBatchWriteState, + resolvedChunkIds: ReadonlySet = new Set(), + ): void { if (state.recordsWritten > 0) { - state.writer.commit(); + // Deduplicate by chunk ID and drop resolved retries. Process in reverse + // so the last-written record (highest attemptCount) wins; the + // checkpoint reconstruction loop and the retry phase can both + // materialize the same pending retry. + const seenChunkIds = new Set(); + const retained: FailedBatchRecordInput[] = []; + const records = Array.from(readFailedBatchRecords(state.writer.temporaryPath)); + for (let i = records.length - 1; i >= 0; i--) { + const chunks = records[i].chunks.filter((rawChunk) => { + const chunkId = getPendingChunkId(rawChunk); + if (chunkId !== null) { + if (resolvedChunkIds.has(chunkId)) return false; + if (seenChunkIds.has(chunkId)) return false; + seenChunkIds.add(chunkId); + } + return true; + }); + if (chunks.length > 0) { + retained.unshift({ ...records[i], chunks }); + } + } + state.writer.cleanup(); + if (retained.length > 0) { + writeFailedBatchRecords(this.failedBatchesPath, retained); + } else { + writeFailedBatchRecords(this.failedBatchesPath, []); + this.clearFailedBatchState(); + } return; } - state.writer.cleanup(); + state.writer.commit(); this.clearFailedBatchState(); } + private getCheckpointIntervalChunks(totalChunks: number): number { + return Math.max( + this.checkpointIntervalChunks ?? 2000, + Math.floor(totalChunks / 10), + ); + } + + private checkpointIndexRun( + database: Database, + store: VectorStore, + invertedIndex: InvertedIndex, + failedProcessing: FailedBatchProcessingState, + resolvedRetryChunkIds: ReadonlySet, + currentFileHashes: Map, + committedFilePaths: Set, + scopedRoots: string[] | null, + configuredProviderInfo: ConfiguredProviderInfo, + ): void { + if (!this.hasProjectForceReembedPending()) { + this.saveIndexMetadata(configuredProviderInfo); + this.indexCompatibility = { compatible: true }; + } + database.commitWriteTransaction(); + database.beginWriteTransaction(); + // Persist the keyword index before the vector store: a crash between the + // two leaves the store as the conservative resume authority, so the next + // run re-embeds and repopulates BM25 instead of skipping addChunk for + // chunks whose vectors are already durable. + this.saveInvertedIndex(invertedIndex); + store.save(); + if ( + failedProcessing.state.recordsWritten > 0 + || failedProcessing.latestById.size > 0 + || failedProcessing.discardedExistingRecords + ) { + // Persist the pending retries alongside the written records so a crash + // after this checkpoint cannot lose them. + for (const metadata of failedProcessing.latestById.values()) { + const alreadyMaterialized = metadata.chunks.some((rawChunk) => { + const chunkId = getPendingChunkId(rawChunk); + return chunkId !== null && failedProcessing.materializedRetryIds.has(chunkId); + }); + if (alreadyMaterialized) continue; + this.writeFailedBatchRecord(failedProcessing.state, { + chunks: metadata.chunks, + attemptCount: metadata.attemptCount, + error: metadata.error, + lastAttempt: metadata.lastAttempt, + }); + for (const rawChunk of metadata.chunks) { + const chunkId = getPendingChunkId(rawChunk); + if (chunkId !== null) { + failedProcessing.materializedRetryIds.add(chunkId); + } + } + } + this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds); + failedProcessing.state = this.createFailedBatchWriteState(); + failedProcessing.discardedExistingRecords = false; + // Preserve the committed records (including out-of-scope projects' + // failed batches) so finalization never deletes them. + for (const record of this.loadSerializedFailedBatches()) { + for (const rawChunk of record.chunks) { + const chunkId = getPendingChunkId(rawChunk); + this.writeFailedBatchRecord(failedProcessing.state, { ...record, chunks: [rawChunk] }); + if (chunkId !== null) { + failedProcessing.materializedRetryIds.add(chunkId); + } + } + } + } + const partialHashes = new Map(); + for (const filePath of committedFilePaths) { + const hash = currentFileHashes.get(filePath); + if (hash !== undefined) { + partialHashes.set(filePath, hash); + } + } + if (scopedRoots) { + this.replaceScopedFileHashCache(partialHashes, scopedRoots); + } else { + this.fileHashCache = partialHashes; + this.saveFileHashCache(); + } + } + private clearFailedBatchState(): void { if (existsSync(this.failedBatchesPath)) { try { @@ -2096,9 +2359,10 @@ export class Indexer { private prepareFailedBatchProcessing( roots: string[] | null, shouldProcess: (filePath: string | null) => boolean, - ): { state: FailedBatchWriteState; latestById: Map } { + ): FailedBatchProcessingState { const state = this.createFailedBatchWriteState(); const latestById = new Map(); + let discardedExistingRecords = false; try { for (const batch of this.loadSerializedFailedBatches()) { @@ -2110,11 +2374,13 @@ export class Indexer { continue; } if (!shouldProcess(filePath)) { + discardedExistingRecords = true; continue; } const chunkId = getPendingChunkId(rawChunk); if (!chunkId) { + discardedExistingRecords = true; continue; } const existing = latestById.get(chunkId); @@ -2123,11 +2389,17 @@ export class Indexer { attemptCount: batch.attemptCount, error: batch.error, lastAttempt: batch.lastAttempt, + chunks: [rawChunk], }); } } } - return { state, latestById }; + return { + state, + latestById, + materializedRetryIds: new Set(), + discardedExistingRecords, + }; } catch (error) { state.writer.cleanup(); throw error; @@ -2177,6 +2449,33 @@ export class Indexer { } } + private restoreMissingChunkRows(database: Database, chunks: readonly PendingChunk[]): void { + const missing: ChunkData[] = []; + for (const chunk of chunks) { + if (database.getChunk(chunk.id)) { + continue; + } + missing.push({ + chunkId: chunk.id, + contentHash: chunk.contentHash, + filePath: chunk.metadata.filePath, + startLine: chunk.metadata.startLine, + endLine: chunk.metadata.endLine, + nodeType: chunk.metadata.chunkType, + name: chunk.metadata.name, + language: chunk.metadata.language, + blameSha: chunk.metadata.blameSha, + blameAuthor: chunk.metadata.blameAuthor, + blameAuthorEmail: chunk.metadata.blameAuthorEmail, + blameCommittedAt: chunk.metadata.blameCommittedAt, + blameSummary: chunk.metadata.blameSummary, + }); + } + if (missing.length > 0) { + database.upsertChunksBatch(missing); + } + } + private getProviderRateLimits(provider: string): { concurrency: number; intervalMs: number; @@ -3030,7 +3329,22 @@ export class Indexer { ]); } if (recoveredOwners.length > 0 && this.config.scope === "project") { - await this.resetLocalIndexArtifacts(); + const unknownLegacyForceIndex = recoveredOwners.find( + (owner) => this.hasUnknownLegacyForceIndexClear(owner), + ); + if (unknownLegacyForceIndex) { + throw new Error( + `Cannot automatically recover interrupted force-index ${unknownLegacyForceIndex.token}: ` + + "the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection." + ); + } + const shouldReset = recoveredOwners.some( + (owner) => owner.clearRecovery !== undefined + || (owner.operation === "clear" && owner.recoveryProtocolVersion !== 1), + ); + if (shouldReset) { + await this.resetLocalIndexArtifacts(); + } } this.store = new VectorStore(storePath, dimensions); @@ -3862,9 +4176,12 @@ export class Indexer { reparseCachedSwiftFiles && path.extname(storedPath).toLowerCase() === ".swift"; const requiresMetalParserUpgrade = reparseCachedMetalFiles && path.extname(storedPath).toLowerCase() === ".metal"; + const inMigrationScope = + forceScopedReembed && scopedRoots !== null && this.isFileInCurrentScope(storedPath, scopedRoots); if ( cachedHashMatches && + !inMigrationScope && !needsCallGraphRefresh && !requiresSwiftParserUpgrade && !requiresMetalParserUpgrade && @@ -4013,6 +4330,9 @@ export class Indexer { } let processedChangedFiles = 0; + let lastCheckpointChunks = 0; + const committedFilePaths = new Set(unchangedFilePaths); + const resolvedRetryChunkIds = new Set(); for (const descriptorBatch of iterateOrderedFileBatches( changedFileDescriptors, (descriptor) => descriptor.sourceBytes, @@ -4206,6 +4526,10 @@ export class Indexer { } if (symbolBatch.length > 0) { database.upsertSymbolsBatch(symbolBatch); + database.addSymbolsToBranchBatch( + this.getBranchCatalogKey(), + symbolBatch.map((symbol) => symbol.id), + ); } if (edgeBatch.length > 0) { database.upsertCallEdgesBatch(edgeBatch); @@ -4243,6 +4567,12 @@ export class Indexer { forceReembed: forceScopedReembed, reuseCachedEmbeddings: true, incrementRepeatedFailures: true, + onSucceeded: (succeededChunks) => { + database.addChunksToBranchBatch( + this.getBranchCatalogKey(), + succeededChunks.map((chunk) => chunk.id), + ); + }, onProgress: (batchProgress) => onProgress?.({ phase: "embedding", filesProcessed: unchangedFilePaths.size + processedChangedFiles, @@ -4261,6 +4591,28 @@ export class Indexer { } } } + + for (const descriptor of descriptorBatch) { + const existingFileChunks = existingChunksByFile.get(descriptor.storedPath); + if (!existingFileChunks || existingFileChunks.size === 0) { + committedFilePaths.add(descriptor.storedPath); + } + } + const checkpointInterval = this.getCheckpointIntervalChunks(stats.totalChunks); + if (stats.totalChunks - lastCheckpointChunks >= checkpointInterval) { + lastCheckpointChunks = stats.totalChunks; + this.checkpointIndexRun( + database, + store, + invertedIndex, + failedProcessing, + resolvedRetryChunkIds, + currentFileHashes, + committedFilePaths, + scopedRoots, + configuredProviderInfo, + ); + } } const retryableFailedChunks = this.iterateLatestFailedChunks( @@ -4282,6 +4634,12 @@ export class Indexer { retryableChunksWithExistingData.add(chunk.id); } } + // A failed chunk checkpointed before its embedding attempt has a + // committed SQLite row only when the parsing phase upserted it; a + // crash before that checkpoint can leave the row missing while the + // failed-batches record survives. Restore the row so the retry does + // not leave a branch reference without chunk metadata. + this.restoreMissingChunkRows(database, pendingChunks); stats.totalChunks += pendingChunks.length; onProgress?.({ phase: "embedding", @@ -4304,6 +4662,16 @@ export class Indexer { forceReembed: forceScopedReembed, reuseCachedEmbeddings: true, incrementRepeatedFailures: true, + onSucceeded: (succeededChunks) => { + database.addChunksToBranchBatch( + this.getBranchCatalogKey(), + succeededChunks.map((chunk) => chunk.id), + ); + for (const chunk of succeededChunks) { + failedProcessing.latestById.delete(chunk.id); + resolvedRetryChunkIds.add(chunk.id); + } + }, onProgress: (batchProgress) => onProgress?.({ phase: "embedding", filesProcessed: files.length, @@ -4321,6 +4689,20 @@ export class Indexer { failedForcedChunkIds.add(chunkId); } } + if (stats.totalChunks - lastCheckpointChunks >= this.getCheckpointIntervalChunks(stats.totalChunks)) { + lastCheckpointChunks = stats.totalChunks; + this.checkpointIndexRun( + database, + store, + invertedIndex, + failedProcessing, + resolvedRetryChunkIds, + currentFileHashes, + committedFilePaths, + scopedRoots, + configuredProviderInfo, + ); + } } const removedChunkIds: string[] = []; @@ -4362,13 +4744,6 @@ export class Indexer { if (removedStoredChunks) { this.saveInvertedIndex(invertedIndex); } - if (scopedRoots) { - this.replaceScopedFileHashCache(currentFileHashes, scopedRoots); - } else { - this.fileHashCache = currentFileHashes; - this.saveFileHashCache(); - } - this.finalizeFailedBatchWriteState(failedProcessing.state); database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION); database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION); database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION); @@ -4377,6 +4752,13 @@ export class Indexer { this.indexCompatibility = { compatible: true }; database.commitWriteTransaction(); writeTransactionActive = false; + this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds); + if (scopedRoots) { + this.replaceScopedFileHashCache(currentFileHashes, scopedRoots); + } else { + this.fileHashCache = currentFileHashes; + this.saveFileHashCache(); + } stats.durationMs = Date.now() - startTime; onProgress?.({ phase: "complete", @@ -4401,13 +4783,6 @@ export class Indexer { ); store.save(); this.saveInvertedIndex(invertedIndex); - if (scopedRoots) { - this.replaceScopedFileHashCache(currentFileHashes, scopedRoots); - } else { - this.fileHashCache = currentFileHashes; - this.saveFileHashCache(); - } - this.finalizeFailedBatchWriteState(failedProcessing.state); database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION); database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION); database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION); @@ -4416,6 +4791,13 @@ export class Indexer { this.indexCompatibility = { compatible: true }; database.commitWriteTransaction(); writeTransactionActive = false; + this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds); + if (scopedRoots) { + this.replaceScopedFileHashCache(currentFileHashes, scopedRoots); + } else { + this.fileHashCache = currentFileHashes; + this.saveFileHashCache(); + } stats.durationMs = Date.now() - startTime; onProgress?.({ phase: "complete", @@ -4453,16 +4835,15 @@ export class Indexer { store.save(); this.saveInvertedIndex(invertedIndex); + database.commitWriteTransaction(); + writeTransactionActive = false; + this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds); if (scopedRoots) { this.replaceScopedFileHashCache(currentFileHashes, scopedRoots); } else { this.fileHashCache = currentFileHashes; this.saveFileHashCache(); } - this.finalizeFailedBatchWriteState(failedProcessing.state); - - database.commitWriteTransaction(); - writeTransactionActive = false; if (this.config.indexing.autoGc && stats.removedChunks > 0) { const gcReset = await this.maybeRunOrphanGc(); @@ -4488,6 +4869,9 @@ export class Indexer { if (forceScopedReembed && failedForcedChunkIds.size === 0) { database.deleteMetadata(this.getProjectForceReembedMetadataKey()); } + if (forceScopedReembed) { + database.setMetadata(this.getProjectMigrationFinalizedMetadataKey(), "true"); + } database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION); database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION); database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION); @@ -5187,7 +5571,9 @@ export class Indexer { async forceIndex(onProgress?: ProgressCallback): Promise { return this.withIndexMutationLease("force-index", async (recoveredOwners) => { await this.ensureInitializedUnlocked(recoveredOwners); - await this.clearIndexUnlocked(); + const recovery = this.beginClearRecoveryState(); + await this.clearIndexUnlocked(recovery.compatibilityDecision); + this.finishClearRecoveryState(); return this.indexUnlocked(onProgress, [], true); }); } @@ -5195,78 +5581,108 @@ export class Indexer { async clearIndex(): Promise { await this.withIndexMutationLease("clear", async (recoveredOwners) => { await this.ensureInitializedUnlocked(recoveredOwners); - await this.clearIndexUnlocked(); + const recovery = this.beginClearRecoveryState(); + await this.clearIndexUnlocked(recovery.compatibilityDecision); }); } - private async clearIndexUnlocked(): Promise { + private clearGlobalIndexDataUnlocked(projectRoot = this.projectRoot): void { const { store, invertedIndex, database } = this.requireLoadedIndexState(); + const clearedBranchKeys = database.getAllBranches(); + store.clear(); + store.save(); + invertedIndex.clear(); + this.saveInvertedIndex(invertedIndex); - if (this.config.scope === "global") { - store.load(); - invertedIndex.load(); - this.loadFileHashCache(); - const roots = this.getScopedRoots(); - const compatibility = this.checkCompatibility(); - const allMetadata = store.getAllMetadata(); - const hasForeignData = - allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || - this.hasForeignScopedBranchData() || - this.hasForeignScopedFileHashData(roots) || - this.hasForeignScopedFailedBatches(roots); - - if (!compatibility.compatible && hasForeignData) { - if (compatibility.code === IncompatibilityCode.EMBEDDING_STRATEGY_MISMATCH) { - this.clearSharedIndexProjectData(store, invertedIndex, database, roots); - this.clearScopedFileHashCache(roots); - this.clearScopedFailedBatches(roots); - database.setMetadata(this.getProjectForceReembedMetadataKey(), "true"); - database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey()); - this.indexCompatibility = { compatible: true }; - return; - } - - throw new Error( - `Global index compatibility reset is unsafe because the shared index contains files from other projects. ` + - `The current global index cannot be force-rebuilt for ${this.projectRoot} without deleting other repositories' indexed data. ` + - `Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.` - ); - } - - if (!hasForeignData) { - const clearedBranchKeys = database.getAllBranches(); - store.clear(); - store.save(); - invertedIndex.clear(); - this.saveInvertedIndex(invertedIndex); + this.fileHashCache.clear(); + this.saveFileHashCache(); - this.fileHashCache.clear(); - this.saveFileHashCache(); + database.clearAllIndexedData(); + this.deleteBranchCommitMetadata(database, clearedBranchKeys); + this.clearFailedBatchState(); - database.clearAllIndexedData(); - this.deleteBranchCommitMetadata(database, clearedBranchKeys); - this.clearFailedBatchState(); + database.deleteMetadata("index.version"); + database.deleteMetadata("index.pathStorageVersion"); + database.deleteMetadata("index.embeddingProvider"); + database.deleteMetadata("index.embeddingModel"); + database.deleteMetadata("index.embeddingDimensions"); + database.deleteMetadata("index.embeddingStrategyVersion"); + const projectIdentityHash = this.getProjectIdentityHash(projectRoot); + database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash)); + database.deleteMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash)); + database.deleteMetadata(this.getLegacyMigrationMetadataKey(projectIdentityHash)); + database.deleteMetadata("index.createdAt"); + database.deleteMetadata("index.updatedAt"); - database.deleteMetadata("index.version"); - database.deleteMetadata("index.pathStorageVersion"); - database.deleteMetadata("index.embeddingProvider"); - database.deleteMetadata("index.embeddingModel"); - database.deleteMetadata("index.embeddingDimensions"); - database.deleteMetadata("index.embeddingStrategyVersion"); - database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey()); - database.deleteMetadata(this.getProjectForceReembedMetadataKey()); - database.deleteMetadata(this.getLegacyMigrationMetadataKey()); - database.deleteMetadata("index.createdAt"); - database.deleteMetadata("index.updatedAt"); + this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo!); + } - this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo!); + private clearGlobalIndexUnlocked( + projectRoot = this.projectRoot, + roots = this.getScopedRoots(), + recoveryDecision?: IndexLockClearRecoveryState["compatibilityDecision"], + ): void { + const { store, invertedIndex, database } = this.requireLoadedIndexState(); + store.load(); + invertedIndex.load(); + this.loadFileHashCache(); + const compatibility = this.checkCompatibility(); + const compatibilityDecision = recoveryDecision ?? ( + compatibility.compatible + ? "compatible" + : compatibility.code === IncompatibilityCode.EMBEDDING_STRATEGY_MISMATCH + ? "embedding-strategy-mismatch" + : "incompatible" + ); + const allMetadata = store.getAllMetadata(); + const hasForeignData = + allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || + this.hasForeignScopedBranchData(projectRoot, roots) || + this.hasForeignScopedFileHashData(roots) || + this.hasForeignScopedFailedBatches(roots); + + if (compatibilityDecision !== "compatible" && hasForeignData) { + if (compatibilityDecision === "embedding-strategy-mismatch") { + this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot); + this.clearScopedFileHashCache(roots); + this.clearScopedFailedBatches(roots); + const projectIdentityHash = this.getProjectIdentityHash(projectRoot); + database.setMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash), "true"); + database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash)); + database.deleteMetadata(this.getProjectMigrationFinalizedMetadataKey(projectIdentityHash)); + if (projectRoot === this.projectRoot) { + this.indexCompatibility = { compatible: true }; + } return; } - this.clearSharedIndexProjectData(store, invertedIndex, database, roots); - this.clearScopedFileHashCache(roots); - this.clearScopedFailedBatches(roots); + throw new Error( + `Global index compatibility reset is unsafe because the shared index contains files from other projects. ` + + `The current global index cannot be force-rebuilt for ${projectRoot} without deleting other repositories' indexed data. ` + + `Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.` + ); + } + + if (!hasForeignData) { + this.clearGlobalIndexDataUnlocked(projectRoot); + return; + } + + this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot); + this.clearScopedFileHashCache(roots); + this.clearScopedFailedBatches(roots); + if (projectRoot === this.projectRoot) { this.indexCompatibility = compatibility; + } + } + + private async clearIndexUnlocked( + recoveryDecision?: IndexLockClearRecoveryState["compatibilityDecision"], + ): Promise { + const { store, invertedIndex, database } = this.requireLoadedIndexState(); + + if (this.config.scope === "global") { + this.clearGlobalIndexUnlocked(this.projectRoot, this.getScopedRoots(), recoveryDecision); return; } @@ -5464,6 +5880,10 @@ export class Indexer { )) { const chunks = retryBatch.map(({ chunk }) => chunk); const attemptCounts = new Map(retryBatch.map(({ chunk, attemptCount }) => [chunk.id, attemptCount])); + // Restore chunk rows that a recovery health check may have collected + // as orphans: a failed chunk checkpointed before its embedding has a + // committed SQLite row but no branch association until it succeeds. + this.restoreMissingChunkRows(database, chunks); const batchResult = await this.processPendingChunkBatch(chunks, { store, provider, @@ -5502,9 +5922,13 @@ export class Indexer { } if (roots && succeeded > 0 && remaining === 0 && this.hasProjectForceReembedPending()) { - database.deleteMetadata(this.getProjectForceReembedMetadataKey()); - this.saveIndexMetadata(configuredProviderInfo); - this.indexCompatibility = { compatible: true }; + const migrationFinalized = + database.getMetadata(this.getProjectMigrationFinalizedMetadataKey()) === "true"; + if (migrationFinalized) { + database.deleteMetadata(this.getProjectForceReembedMetadataKey()); + this.saveIndexMetadata(configuredProviderInfo); + this.indexCompatibility = { compatible: true }; + } } return { succeeded, failed, remaining }; @@ -5529,6 +5953,7 @@ export class Indexer { attemptCount: batch.attemptCount, error: batch.error, lastAttempt: batch.lastAttempt, + chunks: [rawChunk], }); } } diff --git a/tests/indexer-checkpoint-resume.test.ts b/tests/indexer-checkpoint-resume.test.ts new file mode 100644 index 0000000..c94f2a8 --- /dev/null +++ b/tests/indexer-checkpoint-resume.test.ts @@ -0,0 +1,1322 @@ +import { spawnSync } from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { parseConfig } from "../src/config/schema.js"; +import { readFailedBatchRecords } from "../src/indexer/failed-state-persistence.js"; +import { Indexer } from "../src/indexer/index.js"; +import { Database, hashContent } from "../src/native/index.js"; + +function writeSourceFile(filePath: string, functionNames: string[]): void { + fs.writeFileSync( + filePath, + functionNames.map((name, index) => ( + `export function ${name}() {\n` + + ` const value = ${index};\n` + + ` return value * 2;\n` + + `}\n` + )).join("\n"), + "utf8", + ); +} + +function countEmbeddedTexts(fetchSpy: ReturnType, fromCall = 0): number { + let total = 0; + for (let index = fromCall; index < fetchSpy.mock.calls.length; index++) { + const body = JSON.parse(String(fetchSpy.mock.calls[index][1]?.body ?? "{}")) as { input?: string[] }; + total += body.input?.length ?? 0; + } + return total; +} + +function canonicalPath(filePath: string): string { + return fs.realpathSync.native(filePath); +} + +function projectIdentityHash(projectRoot: string): string { + return hashContent(canonicalPath(projectRoot)).slice(0, 16); +} + +describe("indexer checkpoint resume", () => { + let projectDir: string; + let indexDir: string; + let indexers: Indexer[]; + let fetchSpy: ReturnType; + let failEmbeddingText: string | null; + + function createIndexer(checkpointIntervalChunks?: number, indexPath = indexDir): Indexer { + const indexer = new Indexer(projectDir, parseConfig({ + embeddingProvider: "custom", + customProvider: { + baseUrl: "http://localhost:11434/v1", + model: "checkpoint-test-model", + dimensions: 8, + maxBatchSize: 8, + concurrency: 1, + requestIntervalMs: 0, + }, + indexing: { + watchFiles: false, + retries: 0, + autoGc: false, + }, + }), "opencode", { + indexPath, + checkpointIntervalChunks, + // One file per batch so every batch triggers a checkpoint with + // checkpointIntervalChunks: 1. + fileBatchLimits: { maxFiles: 1, maxBytes: 8 * 1024 * 1024 }, + }); + indexers.push(indexer); + return indexer; + } + + function createGlobalIndexer(projectRoot: string, model: string, kbDir: string): Indexer { + const indexer = new Indexer(projectRoot, parseConfig({ + embeddingProvider: "custom", + customProvider: { + baseUrl: "http://localhost:11434/v1", + model, + dimensions: 8, + maxBatchSize: 8, + concurrency: 1, + requestIntervalMs: 0, + }, + scope: "global", + knowledgeBases: [kbDir], + indexing: { + watchFiles: false, + retries: 0, + autoGc: false, + }, + }), "opencode", { + indexPath: indexDir, + checkpointIntervalChunks: 1, + fileBatchLimits: { maxFiles: 1, maxBytes: 8 * 1024 * 1024 }, + }); + indexers.push(indexer); + return indexer; + } + + function setupGlobalScope(): { projectBDir: string; kbDir: string } { + const projectBDir = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-project-b-")); + const kbDir = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-kb-")); + const projectAFile = path.join(projectDir, "src", "alpha.ts"); + const projectBFile = path.join(projectBDir, "src", "beta.ts"); + const kbFile = path.join(kbDir, "docs", "shared.ts"); + fs.mkdirSync(path.dirname(projectBFile), { recursive: true }); + fs.mkdirSync(path.dirname(kbFile), { recursive: true }); + writeSourceFile(projectAFile, ["alphaOne", "alphaTwo"]); + writeSourceFile(projectBFile, ["betaOne", "betaTwo"]); + writeSourceFile(kbFile, ["sharedOne", "sharedTwo"]); + return { projectBDir, kbDir }; + } + + function getClearRecoveryState( + model = "checkpoint-test-model", + compatibilityDecision: "compatible" | "embedding-strategy-mismatch" | "incompatible" = "compatible", + ): { + phase: "clearing"; + embeddingProvider: "custom"; + embeddingModel: string; + embeddingDimensions: number; + embeddingStrategyVersion: string; + compatibilityDecision: "compatible" | "embedding-strategy-mismatch" | "incompatible"; + } { + return { + phase: "clearing", + embeddingProvider: "custom", + embeddingModel: model, + embeddingDimensions: 8, + embeddingStrategyVersion: "2", + compatibilityDecision, + }; + } + + beforeEach(() => { + failEmbeddingText = null; + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-project-")); + indexDir = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-index-")); + indexers = []; + fs.mkdirSync(path.join(projectDir, "src"), { recursive: true }); + + fetchSpy = vi.spyOn(globalThis, "fetch").mockImplementation(async ( + _url: string | URL | Request, + init?: RequestInit, + ) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + const texts = body.input ?? []; + if (failEmbeddingText !== null && texts.some((text) => text.includes(failEmbeddingText))) { + return new Response(JSON.stringify({ error: "simulated embedding failure" }), { status: 500 }); + } + return new Response(JSON.stringify({ + data: texts.map((_text, index) => ({ + embedding: Array.from({ length: 8 }, (_, dimension) => (index + dimension + 1) / 10), + })), + usage: { total_tokens: Math.max(1, texts.length) }, + }), { status: 200 }); + }); + }); + + afterEach(async () => { + await Promise.all(indexers.map((indexer) => indexer.close())); + fetchSpy.mockRestore(); + vi.unstubAllEnvs(); + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(indexDir, { recursive: true, force: true }); + }); + + it("interrupted run after a checkpoint resumes incrementally", async () => { + const indexer = createIndexer(1); + const sourceFiles = [ + path.join(projectDir, "src", "alpha.ts"), + path.join(projectDir, "src", "beta.ts"), + path.join(projectDir, "src", "gamma.ts"), + ]; + for (const [fileIndex, filePath] of sourceFiles.entries()) { + writeSourceFile( + filePath, + ["first", "second", "third", "fourth", "fifth", "sixth"].map((name) => `${name}${fileIndex}`), + ); + } + + // The embedding progress for the last batch fires after every previous + // batch has been checkpointed, so throwing there interrupts the run with + // the first two batches already durable. + await expect(indexer.index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === progress.totalFiles) { + throw new Error("simulated interruption after checkpoint"); + } + })).rejects.toThrow("simulated interruption after checkpoint"); + + const databasePath = path.join(indexDir, "codebase.db"); + const fileHashesPath = path.join(indexDir, "file-hashes.json"); + + // The batch order is not guaranteed, so derive the committed files from + // the partial hash cache: exactly two files were checkpointed before the + // interruption, and the third remains pending. + const partialHashes = JSON.parse(fs.readFileSync(fileHashesPath, "utf-8")) as Record; + const committedFiles = Object.keys(partialHashes); + expect(committedFiles.length).toBe(2); + const pendingFiles = sourceFiles + .map((filePath) => path.relative(projectDir, filePath)) + .filter((filePath) => !committedFiles.includes(filePath)); + expect(pendingFiles.length).toBe(1); + + const after = Database.openReadOnly(databasePath); + let committedChunks = 0; + try { + for (const filePath of committedFiles) { + const chunks = after.getChunksByFile(filePath); + expect(chunks.length).toBeGreaterThan(0); + committedChunks += chunks.length; + } + expect(after.getChunksByFile(pendingFiles[0])).toEqual([]); + } finally { + after.close(); + } + + expect(fs.existsSync(path.join(indexDir, "vectors"))).toBe(true); + + const callsAfterRun1 = fetchSpy.mock.calls.length; + const run1EmbeddedTexts = countEmbeddedTexts(fetchSpy, 0); + + const run2Stats = await indexer.index(); + expect(run2Stats.failedChunks).toBe(0); + + const run2EmbeddedTexts = countEmbeddedTexts(fetchSpy, callsAfterRun1); + expect(run2EmbeddedTexts).toBeGreaterThan(0); + expect(run2EmbeddedTexts).toBeLessThan(run1EmbeddedTexts); + expect(run2Stats.indexedChunks).toBe(run2EmbeddedTexts); + expect(run2Stats.existingChunks).toBe(committedChunks); + + const finalHashes = JSON.parse(fs.readFileSync(fileHashesPath, "utf-8")) as Record; + for (const filePath of [...committedFiles, ...pendingFiles]) { + expect(finalHashes[filePath]).toBeDefined(); + } + + const finalDb = Database.openReadOnly(databasePath); + try { + for (const filePath of [...committedFiles, ...pendingFiles]) { + expect(finalDb.getChunksByFile(filePath).length).toBeGreaterThan(0); + } + } finally { + finalDb.close(); + } + }); + + it("checkpoint persists failed batches", async () => { + const indexer = createIndexer(1); + const sourceFiles = [ + path.join(projectDir, "src", "alpha.ts"), + path.join(projectDir, "src", "beta.ts"), + path.join(projectDir, "src", "gamma.ts"), + ]; + // The batch order is not guaranteed, so every file contains a failing + // chunk: whichever files are checkpointed before the interruption carry a + // persisted failure record. + writeSourceFile( + sourceFiles[0], + ["first", "second", "triggerFailure", "fourth", "fifth", "sixth"].map((name) => `${name}0`), + ); + writeSourceFile( + sourceFiles[1], + ["first", "second", "triggerFailure", "fourth", "fifth", "sixth"].map((name) => `${name}1`), + ); + writeSourceFile( + sourceFiles[2], + ["first", "second", "triggerFailure", "fourth", "fifth", "sixth"].map((name) => `${name}2`), + ); + + failEmbeddingText = "triggerFailure"; + await expect(indexer.index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === progress.totalFiles) { + throw new Error("simulated interruption after failed batch checkpoint"); + } + })).rejects.toThrow("simulated interruption after failed batch checkpoint"); + + const failedBatchesPath = path.join(indexDir, "failed-batches.json"); + expect(fs.existsSync(failedBatchesPath)).toBe(true); + const persistedChunks = Array.from(readFailedBatchRecords<{ content?: string }>(failedBatchesPath)) + .flatMap((record) => record.chunks); + expect(persistedChunks.length).toBeGreaterThan(0); + expect(persistedChunks.some((chunk) => chunk.content?.includes("triggerFailure"))).toBe(true); + + failEmbeddingText = null; + const callsBeforeRun2 = fetchSpy.mock.calls.length; + const run2Stats = await indexer.index(); + expect(run2Stats.failedChunks).toBe(0); + expect(run2Stats.indexedChunks).toBeGreaterThan(0); + + const run2Texts = fetchSpy.mock.calls.slice(callsBeforeRun2).flatMap((call) => { + const body = JSON.parse(String(call[1]?.body ?? "{}")) as { input?: string[] }; + return body.input ?? []; + }); + expect(run2Texts.some((text) => text.includes("triggerFailure"))).toBe(true); + expect(fs.existsSync(failedBatchesPath)).toBe(false); + + const finalDb = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + for (const filePath of ["src/alpha.ts", "src/beta.ts", "src/gamma.ts"]) { + expect(finalDb.getChunksByFile(filePath).length).toBeGreaterThan(0); + } + } finally { + finalDb.close(); + } + }); + + it("full run result is unchanged by checkpoints", async () => { + const sourceFiles = [ + path.join(projectDir, "src", "alpha.ts"), + path.join(projectDir, "src", "beta.ts"), + path.join(projectDir, "src", "gamma.ts"), + ]; + for (const [fileIndex, filePath] of sourceFiles.entries()) { + writeSourceFile( + filePath, + ["first", "second", "third", "fourth", "fifth", "sixth"].map((name) => `${name}${fileIndex}`), + ); + } + + const checkpointedIndexDir = path.join(indexDir, "checkpointed"); + const plainIndexDir = path.join(indexDir, "plain"); + fs.mkdirSync(checkpointedIndexDir, { recursive: true }); + fs.mkdirSync(plainIndexDir, { recursive: true }); + + const checkpointedIndexer = createIndexer(1, checkpointedIndexDir); + const plainIndexer = createIndexer(undefined, plainIndexDir); + + const checkpointedStats = await checkpointedIndexer.index(); + const plainStats = await plainIndexer.index(); + + expect(checkpointedStats.totalChunks).toBe(plainStats.totalChunks); + expect(checkpointedStats.indexedChunks).toBe(plainStats.indexedChunks); + expect(checkpointedStats.existingChunks).toBe(plainStats.existingChunks); + expect(checkpointedStats.removedChunks).toBe(plainStats.removedChunks); + expect(checkpointedStats.failedChunks).toBe(plainStats.failedChunks); + expect(checkpointedStats.tokensUsed).toBe(plainStats.tokensUsed); + + const checkpointedDb = Database.openReadOnly(path.join(checkpointedIndexDir, "codebase.db")); + const plainDb = Database.openReadOnly(path.join(plainIndexDir, "codebase.db")); + try { + expect(checkpointedDb.getStats()).toEqual(plainDb.getStats()); + for (const filePath of ["src/alpha.ts", "src/beta.ts", "src/gamma.ts"]) { + expect(checkpointedDb.getChunksByFile(filePath)).toEqual(plainDb.getChunksByFile(filePath)); + } + } finally { + checkpointedDb.close(); + plainDb.close(); + } + }); + + it("interrupted run on an existing branch keeps checkpointed chunks in the branch catalog after resume", async () => { + const indexer = createIndexer(1); + const alphaFile = path.join(projectDir, "src", "alpha.ts"); + writeSourceFile(alphaFile, ["alphaOne", "alphaTwo"]); + + // Full initial index publishes the branch catalog. + await indexer.index(); + + // Add two new files and interrupt the run after the first checkpoint. + const betaFile = path.join(projectDir, "src", "beta.ts"); + const gammaFile = path.join(projectDir, "src", "gamma.ts"); + writeSourceFile(betaFile, ["betaOne", "betaTwo"]); + writeSourceFile(gammaFile, ["gammaOne", "gammaTwo"]); + await expect(indexer.index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === progress.totalFiles) { + throw new Error("simulated interruption after new-file checkpoint"); + } + })).rejects.toThrow("simulated interruption after new-file checkpoint"); + + // The checkpointed file's chunks must survive in the branch catalog after + // the resume, even though the file itself is skipped as unchanged. + const resumedStats = await indexer.index(); + expect(resumedStats.failedChunks).toBe(0); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + const branchChunkIds = new Set(db.getBranchChunkIds("default")); + for (const filePath of ["src/alpha.ts", "src/beta.ts", "src/gamma.ts"]) { + const chunks = db.getChunksByFile(filePath); + expect(chunks.length).toBeGreaterThan(0); + for (const chunk of chunks) { + expect(branchChunkIds.has(chunk.chunkId)).toBe(true); + } + } + const branchSymbolIds = new Set(db.getBranchSymbolIds("default")); + for (const filePath of ["src/alpha.ts", "src/beta.ts", "src/gamma.ts"]) { + for (const symbol of db.getSymbolsByFile(filePath)) { + expect(branchSymbolIds.has(symbol.id)).toBe(true); + } + } + } finally { + db.close(); + } + }); + + it("interrupted run with a modified file re-processes it on resume", async () => { + const indexer = createIndexer(1); + const alphaFile = path.join(projectDir, "src", "alpha.ts"); + writeSourceFile(alphaFile, ["alphaOne", "alphaTwo"]); + + await indexer.index(); + + // Modify the file and add a new one; interrupt after the first checkpoint. + writeSourceFile(alphaFile, ["alphaOne", "alphaThree"]); + const betaFile = path.join(projectDir, "src", "beta.ts"); + writeSourceFile(betaFile, ["betaOne", "betaTwo"]); + await expect(indexer.index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === progress.totalFiles) { + throw new Error("simulated interruption after modified-file checkpoint"); + } + })).rejects.toThrow("simulated interruption after modified-file checkpoint"); + + // A modified file with stale chunks to evict must not be checkpointed. + const fileHashesPath = path.join(indexDir, "file-hashes.json"); + const partialHashes = JSON.parse(fs.readFileSync(fileHashesPath, "utf-8")) as Record; + expect(partialHashes["src/alpha.ts"]).toBeUndefined(); + + // The resume re-processes it and evicts the removed chunk. + const resumedStats = await indexer.index(); + expect(resumedStats.failedChunks).toBe(0); + expect(resumedStats.removedChunks).toBeGreaterThan(0); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + const chunks = db.getChunksByFile("src/alpha.ts"); + expect(chunks.some((chunk) => chunk.name === "alphaTwo")).toBe(false); + expect(chunks.some((chunk) => chunk.name === "alphaThree")).toBe(true); + } finally { + db.close(); + } + }); + + it("global lease recovery invalidates the hash cache after an interrupted clear", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + // Simulate a crashed clear: an orphaned lease owned by a dead pid. + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "clear", + token: "11111111-1111-4111-8111-111111111111", + recoveryProtocolVersion: 1, + projectRoot: projectDir, + scopedRoots: [canonicalPath(projectDir), canonicalPath(kbDir)], + clearRecovery: getClearRecoveryState(), + })); + + const fileHashesPath = path.join(indexDir, "file-hashes.json"); + expect(fs.existsSync(fileHashesPath)).toBe(true); + + // A crashed clear wipes the store and inverted index before the hash cache + // (clearIndexUnlocked order): simulate that partial state so the next run + // would otherwise skip every file against an empty store. + for (const artifact of ["vectors", "vectors.usearch", "vectors.meta.json", "inverted-index.json"]) { + fs.rmSync(path.join(indexDir, artifact), { recursive: true, force: true }); + } + + // The recovery must invalidate the cache after an interrupted clear so the + // next run re-embeds instead of skipping files against an empty store. + const beforeCalls = fetchSpy.mock.calls.length; + const stats = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + expect(stats.failedChunks).toBe(0); + expect(fetchSpy.mock.calls.length).toBeGreaterThan(beforeCalls); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("global lease recovery preserves the checkpointed hash cache", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + // Simulate a crashed indexing session: an orphaned lease owned by a dead pid. + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "index", + token: "11111111-1111-4111-8111-111111111111", + })); + + const fileHashesPath = path.join(indexDir, "file-hashes.json"); + expect(fs.existsSync(fileHashesPath)).toBe(true); + + // The recovery must keep the checkpointed hash cache: unchanged files are + // skipped instead of being re-parsed and re-embedded. + const beforeCalls = fetchSpy.mock.calls.length; + const stats = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + expect(stats.failedChunks).toBe(0); + expect(fs.existsSync(fileHashesPath)).toBe(true); + expect(fetchSpy.mock.calls.length).toBe(beforeCalls); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("retryFailedBatches does not close an interrupted strategy migration", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + const db = new Database(path.join(indexDir, "codebase.db")); + const projectAHash = projectIdentityHash(projectDir); + db.setMetadata(`index.embeddingStrategyVersion.${projectAHash}`, "1"); + + const resettingIndexer = createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir); + await resettingIndexer.clearIndex(); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBe("true"); + + // Interrupt the forced re-embed run after its first checkpoint, with a + // failing chunk so the run leaves failed batches behind. + failEmbeddingText = "One"; + await expect(createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === 1) { + throw new Error("simulated interruption during forced re-embed"); + } + })).rejects.toThrow("simulated interruption during forced re-embed"); + + // The retry recovers the failed chunks but must not close the migration: + // the main run never finished, so unprocessed scope files still hold old + // vectors. + failEmbeddingText = null; + const retry = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).retryFailedBatches(); + expect(retry.remaining).toBe(0); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBe("true"); + + // A full run completes the migration and clears the flag. + const stats = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + expect(stats.failedChunks).toBe(0); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBeNull(); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("interrupted forced re-embed run keeps migration pending and re-embeds every scope file on resume", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + const db = new Database(path.join(indexDir, "codebase.db")); + const projectAHash = projectIdentityHash(projectDir); + db.setMetadata(`index.embeddingStrategyVersion.${projectAHash}`, "1"); + + const resettingIndexer = createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir); + await resettingIndexer.clearIndex(); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBe("true"); + + // Interrupt the forced re-embed run right after its first checkpoint. + await expect(createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === 1) { + throw new Error("simulated interruption during forced re-embed"); + } + })).rejects.toThrow("simulated interruption during forced re-embed"); + + // The checkpoint must not clear the migration flag: the store is still + // partially migrated. + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBe("true"); + + // Resuming without force re-runs the migration and re-embeds every scope + // file, including the ones whose hashes were already checkpointed. + const beforeResumeCalls = fetchSpy.mock.calls.length; + const resumedStats = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + expect(resumedStats.failedChunks).toBe(0); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBeNull(); + + const resumeInputs = fetchSpy.mock.calls.slice(beforeResumeCalls).flatMap((call) => { + const body = JSON.parse(String(call[1]?.body ?? "{}")) as { input?: string[] }; + return body.input ?? []; + }); + expect(resumeInputs.some((text) => text.includes("alphaOne"))).toBe(true); + expect(resumeInputs.some((text) => text.includes("sharedOne"))).toBe(true); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("forced re-embed run re-embeds unchanged scope files while migration is pending", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + const db = new Database(path.join(indexDir, "codebase.db")); + const projectAHash = projectIdentityHash(projectDir); + // Simulate a pending migration whose scope files are already cached: the + // run must still re-embed them instead of skipping them as unchanged. + db.setMetadata(`index.forceReembed.${projectAHash}`, "true"); + + const beforeCalls = fetchSpy.mock.calls.length; + const stats = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + expect(stats.failedChunks).toBe(0); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBeNull(); + + const migrationInputs = fetchSpy.mock.calls.slice(beforeCalls).flatMap((call) => { + const body = JSON.parse(String(call[1]?.body ?? "{}")) as { input?: string[] }; + return body.input ?? []; + }); + expect(migrationInputs.some((text) => text.includes("alphaOne"))).toBe(true); + expect(migrationInputs.some((text) => text.includes("sharedOne"))).toBe(true); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("checkpoint preserves out-of-scope failed batches for later retries", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + // Project A leaves a failed batch behind. + failEmbeddingText = "One"; + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + const failedBatchesPath = path.join(indexDir, "failed-batches.json"); + expect(fs.existsSync(failedBatchesPath)).toBe(true); + + // Project B's run checkpoints and must not drop A's failed batch. + failEmbeddingText = null; + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + expect(fs.existsSync(failedBatchesPath)).toBe(true); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("checkpoint preserves pending retries alongside new failures", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + // Project A leaves a failed batch behind. + failEmbeddingText = "One"; + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + + // A second run checkpoints a new failure while the old retry is still + // pending: the retry must survive the checkpoint and be re-attempted. + writeSourceFile(path.join(kbDir, "docs", "shared.ts"), ["sharedOne", "sharedTwo", "sharedThree"]); + failEmbeddingText = "Three"; + const beforeCalls = fetchSpy.mock.calls.length; + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + const retryInputs = fetchSpy.mock.calls.slice(beforeCalls).flatMap((call) => { + const body = JSON.parse(String(call[1]?.body ?? "{}")) as { input?: string[] }; + return body.input ?? []; + }); + expect(retryInputs.some((text) => text.includes("alphaOne"))).toBe(true); + + // The resolved retry must not be republished in the final failed-batches + // file: only the new failure remains. + const persisted = fs.readFileSync(path.join(indexDir, "failed-batches.json"), "utf-8"); + expect(persisted).toContain("sharedThree"); + expect(persisted).not.toContain("alphaOne"); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("interrupted force-index run resumes from checkpoints instead of clearing", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + // The force-index clear phase completes, then the indexing phase is + // interrupted after the first file is checkpointed. + await expect(createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).forceIndex((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === 2) { + throw new Error("simulated interruption during force-index"); + } + })).rejects.toThrow("simulated interruption during force-index"); + + // Simulate a crash during the indexing phase: an orphaned "force-index" + // lease with no clearing-phase marker left behind. + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "force-index", + token: "11111111-1111-4111-8111-111111111111", + })); + + // The recovery must not clear the checkpointed data: the next run resumes + // incrementally instead of re-embedding the already checkpointed file. + const beforeCalls = fetchSpy.mock.calls.length; + const stats = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + expect(stats.failedChunks).toBe(0); + const resumeInputs = fetchSpy.mock.calls.slice(beforeCalls).flatMap((call) => { + const body = JSON.parse(String(call[1]?.body ?? "{}")) as { input?: string[] }; + return body.input ?? []; + }); + expect(resumeInputs.some((text) => text.includes("sharedOne"))).toBe(true); + expect(resumeInputs.some((text) => text.includes("alphaOne"))).toBe(false); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("checkpoint persists the keyword index before vectors so a crash cannot orphan BM25 chunks", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + // Interrupt the first run between the vector store save and the keyword + // index save: with the keyword index persisted first, the store is not yet + // durable and the resume re-embeds, keeping BM25 complete. + const invertedSave = vi.spyOn( + Indexer.prototype as unknown as { saveInvertedIndex: () => void }, + "saveInvertedIndex", + ).mockImplementation(() => { + throw new Error("simulated crash between artifact writes"); + }); + await expect(createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index()).rejects.toThrow( + "simulated crash between artifact writes", + ); + invertedSave.mockRestore(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + const inverted = fs.readFileSync(path.join(indexDir, "inverted-index.json"), "utf-8"); + expect(inverted).toContain("alphaone"); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("project-scope dead-lease recovery preserves checkpoints for incremental resume", async () => { + const projectOwnedIndex = path.join(projectDir, ".opencode", "index"); + const sourceFiles = [ + path.join(projectDir, "src", "alpha.ts"), + path.join(projectDir, "src", "beta.ts"), + path.join(projectDir, "src", "gamma.ts"), + ]; + for (const [fileIndex, filePath] of sourceFiles.entries()) { + writeSourceFile( + filePath, + ["first", "second", "third", "fourth", "fifth", "sixth"].map((name) => `${name}${fileIndex}`), + ); + } + + // Interrupt after two files are checkpointed. + const indexer = createIndexer(1, projectOwnedIndex); + await expect(indexer.index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === progress.totalFiles) { + throw new Error("simulated crash after checkpoint"); + } + })).rejects.toThrow("simulated crash after checkpoint"); + + // Simulate a dead "index" lease (not "clear" or "force-index"): recovery + // must preserve checkpointed artifacts and resume incrementally. + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(projectOwnedIndex, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "index", + token: "22222222-2222-4222-8222-222222222222", + })); + + const beforeCalls = fetchSpy.mock.calls.length; + const resumeIndexer = createIndexer(1, projectOwnedIndex); + const stats = await resumeIndexer.index(); + expect(stats.failedChunks).toBe(0); + + // Only the pending file was re-embedded: the two checkpointed files are + // preserved and skipped on resume. + const resumeInputs = fetchSpy.mock.calls.slice(beforeCalls).flatMap((call) => { + const body = JSON.parse(String(call[1]?.body ?? "{}")) as { input?: string[] }; + return body.input ?? []; + }); + expect(resumeInputs.some((text) => text.includes("first2"))).toBe(true); + expect(resumeInputs.some((text) => text.includes("first0"))).toBe(false); + expect(resumeInputs.some((text) => text.includes("first1"))).toBe(false); + }); + it("checkpoint drops stale failures for files modified and reindexed successfully", async () => { + const alphaFile = path.join(projectDir, "src", "alpha.ts"); + const gammaFile = path.join(projectDir, "src", "gamma.ts"); + writeSourceFile(alphaFile, ["alphaOne", "alphaTwo"]); + + // Run 1: alphaOne fails embedding, leaving a failed-batches record. + failEmbeddingText = "alphaOne"; + const indexer1 = createIndexer(1); + await indexer1.index(); + expect(fs.existsSync(path.join(indexDir, "failed-batches.json"))).toBe(true); + const failedContent1 = fs.readFileSync(path.join(indexDir, "failed-batches.json"), "utf-8"); + expect(failedContent1).toContain("alphaOne"); + failEmbeddingText = null; + + // Run 2: alpha.ts is modified and reindexed successfully, while a new + // file gamma.ts has a chunk that fails — ensuring the checkpoint's + // failed-batches reconstruction path executes. + writeSourceFile(alphaFile, ["alphaOne", "alphaTwo", "alphaThree"]); + writeSourceFile(gammaFile, ["gammaOne"]); + failEmbeddingText = "gammaOne"; + const indexer2 = createIndexer(1); + await indexer2.index(); + + // The old alphaOne failure must not survive: alpha.ts was modified and + // all its chunks embedded successfully. Only gammaOne should remain. + const failedContent2 = fs.readFileSync(path.join(indexDir, "failed-batches.json"), "utf-8"); + expect(failedContent2).toContain("gammaOne"); + expect(failedContent2).not.toContain("alphaOne"); + }); + + async function verifyEmptyFailureStateBeforeHash(checkpointIntervalChunks?: number): Promise { + const alphaFile = path.join(projectDir, "src", "alpha.ts"); + const failedBatchesPath = path.join(indexDir, "failed-batches.json"); + writeSourceFile(alphaFile, ["alphaOne"]); + + failEmbeddingText = "alphaOne"; + await createIndexer(checkpointIntervalChunks).index(); + expect(fs.readFileSync(failedBatchesPath, "utf-8")).toContain("alphaOne"); + failEmbeddingText = null; + + writeSourceFile(alphaFile, ["alphaReplacement"]); + const prototype = Indexer.prototype as unknown as { + saveFileHashCache(this: Indexer): void; + }; + const saveFileHashCache = prototype.saveFileHashCache; + const hashSave = vi.spyOn(prototype, "saveFileHashCache").mockImplementation(function (this: Indexer) { + saveFileHashCache.call(this); + throw new Error("simulated interruption after empty failed-batch checkpoint"); + }); + try { + await expect(createIndexer(checkpointIntervalChunks).index()).rejects.toThrow( + "simulated interruption after empty failed-batch checkpoint", + ); + } finally { + hashSave.mockRestore(); + } + + expect(fs.existsSync(failedBatchesPath)).toBe(false); + const beforeResumeCalls = fetchSpy.mock.calls.length; + await createIndexer(checkpointIntervalChunks).index(); + expect(countEmbeddedTexts(fetchSpy, beforeResumeCalls)).toBe(0); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + const branchChunkIds = new Set(db.getBranchChunkIds("default")); + const staleChunks = db.getChunksByName("alphaOne"); + const currentChunks = db.getChunksByName("alphaReplacement"); + expect(staleChunks.every((chunk) => !branchChunkIds.has(chunk.chunkId))).toBe(true); + expect(currentChunks.some((chunk) => branchChunkIds.has(chunk.chunkId))).toBe(true); + } finally { + db.close(); + } + } + + it("empty checkpoint state removes stale failures before persisting the new file hash", async () => { + await verifyEmptyFailureStateBeforeHash(1); + }); + + it("final empty failed-batch state is published before the new file hash", async () => { + await verifyEmptyFailureStateBeforeHash(); + }); + + it("checkpoint does not duplicate pending retry chunks across multiple checkpoints", async () => { + const alphaFile = path.join(projectDir, "src", "alpha.ts"); + const betaFile = path.join(projectDir, "src", "beta.ts"); + const gammaFile = path.join(projectDir, "src", "gamma.ts"); + writeSourceFile(alphaFile, ["alphaOne", "alphaTwo"]); + + // Run 1: alphaOne fails, leaving a failed-batches record. + failEmbeddingText = "alphaOne"; + await createIndexer(1).index(); + failEmbeddingText = null; + + // Run 2: alpha.ts is unchanged (alphaOne stays in latestById for retry), + // while two new files beta.ts and gamma.ts each fail — triggering two + // separate checkpoints. Without deduplication, alphaOne would be + // re-written from latestById at every checkpoint, duplicating it. + writeSourceFile(betaFile, ["betaOne", "betaTwo"]); + writeSourceFile(gammaFile, ["gammaOne", "gammaTwo"]); + failEmbeddingText = "One"; + await createIndexer(1).index(); + + // The failed-batches file should contain exactly one record for + // alphaOne, not one per checkpoint. + const failedContent = fs.readFileSync(path.join(indexDir, "failed-batches.json"), "utf-8"); + const alphaOneOccurrences = (failedContent.match(/"name":"alphaOne"/g) ?? []).length; + expect(alphaOneOccurrences).toBe(1); + }); + + it("global lease recovery replays an interrupted clear against the originating project scope", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + // Simulate a crashed clear started by project A: the lease records the + // originating project scope so the recovery replays the clear against A + // instead of the project that reclaims the lease. + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "clear", + token: "11111111-1111-4111-8111-111111111111", + recoveryProtocolVersion: 1, + projectRoot: projectDir, + scopedRoots: [canonicalPath(projectDir), canonicalPath(kbDir)], + clearRecovery: getClearRecoveryState(), + })); + + // Project B reclaims the lease. The recovery must clear A's data (and the + // shared knowledge base) while preserving B's checkpointed data. B's run + // only scans B and the knowledge base, so A's removal is observable in + // the shared database and hash cache, not in B's embedding calls. + const beforeCalls = fetchSpy.mock.calls.length; + const stats = await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + expect(stats.failedChunks).toBe(0); + const resumeInputs = fetchSpy.mock.calls.slice(beforeCalls).flatMap((call) => { + const body = JSON.parse(String(call[1]?.body ?? "{}")) as { input?: string[] }; + return body.input ?? []; + }); + // B's checkpointed data is preserved: B's files are not re-embedded, and + // the shared knowledge base (cleared with A's scope) is re-cataloged by + // B's run (reusing the cached embeddings, so no new embedding calls). + expect(resumeInputs.some((text) => text.includes("betaOne"))).toBe(false); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + expect(db.getChunksByName("alphaOne").length).toBe(0); + expect(db.getChunksByName("betaOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("sharedOne").length).toBeGreaterThan(0); + } finally { + db.close(); + } + const fileHashes = JSON.parse(fs.readFileSync(path.join(indexDir, "file-hashes.json"), "utf-8")) as Record; + expect(Object.keys(fileHashes).some((filePath) => filePath.includes("alpha.ts"))).toBe(false); + expect(Object.keys(fileHashes).some((filePath) => filePath.includes("beta.ts"))).toBe(true); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("cross-project clear recovery preserves the reclaiming project branch catalog", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + // Simulate a crashed clear started by project A, reclaimed by project B + // through a retry run: the recovery must not remove the shared knowledge + // base entries from B's branch catalog, because B never re-scans them. + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "clear", + token: "11111111-1111-4111-8111-111111111111", + recoveryProtocolVersion: 1, + projectRoot: projectDir, + scopedRoots: [canonicalPath(projectDir), canonicalPath(kbDir)], + clearRecovery: getClearRecoveryState(), + })); + + // B reclaims the lease through retryFailedBatches, which runs the + // recovery but never re-scans the knowledge base. + const retry = await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).retryFailedBatches(); + expect(retry.remaining).toBe(0); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + const branchKey = `${projectIdentityHash(projectBDir)}:default`; + const branchChunkIds = db.getBranchChunkIds(branchKey); + expect(branchChunkIds.length).toBeGreaterThan(0); + expect(db.getChunksByName("sharedOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("betaOne").length).toBeGreaterThan(0); + } finally { + db.close(); + } + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("recovery health check does not orphan checkpointed failed chunk rows", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + // Every project A file contains a failing chunk so whichever files are + // checkpointed before the interruption carry a persisted failure record + // with a committed SQLite chunk row. + const sourceFiles = [ + path.join(projectDir, "src", "alpha.ts"), + path.join(projectDir, "src", "beta.ts"), + path.join(projectDir, "src", "gamma.ts"), + ]; + for (const [fileIndex, filePath] of sourceFiles.entries()) { + writeSourceFile( + filePath, + ["first", "second", "triggerFailure", "fourth", "fifth", "sixth"].map((name) => `${name}${fileIndex}`), + ); + } + + failEmbeddingText = "triggerFailure"; + await expect(createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === progress.totalFiles) { + throw new Error("simulated interruption after failed batch checkpoint"); + } + })).rejects.toThrow("simulated interruption after failed batch checkpoint"); + failEmbeddingText = null; + + // Simulate a crashed indexing session: the recovery health check runs + // gcOrphanChunks() and deletes the committed rows of failed chunks that + // have no branch association yet. + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "index", + token: "11111111-1111-4111-8111-111111111111", + })); + + // The dedicated retry must restore the missing chunk rows before + // re-embedding, so the branch catalog never references a chunk without + // metadata. + const retry = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).retryFailedBatches(); + expect(retry.remaining).toBe(0); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + const branchKey = `${projectIdentityHash(projectDir)}:default`; + const branchChunkIds = db.getBranchChunkIds(branchKey); + expect(branchChunkIds.length).toBeGreaterThan(0); + for (const chunkId of branchChunkIds) { + expect(db.getChunk(chunkId)).not.toBeNull(); + } + for (const filePath of ["src/alpha.ts", "src/beta.ts", "src/gamma.ts"]) { + expect(db.getChunksByFile(path.join(projectDir, filePath)).length).toBeGreaterThan(0); + } + } finally { + db.close(); + } + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("global recovery retains legacy clear markers when the originating state is unknown", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const token = "22222222-2222-4222-8222-222222222222"; + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "clear", + token, + })); + + await expect( + createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).retryFailedBatches(), + ).rejects.toThrow("originating recovery state is unknown"); + + expect(fs.existsSync(path.join(indexDir, `indexing.lock.recovery.${token}`))).toBe(true); + expect(fs.existsSync(lockPath)).toBe(false); + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + expect(db.getChunksByName("alphaOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("betaOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("sharedOne").length).toBeGreaterThan(0); + } finally { + db.close(); + } + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("cross-project clear recovery rejects a different embedding configuration", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const token = "33333333-3333-4333-8333-333333333333"; + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "clear", + token, + recoveryProtocolVersion: 1, + projectRoot: projectDir, + scopedRoots: [canonicalPath(projectDir), canonicalPath(kbDir)], + clearRecovery: getClearRecoveryState("originating-model"), + })); + + await expect( + createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).retryFailedBatches(), + ).rejects.toThrow("embedding configuration does not match the originating lease"); + + expect(fs.existsSync(path.join(indexDir, `indexing.lock.recovery.${token}`))).toBe(true); + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + expect(db.getChunksByName("alphaOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("betaOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("sharedOne").length).toBeGreaterThan(0); + } finally { + db.close(); + } + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("cross-project clear recovery preserves the originating compatibility decision", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + const projectAHash = projectIdentityHash(projectDir); + const writableDb = new Database(path.join(indexDir, "codebase.db")); + writableDb.setMetadata(`index.embeddingStrategyVersion.${projectAHash}`, "1"); + writableDb.close(); + + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "clear", + token: "66666666-6666-4666-8666-666666666666", + recoveryProtocolVersion: 1, + projectRoot: projectDir, + scopedRoots: [canonicalPath(projectDir), canonicalPath(kbDir)], + clearRecovery: getClearRecoveryState("checkpoint-test-model", "embedding-strategy-mismatch"), + })); + + const retry = await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).retryFailedBatches(); + expect(retry.remaining).toBe(0); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBe("true"); + expect(db.getMetadata(`index.embeddingStrategyVersion.${projectAHash}`)).toBeNull(); + expect(db.getChunksByName("alphaOne").length).toBe(0); + expect(db.getChunksByName("betaOne").length).toBeGreaterThan(0); + } finally { + db.close(); + } + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("stale force-index phase markers cannot clear another recovered owner", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + const beforeCalls = fetchSpy.mock.calls.length; + + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "force-index", + token: "44444444-4444-4444-8444-444444444444", + recoveryProtocolVersion: 1, + projectRoot: projectDir, + scopedRoots: [canonicalPath(projectDir), canonicalPath(kbDir)], + })); + fs.writeFileSync(path.join(indexDir, "force-index-phase"), JSON.stringify({ + phase: "clearing", + ownerToken: "55555555-5555-4555-8555-555555555555", + })); + + const stats = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + expect(stats.failedChunks).toBe(0); + expect(countEmbeddedTexts(fetchSpy, beforeCalls)).toBe(0); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + expect(db.getChunksByName("alphaOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("sharedOne").length).toBeGreaterThan(0); + } finally { + db.close(); + } + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("legacy force-index phase markers fail closed when ownership is unknown", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const token = "77777777-7777-4777-8777-777777777777"; + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "force-index", + token, + })); + fs.writeFileSync(path.join(indexDir, "force-index-phase"), "clearing"); + + await expect( + createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(), + ).rejects.toThrow("legacy clearing phase ownership is unknown"); + + expect(fs.existsSync(path.join(indexDir, `indexing.lock.recovery.${token}`))).toBe(true); + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + expect(db.getChunksByName("alphaOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("sharedOne").length).toBeGreaterThan(0); + } finally { + db.close(); + } + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); +});