From c79cb26fe8625687d5e59da99fff19e01faa1099 Mon Sep 17 00:00:00 2001 From: Nicolas COMPAIN Date: Wed, 12 Aug 2026 17:32:54 +0200 Subject: [PATCH 1/5] fix(indexer): keep forced re-embed migrations coherent across interruptions While a project-scoped forced re-embed is pending: - checkpoints no longer stamp embedding metadata, branch commit, or the compatibility certificate, so an interrupted migration cannot resume into a silent mix of old and new vector spaces; - unchanged scope files are re-embedded instead of skipped, so cached files cannot be dropped from the branch catalog mid-migration; - retryFailedBatches only clears the pending migration after the main run has committed its new metadata, instead of closing the migration from a partially migrated store. Adds simulated-interruption tests covering the pending flag, full scope re-embed on resume, and unchanged-file re-embedding. --- CHANGELOG.md | 2 + src/indexer/index.ts | 83 ++++- tests/indexer-checkpoint-resume.test.ts | 411 ++++++++++++++++++++++++ 3 files changed, 493 insertions(+), 3 deletions(-) create mode 100644 tests/indexer-checkpoint-resume.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 697723c..d83d6b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **Interrupted indexing resume**: Interrupted indexing runs now persist incremental checkpoints (database, vectors, BM25, failed batches, and file hashes), so a subsequent run resumes incrementally instead of re-embedding the whole project. +- **Forced re-embed migration safety**: While a project-scoped forced re-embed is pending, checkpoints no longer stamp new embedding metadata or a compatibility certificate, unchanged scope files are re-embedded instead of skipped, and `retryFailedBatches` only clears the pending migration after the main run has committed its new metadata. This prevents an interrupted migration from resuming into a silent mix of old and new vector spaces. - **Conceptual context retrieval**: Prefer implementation paths over tests and documentation for code-focused `codebase_context` searches, without applying a hard source-only filter or changing documentation and test queries. - **Scoped definition lookup**: Allow explicit definition searches constrained to a file type or directory to return matching declarations in fixtures and test paths without weakening source-first ranking for ordinary searches. diff --git a/src/indexer/index.ts b/src/indexer/index.ts index 112d9ff..0850fda 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -345,6 +345,8 @@ export interface IndexerRuntimeOptions { indexPath?: string; /** Internal test and benchmark override. Production uses the fixed limits. */ fileBatchLimits?: FileBatchLimits; + /** Internal test and benchmark override. Production uses a fixed default. */ + checkpointIntervalChunks?: number; } export interface BranchIndexResult { @@ -1172,6 +1174,7 @@ export class Indexer { private writerArtifactFingerprint: ReaderArtifactFingerprint | null = null; private readerArtifactRetryAfter = new Map(); private readonly fileBatchLimits?: FileBatchLimits; + private readonly checkpointIntervalChunks?: number; constructor( projectRoot: string, @@ -1190,6 +1193,7 @@ export class Indexer { this.expectedCommitOverride = runtimeOptions.expectedCommit?.toLowerCase(); this.indexPathOverride = runtimeOptions.indexPath; this.fileBatchLimits = runtimeOptions.fileBatchLimits; + this.checkpointIntervalChunks = runtimeOptions.checkpointIntervalChunks; this.config = config; this.host = host; if (isGitRepo(this.materializedProjectRoot)) { @@ -2067,6 +2071,45 @@ export class Indexer { this.clearFailedBatchState(); } + private checkpointIndexRun( + database: Database, + store: VectorStore, + invertedIndex: InvertedIndex, + failedProcessing: { state: FailedBatchWriteState; latestById: Map }, + currentFileHashes: Map, + committedFilePaths: Set, + scopedRoots: string[] | null, + configuredProviderInfo: ConfiguredProviderInfo, + indexedCommit: string | null, + ): void { + database.commitWriteTransaction(); + database.beginWriteTransaction(); + if (!this.hasProjectForceReembedPending()) { + this.saveIndexMetadata(configuredProviderInfo); + this.saveBranchCommit(database, indexedCommit); + this.indexCompatibility = { compatible: true }; + } + store.save(); + this.saveInvertedIndex(invertedIndex); + if (failedProcessing.state.recordsWritten > 0) { + failedProcessing.state.writer.commit(); + failedProcessing.state = this.createFailedBatchWriteState(); + } + const partialHashes = new Map(); + for (const filePath of committedFilePaths) { + const hash = currentFileHashes.get(filePath); + if (hash !== undefined) { + partialHashes.set(filePath, hash); + } + } + if (scopedRoots) { + this.replaceScopedFileHashCache(partialHashes, scopedRoots); + } else { + this.fileHashCache = partialHashes; + this.saveFileHashCache(); + } + } + private clearFailedBatchState(): void { if (existsSync(this.failedBatchesPath)) { try { @@ -3862,9 +3905,12 @@ export class Indexer { reparseCachedSwiftFiles && path.extname(storedPath).toLowerCase() === ".swift"; const requiresMetalParserUpgrade = reparseCachedMetalFiles && path.extname(storedPath).toLowerCase() === ".metal"; + const inMigrationScope = + forceScopedReembed && scopedRoots !== null && this.isFileInCurrentScope(storedPath, scopedRoots); if ( cachedHashMatches && + !inMigrationScope && !needsCallGraphRefresh && !requiresSwiftParserUpgrade && !requiresMetalParserUpgrade && @@ -4013,6 +4059,8 @@ export class Indexer { } let processedChangedFiles = 0; + let lastCheckpointChunks = 0; + const committedFilePaths = new Set(unchangedFilePaths); for (const descriptorBatch of iterateOrderedFileBatches( changedFileDescriptors, (descriptor) => descriptor.sourceBytes, @@ -4261,6 +4309,28 @@ export class Indexer { } } } + + for (const descriptor of descriptorBatch) { + committedFilePaths.add(descriptor.storedPath); + } + const checkpointInterval = Math.max( + this.checkpointIntervalChunks ?? 2000, + Math.floor(stats.totalChunks / 10), + ); + if (stats.totalChunks - lastCheckpointChunks >= checkpointInterval) { + lastCheckpointChunks = stats.totalChunks; + this.checkpointIndexRun( + database, + store, + invertedIndex, + failedProcessing, + currentFileHashes, + committedFilePaths, + scopedRoots, + configuredProviderInfo, + indexedCommit, + ); + } } const retryableFailedChunks = this.iterateLatestFailedChunks( @@ -5502,9 +5572,16 @@ export class Indexer { } if (roots && succeeded > 0 && remaining === 0 && this.hasProjectForceReembedPending()) { - database.deleteMetadata(this.getProjectForceReembedMetadataKey()); - this.saveIndexMetadata(configuredProviderInfo); - this.indexCompatibility = { compatible: true }; + const storedProvider = database.getMetadata("index.embeddingProvider"); + const storedModel = database.getMetadata("index.embeddingModel"); + const migrationCommitted = + storedProvider === configuredProviderInfo.provider && + storedModel === configuredProviderInfo.modelInfo.model; + if (migrationCommitted) { + database.deleteMetadata(this.getProjectForceReembedMetadataKey()); + this.saveIndexMetadata(configuredProviderInfo); + this.indexCompatibility = { compatible: true }; + } } return { succeeded, failed, remaining }; diff --git a/tests/indexer-checkpoint-resume.test.ts b/tests/indexer-checkpoint-resume.test.ts new file mode 100644 index 0000000..32aacce --- /dev/null +++ b/tests/indexer-checkpoint-resume.test.ts @@ -0,0 +1,411 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { parseConfig } from "../src/config/schema.js"; +import { readFailedBatchRecords } from "../src/indexer/failed-state-persistence.js"; +import { Indexer } from "../src/indexer/index.js"; +import { Database, hashContent } from "../src/native/index.js"; + +function writeSourceFile(filePath: string, functionNames: string[]): void { + fs.writeFileSync( + filePath, + functionNames.map((name, index) => ( + `export function ${name}() {\n` + + ` const value = ${index};\n` + + ` return value * 2;\n` + + `}\n` + )).join("\n"), + "utf8", + ); +} + +function countEmbeddedTexts(fetchSpy: ReturnType, fromCall = 0): number { + let total = 0; + for (let index = fromCall; index < fetchSpy.mock.calls.length; index++) { + const body = JSON.parse(String(fetchSpy.mock.calls[index][1]?.body ?? "{}")) as { input?: string[] }; + total += body.input?.length ?? 0; + } + return total; +} + +function canonicalPath(filePath: string): string { + return fs.realpathSync.native(filePath); +} + +function projectIdentityHash(projectRoot: string): string { + return hashContent(canonicalPath(projectRoot)).slice(0, 16); +} + +describe("indexer checkpoint resume", () => { + let projectDir: string; + let indexDir: string; + let indexers: Indexer[]; + let fetchSpy: ReturnType; + let failEmbeddingText: string | null; + + function createIndexer(checkpointIntervalChunks?: number, indexPath = indexDir): Indexer { + const indexer = new Indexer(projectDir, parseConfig({ + embeddingProvider: "custom", + customProvider: { + baseUrl: "http://localhost:11434/v1", + model: "checkpoint-test-model", + dimensions: 8, + maxBatchSize: 8, + concurrency: 1, + requestIntervalMs: 0, + }, + indexing: { + watchFiles: false, + retries: 0, + autoGc: false, + }, + }), "opencode", { + indexPath, + checkpointIntervalChunks, + // One file per batch so every batch triggers a checkpoint with + // checkpointIntervalChunks: 1. + fileBatchLimits: { maxFiles: 1, maxBytes: 8 * 1024 * 1024 }, + }); + indexers.push(indexer); + return indexer; + } + + function createGlobalIndexer(projectRoot: string, model: string, kbDir: string): Indexer { + const indexer = new Indexer(projectRoot, parseConfig({ + embeddingProvider: "custom", + customProvider: { + baseUrl: "http://localhost:11434/v1", + model, + dimensions: 8, + maxBatchSize: 8, + concurrency: 1, + requestIntervalMs: 0, + }, + scope: "global", + knowledgeBases: [kbDir], + indexing: { + watchFiles: false, + retries: 0, + autoGc: false, + }, + }), "opencode", { + indexPath: indexDir, + checkpointIntervalChunks: 1, + fileBatchLimits: { maxFiles: 1, maxBytes: 8 * 1024 * 1024 }, + }); + indexers.push(indexer); + return indexer; + } + + function setupGlobalScope(): { projectBDir: string; kbDir: string } { + const projectBDir = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-project-b-")); + const kbDir = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-kb-")); + const projectAFile = path.join(projectDir, "src", "alpha.ts"); + const projectBFile = path.join(projectBDir, "src", "beta.ts"); + const kbFile = path.join(kbDir, "docs", "shared.ts"); + fs.mkdirSync(path.dirname(projectBFile), { recursive: true }); + fs.mkdirSync(path.dirname(kbFile), { recursive: true }); + writeSourceFile(projectAFile, ["alphaOne", "alphaTwo"]); + writeSourceFile(projectBFile, ["betaOne", "betaTwo"]); + writeSourceFile(kbFile, ["sharedOne", "sharedTwo"]); + return { projectBDir, kbDir }; + } + + beforeEach(() => { + failEmbeddingText = null; + projectDir = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-project-")); + indexDir = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-index-")); + indexers = []; + fs.mkdirSync(path.join(projectDir, "src"), { recursive: true }); + + fetchSpy = vi.spyOn(globalThis, "fetch").mockImplementation(async ( + _url: string | URL | Request, + init?: RequestInit, + ) => { + const body = JSON.parse(String(init?.body ?? "{}")) as { input?: string[] }; + const texts = body.input ?? []; + if (failEmbeddingText !== null && texts.some((text) => text.includes(failEmbeddingText))) { + return new Response(JSON.stringify({ error: "simulated embedding failure" }), { status: 500 }); + } + return new Response(JSON.stringify({ + data: texts.map((_text, index) => ({ + embedding: Array.from({ length: 8 }, (_, dimension) => (index + dimension + 1) / 10), + })), + usage: { total_tokens: Math.max(1, texts.length) }, + }), { status: 200 }); + }); + }); + + afterEach(async () => { + await Promise.all(indexers.map((indexer) => indexer.close())); + fetchSpy.mockRestore(); + vi.unstubAllEnvs(); + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(indexDir, { recursive: true, force: true }); + }); + + it("interrupted run after a checkpoint resumes incrementally", async () => { + const indexer = createIndexer(1); + const sourceFiles = [ + path.join(projectDir, "src", "alpha.ts"), + path.join(projectDir, "src", "beta.ts"), + path.join(projectDir, "src", "gamma.ts"), + ]; + for (const [fileIndex, filePath] of sourceFiles.entries()) { + writeSourceFile( + filePath, + ["first", "second", "third", "fourth", "fifth", "sixth"].map((name) => `${name}${fileIndex}`), + ); + } + + // The embedding progress for the last batch fires after every previous + // batch has been checkpointed, so throwing there interrupts the run with + // the first two batches already durable. + await expect(indexer.index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === progress.totalFiles) { + throw new Error("simulated interruption after checkpoint"); + } + })).rejects.toThrow("simulated interruption after checkpoint"); + + const databasePath = path.join(indexDir, "codebase.db"); + const fileHashesPath = path.join(indexDir, "file-hashes.json"); + + // The batch order is not guaranteed, so derive the committed files from + // the partial hash cache: exactly two files were checkpointed before the + // interruption, and the third remains pending. + const partialHashes = JSON.parse(fs.readFileSync(fileHashesPath, "utf-8")) as Record; + const committedFiles = Object.keys(partialHashes); + expect(committedFiles.length).toBe(2); + const pendingFiles = sourceFiles + .map((filePath) => path.relative(projectDir, filePath)) + .filter((filePath) => !committedFiles.includes(filePath)); + expect(pendingFiles.length).toBe(1); + + const after = Database.openReadOnly(databasePath); + let committedChunks = 0; + try { + for (const filePath of committedFiles) { + const chunks = after.getChunksByFile(filePath); + expect(chunks.length).toBeGreaterThan(0); + committedChunks += chunks.length; + } + expect(after.getChunksByFile(pendingFiles[0])).toEqual([]); + } finally { + after.close(); + } + + expect(fs.existsSync(path.join(indexDir, "vectors"))).toBe(true); + + const callsAfterRun1 = fetchSpy.mock.calls.length; + const run1EmbeddedTexts = countEmbeddedTexts(fetchSpy, 0); + + const run2Stats = await indexer.index(); + expect(run2Stats.failedChunks).toBe(0); + + const run2EmbeddedTexts = countEmbeddedTexts(fetchSpy, callsAfterRun1); + expect(run2EmbeddedTexts).toBeGreaterThan(0); + expect(run2EmbeddedTexts).toBeLessThan(run1EmbeddedTexts); + expect(run2Stats.indexedChunks).toBe(run2EmbeddedTexts); + expect(run2Stats.existingChunks).toBe(committedChunks); + + const finalHashes = JSON.parse(fs.readFileSync(fileHashesPath, "utf-8")) as Record; + for (const filePath of [...committedFiles, ...pendingFiles]) { + expect(finalHashes[filePath]).toBeDefined(); + } + + const finalDb = Database.openReadOnly(databasePath); + try { + for (const filePath of [...committedFiles, ...pendingFiles]) { + expect(finalDb.getChunksByFile(filePath).length).toBeGreaterThan(0); + } + } finally { + finalDb.close(); + } + }); + + it("checkpoint persists failed batches", async () => { + const indexer = createIndexer(1); + const sourceFiles = [ + path.join(projectDir, "src", "alpha.ts"), + path.join(projectDir, "src", "beta.ts"), + path.join(projectDir, "src", "gamma.ts"), + ]; + // The batch order is not guaranteed, so every file contains a failing + // chunk: whichever files are checkpointed before the interruption carry a + // persisted failure record. + writeSourceFile( + sourceFiles[0], + ["first", "second", "triggerFailure", "fourth", "fifth", "sixth"].map((name) => `${name}0`), + ); + writeSourceFile( + sourceFiles[1], + ["first", "second", "triggerFailure", "fourth", "fifth", "sixth"].map((name) => `${name}1`), + ); + writeSourceFile( + sourceFiles[2], + ["first", "second", "triggerFailure", "fourth", "fifth", "sixth"].map((name) => `${name}2`), + ); + + failEmbeddingText = "triggerFailure"; + await expect(indexer.index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === progress.totalFiles) { + throw new Error("simulated interruption after failed batch checkpoint"); + } + })).rejects.toThrow("simulated interruption after failed batch checkpoint"); + + const failedBatchesPath = path.join(indexDir, "failed-batches.json"); + expect(fs.existsSync(failedBatchesPath)).toBe(true); + const persistedChunks = Array.from(readFailedBatchRecords<{ content?: string }>(failedBatchesPath)) + .flatMap((record) => record.chunks); + expect(persistedChunks.length).toBeGreaterThan(0); + expect(persistedChunks.some((chunk) => chunk.content?.includes("triggerFailure"))).toBe(true); + + failEmbeddingText = null; + const callsBeforeRun2 = fetchSpy.mock.calls.length; + const run2Stats = await indexer.index(); + expect(run2Stats.failedChunks).toBe(0); + expect(run2Stats.indexedChunks).toBeGreaterThan(0); + + const run2Texts = fetchSpy.mock.calls.slice(callsBeforeRun2).flatMap((call) => { + const body = JSON.parse(String(call[1]?.body ?? "{}")) as { input?: string[] }; + return body.input ?? []; + }); + expect(run2Texts.some((text) => text.includes("triggerFailure"))).toBe(true); + expect(fs.existsSync(failedBatchesPath)).toBe(false); + + const finalDb = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + for (const filePath of ["src/alpha.ts", "src/beta.ts", "src/gamma.ts"]) { + expect(finalDb.getChunksByFile(filePath).length).toBeGreaterThan(0); + } + } finally { + finalDb.close(); + } + }); + + it("full run result is unchanged by checkpoints", async () => { + const sourceFiles = [ + path.join(projectDir, "src", "alpha.ts"), + path.join(projectDir, "src", "beta.ts"), + path.join(projectDir, "src", "gamma.ts"), + ]; + for (const [fileIndex, filePath] of sourceFiles.entries()) { + writeSourceFile( + filePath, + ["first", "second", "third", "fourth", "fifth", "sixth"].map((name) => `${name}${fileIndex}`), + ); + } + + const checkpointedIndexDir = path.join(indexDir, "checkpointed"); + const plainIndexDir = path.join(indexDir, "plain"); + fs.mkdirSync(checkpointedIndexDir, { recursive: true }); + fs.mkdirSync(plainIndexDir, { recursive: true }); + + const checkpointedIndexer = createIndexer(1, checkpointedIndexDir); + const plainIndexer = createIndexer(undefined, plainIndexDir); + + const checkpointedStats = await checkpointedIndexer.index(); + const plainStats = await plainIndexer.index(); + + expect(checkpointedStats.totalChunks).toBe(plainStats.totalChunks); + expect(checkpointedStats.indexedChunks).toBe(plainStats.indexedChunks); + expect(checkpointedStats.existingChunks).toBe(plainStats.existingChunks); + expect(checkpointedStats.removedChunks).toBe(plainStats.removedChunks); + expect(checkpointedStats.failedChunks).toBe(plainStats.failedChunks); + expect(checkpointedStats.tokensUsed).toBe(plainStats.tokensUsed); + + const checkpointedDb = Database.openReadOnly(path.join(checkpointedIndexDir, "codebase.db")); + const plainDb = Database.openReadOnly(path.join(plainIndexDir, "codebase.db")); + try { + expect(checkpointedDb.getStats()).toEqual(plainDb.getStats()); + for (const filePath of ["src/alpha.ts", "src/beta.ts", "src/gamma.ts"]) { + expect(checkpointedDb.getChunksByFile(filePath)).toEqual(plainDb.getChunksByFile(filePath)); + } + } finally { + checkpointedDb.close(); + plainDb.close(); + } + }); + + it("interrupted forced re-embed run keeps migration pending and re-embeds every scope file on resume", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + const db = new Database(path.join(indexDir, "codebase.db")); + const projectAHash = projectIdentityHash(projectDir); + db.setMetadata(`index.embeddingStrategyVersion.${projectAHash}`, "1"); + + const resettingIndexer = createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir); + await resettingIndexer.clearIndex(); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBe("true"); + + // Interrupt the forced re-embed run right after its first checkpoint. + await expect(createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === 1) { + throw new Error("simulated interruption during forced re-embed"); + } + })).rejects.toThrow("simulated interruption during forced re-embed"); + + // The checkpoint must not clear the migration flag: the store is still + // partially migrated. + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBe("true"); + + // Resuming without force re-runs the migration and re-embeds every scope + // file, including the ones whose hashes were already checkpointed. + const beforeResumeCalls = fetchSpy.mock.calls.length; + const resumedStats = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + expect(resumedStats.failedChunks).toBe(0); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBeNull(); + + const resumeInputs = fetchSpy.mock.calls.slice(beforeResumeCalls).flatMap((call) => { + const body = JSON.parse(String(call[1]?.body ?? "{}")) as { input?: string[] }; + return body.input ?? []; + }); + expect(resumeInputs.some((text) => text.includes("alphaOne"))).toBe(true); + expect(resumeInputs.some((text) => text.includes("sharedOne"))).toBe(true); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("forced re-embed run re-embeds unchanged scope files while migration is pending", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + const db = new Database(path.join(indexDir, "codebase.db")); + const projectAHash = projectIdentityHash(projectDir); + // Simulate a pending migration whose scope files are already cached: the + // run must still re-embed them instead of skipping them as unchanged. + db.setMetadata(`index.forceReembed.${projectAHash}`, "true"); + + const beforeCalls = fetchSpy.mock.calls.length; + const stats = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + expect(stats.failedChunks).toBe(0); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBeNull(); + + const migrationInputs = fetchSpy.mock.calls.slice(beforeCalls).flatMap((call) => { + const body = JSON.parse(String(call[1]?.body ?? "{}")) as { input?: string[] }; + return body.input ?? []; + }); + expect(migrationInputs.some((text) => text.includes("alphaOne"))).toBe(true); + expect(migrationInputs.some((text) => text.includes("sharedOne"))).toBe(true); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); +}); From 5ed124eb90f22d737e14e9cef29692454d362858 Mon Sep 17 00:00:00 2001 From: Nicolas COMPAIN Date: Thu, 13 Aug 2026 04:25:46 +0200 Subject: [PATCH 2/5] fix(indexer): resume interrupted indexing runs from incremental checkpoints Interrupted indexing runs now persist incremental checkpoints (database, vectors, BM25, failed batches, and file hashes), so a subsequent run resumes incrementally instead of re-embedding the whole project. - Complete interrupted global clears scoped to the current project when foreign data is present, and replay only the indexing phase for interrupted force-indexes. - Persist the BM25 index before the vector store at checkpoints so a crash between the two writes cannot orphan chunks from keyword search. - Persist pending retries at checkpoints and exclude resolved retries from the final failed-batches file; deduplicate pending retries across checkpoints and at finalization, keeping the highest attempt count. - Preserve checkpointed artifacts on project-scope dead-lease recovery for incremental resume; reset only for clear operations and force-indexes with a clearing-phase marker. --- CHANGELOG.md | 2 +- src/indexer/index.ts | 354 ++++++++++++++----- tests/indexer-checkpoint-resume.test.ts | 442 ++++++++++++++++++++++++ 3 files changed, 707 insertions(+), 91 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d83d6b9..9e1b32d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **Interrupted indexing resume**: Interrupted indexing runs now persist incremental checkpoints (database, vectors, BM25, failed batches, and file hashes), so a subsequent run resumes incrementally instead of re-embedding the whole project. +- **Interrupted indexing resume**: Interrupted indexing runs now persist incremental checkpoints (database, vectors, BM25, failed batches, and file hashes), so a subsequent run resumes incrementally instead of re-embedding the whole project. Interrupted global clears and force-indexes are completed on recovery (scoped to the current project when foreign data is present), and checkpointed failed batches and pending retries survive for later retry runs. Checkpoint persistence orders the BM25 index before the vector store so a crash between the two cannot orphan chunks from keyword search. Project-scope dead-lease recovery preserves checkpointed artifacts for incremental resume instead of resetting the local index. Pending retry chunks are no longer duplicated across multiple checkpoints, and finalization deduplicates failed-batch records by chunk ID (keeping the highest attempt count). - **Forced re-embed migration safety**: While a project-scoped forced re-embed is pending, checkpoints no longer stamp new embedding metadata or a compatibility certificate, unchanged scope files are re-embedded instead of skipped, and `retryFailedBatches` only clears the pending migration after the main run has committed its new metadata. This prevents an interrupted migration from resuming into a silent mix of old and new vector spaces. - **Conceptual context retrieval**: Prefer implementation paths over tests and documentation for code-focused `codebase_context` searches, without applying a hard source-only filter or changing documentation and test queries. - **Scoped definition lookup**: Allow explicit definition searches constrained to a file type or directory to return matching declarations in fixtures and test paths without weakening source-first ranking for ordinary searches. diff --git a/src/indexer/index.ts b/src/indexer/index.ts index 0850fda..a7990da 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -102,6 +102,7 @@ import { import { createFailedBatchWriter, readFailedBatchRecords, + writeFailedBatchRecords, type FailedBatchRecordInput, type FailedBatchWriter, } from "./failed-state-persistence.js"; @@ -493,6 +494,7 @@ interface FailedChunkRecordMetadata { attemptCount: number; error: string; lastAttempt: string; + chunks: unknown[]; } interface FailedBatchWriteState { @@ -1431,6 +1433,7 @@ export class Indexer { private loadFileHashCache(): void { if (!existsSync(this.fileHashCachePath)) { + this.fileHashCache = new Map(); return; } @@ -1593,6 +1596,10 @@ export class Indexer { return `index.forceReembed.${this.projectIdentityHash}`; } + private getProjectMigrationFinalizedMetadataKey(): string { + return `index.migrationFinalized.${this.projectIdentityHash}`; + } + private getBranchMigrationMetadataKey( prefix: string, catalogIdentity = this.getBranchCatalogIdentity(), @@ -1988,8 +1995,12 @@ export class Indexer { database.gcOrphanEmbeddings(); database.gcOrphanChunks(); - store.save(); + // Persist the keyword index before the vector store: a crash between the + // two leaves the store as the conservative resume authority, so the next + // run re-embeds and repopulates BM25 instead of skipping addChunk for + // chunks whose vectors are already durable. this.saveInvertedIndex(invertedIndex); + store.save(); return { removedChunkIds: removedChunkIdList, @@ -1997,6 +2008,26 @@ export class Indexer { }; } + private getForceIndexPhaseMarkerPath(): string { + return path.join(this.indexPath, "force-index-phase"); + } + + private writeForceIndexPhaseMarker(): void { + writeFileSync(this.getForceIndexPhaseMarkerPath(), "clearing", { encoding: "utf-8" }); + } + + private removeForceIndexPhaseMarker(): void { + try { + unlinkSync(this.getForceIndexPhaseMarkerPath()); + } catch { + // Best-effort cleanup. + } + } + + private hasForceIndexPhaseMarker(): boolean { + return existsSync(this.getForceIndexPhaseMarkerPath()); + } + private async recoverFromInterruptedIndexingUnlocked(owners: readonly IndexLockOwner[]): Promise { for (const owner of owners) { this.logger.warn("Detected interrupted indexing session, recovering...", { @@ -2008,14 +2039,25 @@ export class Indexer { } if (this.config.scope === "global") { - if (existsSync(this.fileHashCachePath)) { - unlinkSync(this.fileHashCachePath); + const shouldCompleteClear = owners.some( + (owner) => owner.operation === "clear" || (owner.operation === "force-index" && this.hasForceIndexPhaseMarker()), + ); + if (shouldCompleteClear) { + // Re-apply the clear decision: a global clear only wipes the whole + // shared index when no foreign project data is present, otherwise it + // stays scoped to the current project. + this.clearGlobalIndexUnlocked(); } - await this.healthCheckUnlocked(); + this.logger.info( + shouldCompleteClear + ? "Recovery complete, next index will rebuild all files" + : "Recovery complete, next index will resume from the last checkpoint", + ); + return; } - this.logger.info("Recovery complete, next index will re-process all files"); + this.logger.info("Recovery complete, next index will resume from the last checkpoint"); } private *loadSerializedFailedBatches(): Generator { @@ -2061,9 +2103,38 @@ export class Indexer { state.recordsWritten += record.chunks.length; } - private finalizeFailedBatchWriteState(state: FailedBatchWriteState): void { + private finalizeFailedBatchWriteState( + state: FailedBatchWriteState, + resolvedChunkIds: ReadonlySet = new Set(), + ): void { if (state.recordsWritten > 0) { - state.writer.commit(); + // Deduplicate by chunk ID and drop resolved retries. Process in reverse + // so the last-written record (highest attemptCount) wins; the + // checkpoint reconstruction loop and the retry phase can both + // materialize the same pending retry. + const seenChunkIds = new Set(); + const retained: FailedBatchRecordInput[] = []; + const records = Array.from(readFailedBatchRecords(state.writer.temporaryPath)); + for (let i = records.length - 1; i >= 0; i--) { + const chunks = records[i].chunks.filter((rawChunk) => { + const chunkId = getPendingChunkId(rawChunk); + if (chunkId !== null) { + if (resolvedChunkIds.has(chunkId)) return false; + if (seenChunkIds.has(chunkId)) return false; + seenChunkIds.add(chunkId); + } + return true; + }); + if (chunks.length > 0) { + retained.unshift({ ...records[i], chunks }); + } + } + state.writer.cleanup(); + if (retained.length > 0) { + writeFailedBatchRecords(this.failedBatchesPath, retained); + } else { + this.clearFailedBatchState(); + } return; } @@ -2071,29 +2142,75 @@ export class Indexer { this.clearFailedBatchState(); } + private getCheckpointIntervalChunks(totalChunks: number): number { + return Math.max( + this.checkpointIntervalChunks ?? 2000, + Math.floor(totalChunks / 10), + ); + } + private checkpointIndexRun( database: Database, store: VectorStore, invertedIndex: InvertedIndex, - failedProcessing: { state: FailedBatchWriteState; latestById: Map }, + failedProcessing: { state: FailedBatchWriteState; latestById: Map; materializedRetryIds: Set }, currentFileHashes: Map, committedFilePaths: Set, + unchangedFilePaths: Set, scopedRoots: string[] | null, configuredProviderInfo: ConfiguredProviderInfo, - indexedCommit: string | null, ): void { - database.commitWriteTransaction(); - database.beginWriteTransaction(); if (!this.hasProjectForceReembedPending()) { this.saveIndexMetadata(configuredProviderInfo); - this.saveBranchCommit(database, indexedCommit); this.indexCompatibility = { compatible: true }; } - store.save(); + database.commitWriteTransaction(); + database.beginWriteTransaction(); + // Persist the keyword index before the vector store: a crash between the + // two leaves the store as the conservative resume authority, so the next + // run re-embeds and repopulates BM25 instead of skipping addChunk for + // chunks whose vectors are already durable. this.saveInvertedIndex(invertedIndex); - if (failedProcessing.state.recordsWritten > 0) { + store.save(); + if (failedProcessing.state.recordsWritten > 0 || failedProcessing.latestById.size > 0) { + // Persist the pending retries alongside the written records so a crash + // after this checkpoint cannot lose them. + for (const metadata of failedProcessing.latestById.values()) { + const alreadyMaterialized = metadata.chunks.some((rawChunk) => { + const chunkId = getPendingChunkId(rawChunk); + return chunkId !== null && failedProcessing.materializedRetryIds.has(chunkId); + }); + if (alreadyMaterialized) continue; + this.writeFailedBatchRecord(failedProcessing.state, { + chunks: metadata.chunks, + attemptCount: metadata.attemptCount, + error: metadata.error, + lastAttempt: metadata.lastAttempt, + }); + for (const rawChunk of metadata.chunks) { + const chunkId = getPendingChunkId(rawChunk); + if (chunkId !== null) { + failedProcessing.materializedRetryIds.add(chunkId); + } + } + } failedProcessing.state.writer.commit(); failedProcessing.state = this.createFailedBatchWriteState(); + // Preserve the committed records (including out-of-scope projects' + // failed batches) so finalization never deletes them, but drop retries + // that were already completed during this run. + for (const record of this.loadSerializedFailedBatches()) { + for (const rawChunk of record.chunks) { + const chunkId = getPendingChunkId(rawChunk); + const filePath = getPendingChunkFilePath(rawChunk); + const inScope = scopedRoots === null || (filePath !== null && this.isFileInCurrentScope(filePath, scopedRoots)); + const isPendingRetry = chunkId !== null && failedProcessing.latestById.has(chunkId); + const isCompletedRetry = !isPendingRetry && inScope && filePath !== null && unchangedFilePaths.has(filePath); + if (!isCompletedRetry) { + this.writeFailedBatchRecord(failedProcessing.state, { ...record, chunks: [rawChunk] }); + } + } + } } const partialHashes = new Map(); for (const filePath of committedFilePaths) { @@ -2139,7 +2256,7 @@ export class Indexer { private prepareFailedBatchProcessing( roots: string[] | null, shouldProcess: (filePath: string | null) => boolean, - ): { state: FailedBatchWriteState; latestById: Map } { + ): { state: FailedBatchWriteState; latestById: Map; materializedRetryIds: Set } { const state = this.createFailedBatchWriteState(); const latestById = new Map(); @@ -2166,11 +2283,12 @@ export class Indexer { attemptCount: batch.attemptCount, error: batch.error, lastAttempt: batch.lastAttempt, + chunks: [rawChunk], }); } } } - return { state, latestById }; + return { state, latestById, materializedRetryIds: new Set() }; } catch (error) { state.writer.cleanup(); throw error; @@ -3073,7 +3191,12 @@ export class Indexer { ]); } if (recoveredOwners.length > 0 && this.config.scope === "project") { - await this.resetLocalIndexArtifacts(); + const shouldReset = recoveredOwners.some( + (owner) => owner.operation === "clear" || (owner.operation === "force-index" && this.hasForceIndexPhaseMarker()), + ); + if (shouldReset) { + await this.resetLocalIndexArtifacts(); + } } this.store = new VectorStore(storePath, dimensions); @@ -4254,6 +4377,10 @@ export class Indexer { } if (symbolBatch.length > 0) { database.upsertSymbolsBatch(symbolBatch); + database.addSymbolsToBranchBatch( + this.getBranchCatalogKey(), + symbolBatch.map((symbol) => symbol.id), + ); } if (edgeBatch.length > 0) { database.upsertCallEdgesBatch(edgeBatch); @@ -4291,6 +4418,12 @@ export class Indexer { forceReembed: forceScopedReembed, reuseCachedEmbeddings: true, incrementRepeatedFailures: true, + onSucceeded: (succeededChunks) => { + database.addChunksToBranchBatch( + this.getBranchCatalogKey(), + succeededChunks.map((chunk) => chunk.id), + ); + }, onProgress: (batchProgress) => onProgress?.({ phase: "embedding", filesProcessed: unchangedFilePaths.size + processedChangedFiles, @@ -4311,12 +4444,12 @@ export class Indexer { } for (const descriptor of descriptorBatch) { - committedFilePaths.add(descriptor.storedPath); + const existingFileChunks = existingChunksByFile.get(descriptor.storedPath); + if (!existingFileChunks || existingFileChunks.size === 0) { + committedFilePaths.add(descriptor.storedPath); + } } - const checkpointInterval = Math.max( - this.checkpointIntervalChunks ?? 2000, - Math.floor(stats.totalChunks / 10), - ); + const checkpointInterval = this.getCheckpointIntervalChunks(stats.totalChunks); if (stats.totalChunks - lastCheckpointChunks >= checkpointInterval) { lastCheckpointChunks = stats.totalChunks; this.checkpointIndexRun( @@ -4326,13 +4459,14 @@ export class Indexer { failedProcessing, currentFileHashes, committedFilePaths, + unchangedFilePaths, scopedRoots, configuredProviderInfo, - indexedCommit, ); } } + const resolvedRetryChunkIds = new Set(); const retryableFailedChunks = this.iterateLatestFailedChunks( failedProcessing.latestById, scopedRoots, @@ -4374,6 +4508,16 @@ export class Indexer { forceReembed: forceScopedReembed, reuseCachedEmbeddings: true, incrementRepeatedFailures: true, + onSucceeded: (succeededChunks) => { + database.addChunksToBranchBatch( + this.getBranchCatalogKey(), + succeededChunks.map((chunk) => chunk.id), + ); + for (const chunk of succeededChunks) { + failedProcessing.latestById.delete(chunk.id); + resolvedRetryChunkIds.add(chunk.id); + } + }, onProgress: (batchProgress) => onProgress?.({ phase: "embedding", filesProcessed: files.length, @@ -4391,6 +4535,20 @@ export class Indexer { failedForcedChunkIds.add(chunkId); } } + if (stats.totalChunks - lastCheckpointChunks >= this.getCheckpointIntervalChunks(stats.totalChunks)) { + lastCheckpointChunks = stats.totalChunks; + this.checkpointIndexRun( + database, + store, + invertedIndex, + failedProcessing, + currentFileHashes, + committedFilePaths, + unchangedFilePaths, + scopedRoots, + configuredProviderInfo, + ); + } } const removedChunkIds: string[] = []; @@ -4438,7 +4596,7 @@ export class Indexer { this.fileHashCache = currentFileHashes; this.saveFileHashCache(); } - this.finalizeFailedBatchWriteState(failedProcessing.state); + this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds); database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION); database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION); database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION); @@ -4477,7 +4635,7 @@ export class Indexer { this.fileHashCache = currentFileHashes; this.saveFileHashCache(); } - this.finalizeFailedBatchWriteState(failedProcessing.state); + this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds); database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION); database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION); database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION); @@ -4529,7 +4687,7 @@ export class Indexer { this.fileHashCache = currentFileHashes; this.saveFileHashCache(); } - this.finalizeFailedBatchWriteState(failedProcessing.state); + this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds); database.commitWriteTransaction(); writeTransactionActive = false; @@ -4558,6 +4716,9 @@ export class Indexer { if (forceScopedReembed && failedForcedChunkIds.size === 0) { database.deleteMetadata(this.getProjectForceReembedMetadataKey()); } + if (forceScopedReembed) { + database.setMetadata(this.getProjectMigrationFinalizedMetadataKey(), "true"); + } database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION); database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION); database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION); @@ -5257,7 +5418,11 @@ export class Indexer { async forceIndex(onProgress?: ProgressCallback): Promise { return this.withIndexMutationLease("force-index", async (recoveredOwners) => { await this.ensureInitializedUnlocked(recoveredOwners); + // Mark the clearing phase so recovery can distinguish a crash before + // the clear from a crash during indexing. + this.writeForceIndexPhaseMarker(); await this.clearIndexUnlocked(); + this.removeForceIndexPhaseMarker(); return this.indexUnlocked(onProgress, [], true); }); } @@ -5269,74 +5434,85 @@ export class Indexer { }); } - private async clearIndexUnlocked(): Promise { + private clearGlobalIndexDataUnlocked(): void { const { store, invertedIndex, database } = this.requireLoadedIndexState(); + const clearedBranchKeys = database.getAllBranches(); + store.clear(); + store.save(); + invertedIndex.clear(); + this.saveInvertedIndex(invertedIndex); - if (this.config.scope === "global") { - store.load(); - invertedIndex.load(); - this.loadFileHashCache(); - const roots = this.getScopedRoots(); - const compatibility = this.checkCompatibility(); - const allMetadata = store.getAllMetadata(); - const hasForeignData = - allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || - this.hasForeignScopedBranchData() || - this.hasForeignScopedFileHashData(roots) || - this.hasForeignScopedFailedBatches(roots); - - if (!compatibility.compatible && hasForeignData) { - if (compatibility.code === IncompatibilityCode.EMBEDDING_STRATEGY_MISMATCH) { - this.clearSharedIndexProjectData(store, invertedIndex, database, roots); - this.clearScopedFileHashCache(roots); - this.clearScopedFailedBatches(roots); - database.setMetadata(this.getProjectForceReembedMetadataKey(), "true"); - database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey()); - this.indexCompatibility = { compatible: true }; - return; - } - - throw new Error( - `Global index compatibility reset is unsafe because the shared index contains files from other projects. ` + - `The current global index cannot be force-rebuilt for ${this.projectRoot} without deleting other repositories' indexed data. ` + - `Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.` - ); - } + this.fileHashCache.clear(); + this.saveFileHashCache(); - if (!hasForeignData) { - const clearedBranchKeys = database.getAllBranches(); - store.clear(); - store.save(); - invertedIndex.clear(); - this.saveInvertedIndex(invertedIndex); + database.clearAllIndexedData(); + this.deleteBranchCommitMetadata(database, clearedBranchKeys); + this.clearFailedBatchState(); - this.fileHashCache.clear(); - this.saveFileHashCache(); + database.deleteMetadata("index.version"); + database.deleteMetadata("index.pathStorageVersion"); + database.deleteMetadata("index.embeddingProvider"); + database.deleteMetadata("index.embeddingModel"); + database.deleteMetadata("index.embeddingDimensions"); + database.deleteMetadata("index.embeddingStrategyVersion"); + database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey()); + database.deleteMetadata(this.getProjectForceReembedMetadataKey()); + database.deleteMetadata(this.getLegacyMigrationMetadataKey()); + database.deleteMetadata("index.createdAt"); + database.deleteMetadata("index.updatedAt"); - database.clearAllIndexedData(); - this.deleteBranchCommitMetadata(database, clearedBranchKeys); - this.clearFailedBatchState(); + this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo!); + } - database.deleteMetadata("index.version"); - database.deleteMetadata("index.pathStorageVersion"); - database.deleteMetadata("index.embeddingProvider"); - database.deleteMetadata("index.embeddingModel"); - database.deleteMetadata("index.embeddingDimensions"); - database.deleteMetadata("index.embeddingStrategyVersion"); + private clearGlobalIndexUnlocked(): void { + const { store, invertedIndex, database } = this.requireLoadedIndexState(); + store.load(); + invertedIndex.load(); + this.loadFileHashCache(); + const roots = this.getScopedRoots(); + const compatibility = this.checkCompatibility(); + const allMetadata = store.getAllMetadata(); + const hasForeignData = + allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || + this.hasForeignScopedBranchData() || + this.hasForeignScopedFileHashData(roots) || + this.hasForeignScopedFailedBatches(roots); + + if (!compatibility.compatible && hasForeignData) { + if (compatibility.code === IncompatibilityCode.EMBEDDING_STRATEGY_MISMATCH) { + this.clearSharedIndexProjectData(store, invertedIndex, database, roots); + this.clearScopedFileHashCache(roots); + this.clearScopedFailedBatches(roots); + database.setMetadata(this.getProjectForceReembedMetadataKey(), "true"); database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey()); - database.deleteMetadata(this.getProjectForceReembedMetadataKey()); - database.deleteMetadata(this.getLegacyMigrationMetadataKey()); - database.deleteMetadata("index.createdAt"); - database.deleteMetadata("index.updatedAt"); - - this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo!); + database.deleteMetadata(this.getProjectMigrationFinalizedMetadataKey()); + this.indexCompatibility = { compatible: true }; return; } - this.clearSharedIndexProjectData(store, invertedIndex, database, roots); - this.clearScopedFileHashCache(roots); - this.clearScopedFailedBatches(roots); - this.indexCompatibility = compatibility; + throw new Error( + `Global index compatibility reset is unsafe because the shared index contains files from other projects. ` + + `The current global index cannot be force-rebuilt for ${this.projectRoot} without deleting other repositories' indexed data. ` + + `Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.` + ); + } + + if (!hasForeignData) { + this.clearGlobalIndexDataUnlocked(); + return; + } + + this.clearSharedIndexProjectData(store, invertedIndex, database, roots); + this.clearScopedFileHashCache(roots); + this.clearScopedFailedBatches(roots); + this.indexCompatibility = compatibility; + } + + private async clearIndexUnlocked(): Promise { + const { store, invertedIndex, database } = this.requireLoadedIndexState(); + + if (this.config.scope === "global") { + this.clearGlobalIndexUnlocked(); return; } @@ -5572,12 +5748,9 @@ export class Indexer { } if (roots && succeeded > 0 && remaining === 0 && this.hasProjectForceReembedPending()) { - const storedProvider = database.getMetadata("index.embeddingProvider"); - const storedModel = database.getMetadata("index.embeddingModel"); - const migrationCommitted = - storedProvider === configuredProviderInfo.provider && - storedModel === configuredProviderInfo.modelInfo.model; - if (migrationCommitted) { + const migrationFinalized = + database.getMetadata(this.getProjectMigrationFinalizedMetadataKey()) === "true"; + if (migrationFinalized) { database.deleteMetadata(this.getProjectForceReembedMetadataKey()); this.saveIndexMetadata(configuredProviderInfo); this.indexCompatibility = { compatible: true }; @@ -5606,6 +5779,7 @@ export class Indexer { attemptCount: batch.attemptCount, error: batch.error, lastAttempt: batch.lastAttempt, + chunks: [rawChunk], }); } } diff --git a/tests/indexer-checkpoint-resume.test.ts b/tests/indexer-checkpoint-resume.test.ts index 32aacce..ca15bd7 100644 --- a/tests/indexer-checkpoint-resume.test.ts +++ b/tests/indexer-checkpoint-resume.test.ts @@ -1,3 +1,4 @@ +import { spawnSync } from "node:child_process"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; @@ -330,6 +331,212 @@ describe("indexer checkpoint resume", () => { } }); + it("interrupted run on an existing branch keeps checkpointed chunks in the branch catalog after resume", async () => { + const indexer = createIndexer(1); + const alphaFile = path.join(projectDir, "src", "alpha.ts"); + writeSourceFile(alphaFile, ["alphaOne", "alphaTwo"]); + + // Full initial index publishes the branch catalog. + await indexer.index(); + + // Add two new files and interrupt the run after the first checkpoint. + const betaFile = path.join(projectDir, "src", "beta.ts"); + const gammaFile = path.join(projectDir, "src", "gamma.ts"); + writeSourceFile(betaFile, ["betaOne", "betaTwo"]); + writeSourceFile(gammaFile, ["gammaOne", "gammaTwo"]); + await expect(indexer.index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === progress.totalFiles) { + throw new Error("simulated interruption after new-file checkpoint"); + } + })).rejects.toThrow("simulated interruption after new-file checkpoint"); + + // The checkpointed file's chunks must survive in the branch catalog after + // the resume, even though the file itself is skipped as unchanged. + const resumedStats = await indexer.index(); + expect(resumedStats.failedChunks).toBe(0); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + const branchChunkIds = new Set(db.getBranchChunkIds("default")); + for (const filePath of ["src/alpha.ts", "src/beta.ts", "src/gamma.ts"]) { + const chunks = db.getChunksByFile(filePath); + expect(chunks.length).toBeGreaterThan(0); + for (const chunk of chunks) { + expect(branchChunkIds.has(chunk.chunkId)).toBe(true); + } + } + const branchSymbolIds = new Set(db.getBranchSymbolIds("default")); + for (const filePath of ["src/alpha.ts", "src/beta.ts", "src/gamma.ts"]) { + for (const symbol of db.getSymbolsByFile(filePath)) { + expect(branchSymbolIds.has(symbol.id)).toBe(true); + } + } + } finally { + db.close(); + } + }); + + it("interrupted run with a modified file re-processes it on resume", async () => { + const indexer = createIndexer(1); + const alphaFile = path.join(projectDir, "src", "alpha.ts"); + writeSourceFile(alphaFile, ["alphaOne", "alphaTwo"]); + + await indexer.index(); + + // Modify the file and add a new one; interrupt after the first checkpoint. + writeSourceFile(alphaFile, ["alphaOne", "alphaThree"]); + const betaFile = path.join(projectDir, "src", "beta.ts"); + writeSourceFile(betaFile, ["betaOne", "betaTwo"]); + await expect(indexer.index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === progress.totalFiles) { + throw new Error("simulated interruption after modified-file checkpoint"); + } + })).rejects.toThrow("simulated interruption after modified-file checkpoint"); + + // A modified file with stale chunks to evict must not be checkpointed. + const fileHashesPath = path.join(indexDir, "file-hashes.json"); + const partialHashes = JSON.parse(fs.readFileSync(fileHashesPath, "utf-8")) as Record; + expect(partialHashes["src/alpha.ts"]).toBeUndefined(); + + // The resume re-processes it and evicts the removed chunk. + const resumedStats = await indexer.index(); + expect(resumedStats.failedChunks).toBe(0); + expect(resumedStats.removedChunks).toBeGreaterThan(0); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + const chunks = db.getChunksByFile("src/alpha.ts"); + expect(chunks.some((chunk) => chunk.name === "alphaTwo")).toBe(false); + expect(chunks.some((chunk) => chunk.name === "alphaThree")).toBe(true); + } finally { + db.close(); + } + }); + + it("global lease recovery invalidates the hash cache after an interrupted clear", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + // Simulate a crashed clear: an orphaned lease owned by a dead pid. + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "clear", + token: "11111111-1111-4111-8111-111111111111", + })); + + const fileHashesPath = path.join(indexDir, "file-hashes.json"); + expect(fs.existsSync(fileHashesPath)).toBe(true); + + // A crashed clear wipes the store and inverted index before the hash cache + // (clearIndexUnlocked order): simulate that partial state so the next run + // would otherwise skip every file against an empty store. + for (const artifact of ["vectors", "vectors.usearch", "vectors.meta.json", "inverted-index.json"]) { + fs.rmSync(path.join(indexDir, artifact), { recursive: true, force: true }); + } + + // The recovery must invalidate the cache after an interrupted clear so the + // next run re-embeds instead of skipping files against an empty store. + const beforeCalls = fetchSpy.mock.calls.length; + const stats = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + expect(stats.failedChunks).toBe(0); + expect(fetchSpy.mock.calls.length).toBeGreaterThan(beforeCalls); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("global lease recovery preserves the checkpointed hash cache", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + // Simulate a crashed indexing session: an orphaned lease owned by a dead pid. + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "index", + token: "11111111-1111-4111-8111-111111111111", + })); + + const fileHashesPath = path.join(indexDir, "file-hashes.json"); + expect(fs.existsSync(fileHashesPath)).toBe(true); + + // The recovery must keep the checkpointed hash cache: unchanged files are + // skipped instead of being re-parsed and re-embedded. + const beforeCalls = fetchSpy.mock.calls.length; + const stats = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + expect(stats.failedChunks).toBe(0); + expect(fs.existsSync(fileHashesPath)).toBe(true); + expect(fetchSpy.mock.calls.length).toBe(beforeCalls); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("retryFailedBatches does not close an interrupted strategy migration", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + const db = new Database(path.join(indexDir, "codebase.db")); + const projectAHash = projectIdentityHash(projectDir); + db.setMetadata(`index.embeddingStrategyVersion.${projectAHash}`, "1"); + + const resettingIndexer = createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir); + await resettingIndexer.clearIndex(); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBe("true"); + + // Interrupt the forced re-embed run after its first checkpoint, with a + // failing chunk so the run leaves failed batches behind. + failEmbeddingText = "One"; + await expect(createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === 1) { + throw new Error("simulated interruption during forced re-embed"); + } + })).rejects.toThrow("simulated interruption during forced re-embed"); + + // The retry recovers the failed chunks but must not close the migration: + // the main run never finished, so unprocessed scope files still hold old + // vectors. + failEmbeddingText = null; + const retry = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).retryFailedBatches(); + expect(retry.remaining).toBe(0); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBe("true"); + + // A full run completes the migration and clears the flag. + const stats = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + expect(stats.failedChunks).toBe(0); + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBeNull(); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + it("interrupted forced re-embed run keeps migration pending and re-embeds every scope file on resume", async () => { const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); vi.stubEnv("HOME", tempHome); @@ -408,4 +615,239 @@ describe("indexer checkpoint resume", () => { fs.rmSync(kbDir, { recursive: true, force: true }); fs.rmSync(tempHome, { recursive: true, force: true }); }); + + it("checkpoint preserves out-of-scope failed batches for later retries", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + // Project A leaves a failed batch behind. + failEmbeddingText = "One"; + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + const failedBatchesPath = path.join(indexDir, "failed-batches.json"); + expect(fs.existsSync(failedBatchesPath)).toBe(true); + + // Project B's run checkpoints and must not drop A's failed batch. + failEmbeddingText = null; + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + expect(fs.existsSync(failedBatchesPath)).toBe(true); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("checkpoint preserves pending retries alongside new failures", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + // Project A leaves a failed batch behind. + failEmbeddingText = "One"; + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + + // A second run checkpoints a new failure while the old retry is still + // pending: the retry must survive the checkpoint and be re-attempted. + writeSourceFile(path.join(kbDir, "docs", "shared.ts"), ["sharedOne", "sharedTwo", "sharedThree"]); + failEmbeddingText = "Three"; + const beforeCalls = fetchSpy.mock.calls.length; + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + const retryInputs = fetchSpy.mock.calls.slice(beforeCalls).flatMap((call) => { + const body = JSON.parse(String(call[1]?.body ?? "{}")) as { input?: string[] }; + return body.input ?? []; + }); + expect(retryInputs.some((text) => text.includes("alphaOne"))).toBe(true); + + // The resolved retry must not be republished in the final failed-batches + // file: only the new failure remains. + const persisted = fs.readFileSync(path.join(indexDir, "failed-batches.json"), "utf-8"); + expect(persisted).toContain("sharedThree"); + expect(persisted).not.toContain("alphaOne"); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("interrupted force-index run resumes from checkpoints instead of clearing", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + // The force-index clear phase completes, then the indexing phase is + // interrupted after the first file is checkpointed. + await expect(createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).forceIndex((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === 2) { + throw new Error("simulated interruption during force-index"); + } + })).rejects.toThrow("simulated interruption during force-index"); + + // Simulate a crash during the indexing phase: an orphaned "force-index" + // lease with no clearing-phase marker left behind. + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "force-index", + token: "11111111-1111-4111-8111-111111111111", + })); + + // The recovery must not clear the checkpointed data: the next run resumes + // incrementally instead of re-embedding the already checkpointed file. + const beforeCalls = fetchSpy.mock.calls.length; + const stats = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + expect(stats.failedChunks).toBe(0); + const resumeInputs = fetchSpy.mock.calls.slice(beforeCalls).flatMap((call) => { + const body = JSON.parse(String(call[1]?.body ?? "{}")) as { input?: string[] }; + return body.input ?? []; + }); + expect(resumeInputs.some((text) => text.includes("sharedOne"))).toBe(true); + expect(resumeInputs.some((text) => text.includes("alphaOne"))).toBe(false); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("checkpoint persists the keyword index before vectors so a crash cannot orphan BM25 chunks", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + // Interrupt the first run between the vector store save and the keyword + // index save: with the keyword index persisted first, the store is not yet + // durable and the resume re-embeds, keeping BM25 complete. + const invertedSave = vi.spyOn( + Indexer.prototype as unknown as { saveInvertedIndex: () => void }, + "saveInvertedIndex", + ).mockImplementation(() => { + throw new Error("simulated crash between artifact writes"); + }); + await expect(createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index()).rejects.toThrow( + "simulated crash between artifact writes", + ); + invertedSave.mockRestore(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + const inverted = fs.readFileSync(path.join(indexDir, "inverted-index.json"), "utf-8"); + expect(inverted).toContain("alphaone"); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("project-scope dead-lease recovery preserves checkpoints for incremental resume", async () => { + const projectOwnedIndex = path.join(projectDir, ".opencode", "index"); + const sourceFiles = [ + path.join(projectDir, "src", "alpha.ts"), + path.join(projectDir, "src", "beta.ts"), + path.join(projectDir, "src", "gamma.ts"), + ]; + for (const [fileIndex, filePath] of sourceFiles.entries()) { + writeSourceFile( + filePath, + ["first", "second", "third", "fourth", "fifth", "sixth"].map((name) => `${name}${fileIndex}`), + ); + } + + // Interrupt after two files are checkpointed. + const indexer = createIndexer(1, projectOwnedIndex); + await expect(indexer.index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === progress.totalFiles) { + throw new Error("simulated crash after checkpoint"); + } + })).rejects.toThrow("simulated crash after checkpoint"); + + // Simulate a dead "index" lease (not "clear" or "force-index"): recovery + // must preserve checkpointed artifacts and resume incrementally. + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(projectOwnedIndex, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "index", + token: "22222222-2222-4222-8222-222222222222", + })); + + const beforeCalls = fetchSpy.mock.calls.length; + const resumeIndexer = createIndexer(1, projectOwnedIndex); + const stats = await resumeIndexer.index(); + expect(stats.failedChunks).toBe(0); + + // Only the pending file was re-embedded: the two checkpointed files are + // preserved and skipped on resume. + const resumeInputs = fetchSpy.mock.calls.slice(beforeCalls).flatMap((call) => { + const body = JSON.parse(String(call[1]?.body ?? "{}")) as { input?: string[] }; + return body.input ?? []; + }); + expect(resumeInputs.some((text) => text.includes("first2"))).toBe(true); + expect(resumeInputs.some((text) => text.includes("first0"))).toBe(false); + expect(resumeInputs.some((text) => text.includes("first1"))).toBe(false); + }); + it("checkpoint drops stale failures for files modified and reindexed successfully", async () => { + const alphaFile = path.join(projectDir, "src", "alpha.ts"); + const gammaFile = path.join(projectDir, "src", "gamma.ts"); + writeSourceFile(alphaFile, ["alphaOne", "alphaTwo"]); + + // Run 1: alphaOne fails embedding, leaving a failed-batches record. + failEmbeddingText = "alphaOne"; + const indexer1 = createIndexer(1); + await indexer1.index(); + expect(fs.existsSync(path.join(indexDir, "failed-batches.json"))).toBe(true); + const failedContent1 = fs.readFileSync(path.join(indexDir, "failed-batches.json"), "utf-8"); + expect(failedContent1).toContain("alphaOne"); + failEmbeddingText = null; + + // Run 2: alpha.ts is modified and reindexed successfully, while a new + // file gamma.ts has a chunk that fails — ensuring the checkpoint's + // failed-batches reconstruction path executes. + writeSourceFile(alphaFile, ["alphaOne", "alphaTwo", "alphaThree"]); + writeSourceFile(gammaFile, ["gammaOne"]); + failEmbeddingText = "gammaOne"; + const indexer2 = createIndexer(1); + await indexer2.index(); + + // The old alphaOne failure must not survive: alpha.ts was modified and + // all its chunks embedded successfully. Only gammaOne should remain. + const failedContent2 = fs.readFileSync(path.join(indexDir, "failed-batches.json"), "utf-8"); + expect(failedContent2).toContain("gammaOne"); + expect(failedContent2).not.toContain("alphaOne"); + }); + + it("checkpoint does not duplicate pending retry chunks across multiple checkpoints", async () => { + const alphaFile = path.join(projectDir, "src", "alpha.ts"); + const betaFile = path.join(projectDir, "src", "beta.ts"); + const gammaFile = path.join(projectDir, "src", "gamma.ts"); + writeSourceFile(alphaFile, ["alphaOne", "alphaTwo"]); + + // Run 1: alphaOne fails, leaving a failed-batches record. + failEmbeddingText = "alphaOne"; + await createIndexer(1).index(); + failEmbeddingText = null; + + // Run 2: alpha.ts is unchanged (alphaOne stays in latestById for retry), + // while two new files beta.ts and gamma.ts each fail — triggering two + // separate checkpoints. Without deduplication, alphaOne would be + // re-written from latestById at every checkpoint, duplicating it. + writeSourceFile(betaFile, ["betaOne", "betaTwo"]); + writeSourceFile(gammaFile, ["gammaOne", "gammaTwo"]); + failEmbeddingText = "One"; + await createIndexer(1).index(); + + // The failed-batches file should contain exactly one record for + // alphaOne, not one per checkpoint. + const failedContent = fs.readFileSync(path.join(indexDir, "failed-batches.json"), "utf-8"); + const alphaOneOccurrences = (failedContent.match(/"name":"alphaOne"/g) ?? []).length; + expect(alphaOneOccurrences).toBe(1); + }); }); From 91ce2eff650cc4dd4562f2247834c2df0b08b24d Mon Sep 17 00:00:00 2001 From: Nicolas COMPAIN Date: Thu, 13 Aug 2026 19:52:39 +0200 Subject: [PATCH 3/5] fix(indexer): scope interrupted recovery to the originating project Address review feedback on interrupted global-index recovery: - Persist the originating project root and scoped roots in the index lock owner, so a recovered global clear or clearing-phase force-index is replayed against the project that started it instead of the project that reclaims the dead lease. Load the persisted file-hash cache before replaying clears so the scoped purge is written back. - Restore missing SQLite chunk rows before retrying failed batches: a recovery health check can collect checkpointed failure rows as orphans, which previously left branch references without chunk metadata. - Keep the reclaiming project's branch catalog intact during a cross-project clear recovery: current-project branch keys are only added to the cleanup set when the cleared project is the current one. Adds regression tests for both recovery scenarios and updates the changelog. --- CHANGELOG.md | 1 + src/indexer/index-lock.ts | 33 ++++- src/indexer/index.ts | 167 ++++++++++++++++------- tests/indexer-checkpoint-resume.test.ts | 173 ++++++++++++++++++++++++ 4 files changed, 320 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e1b32d..064f492 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **Interrupted indexing resume**: Interrupted indexing runs now persist incremental checkpoints (database, vectors, BM25, failed batches, and file hashes), so a subsequent run resumes incrementally instead of re-embedding the whole project. Interrupted global clears and force-indexes are completed on recovery (scoped to the current project when foreign data is present), and checkpointed failed batches and pending retries survive for later retry runs. Checkpoint persistence orders the BM25 index before the vector store so a crash between the two cannot orphan chunks from keyword search. Project-scope dead-lease recovery preserves checkpointed artifacts for incremental resume instead of resetting the local index. Pending retry chunks are no longer duplicated across multiple checkpoints, and finalization deduplicates failed-batch records by chunk ID (keeping the highest attempt count). +- **Interrupted recovery safety**: Recovery of an interrupted global clear or clearing-phase force-index now replays the clear against the originating project scope persisted with the lease, so a project reclaiming a dead lease can no longer delete another project's indexed data. Retry runs restore SQLite chunk rows that a recovery health check may have collected as orphans, so a branch reference can never point at a chunk without metadata. - **Forced re-embed migration safety**: While a project-scoped forced re-embed is pending, checkpoints no longer stamp new embedding metadata or a compatibility certificate, unchanged scope files are re-embedded instead of skipped, and `retryFailedBatches` only clears the pending migration after the main run has committed its new metadata. This prevents an interrupted migration from resuming into a silent mix of old and new vector spaces. - **Conceptual context retrieval**: Prefer implementation paths over tests and documentation for code-focused `codebase_context` searches, without applying a hard source-only filter or changing documentation and test queries. - **Scoped definition lookup**: Allow explicit definition searches constrained to a file type or directory to return matching declarations in fixtures and test paths without weakening source-first ranking for ordinary searches. diff --git a/src/indexer/index-lock.ts b/src/indexer/index-lock.ts index 8363035..c52735d 100644 --- a/src/indexer/index-lock.ts +++ b/src/indexer/index-lock.ts @@ -28,6 +28,10 @@ export interface IndexLockOwner { startedAt: string; operation: IndexMutationOperation; token: string; + /** Originating project root for interrupted global clears (recovery scope). */ + projectRoot?: string; + /** Originating scoped roots for interrupted global clears (recovery scope). */ + scopedRoots?: string[]; } export interface IndexLockRecovery { @@ -100,6 +104,12 @@ function parseOwner(value: unknown): IndexLockOwner | null { if (typeof candidate.startedAt !== "string" || Number.isNaN(Date.parse(candidate.startedAt))) return null; if (typeof candidate.operation !== "string" || !VALID_OPERATIONS.has(candidate.operation as IndexMutationOperation)) return null; if (typeof candidate.token !== "string" || !UUID_PATTERN.test(candidate.token)) return null; + if (candidate.projectRoot !== undefined && typeof candidate.projectRoot !== "string") return null; + if (candidate.scopedRoots !== undefined) { + if (!Array.isArray(candidate.scopedRoots) || candidate.scopedRoots.some((root) => typeof root !== "string")) { + return null; + } + } return candidate as IndexLockOwner; } @@ -224,6 +234,11 @@ function createOwner(operation: IndexMutationOperation): IndexLockOwner { }; } +export interface IndexLockRecoveryScope { + projectRoot: string; + scopedRoots: string[]; +} + function recoveryMarkerPath(indexPath: string, owner: IndexLockOwner): string { return path.join(indexPath, `${RECOVERY_MARKER_PREFIX}${owner.token}`); } @@ -395,14 +410,24 @@ export function isTransientIndexLockContention(error: unknown): boolean { return error.reason === "active" || error.reason === "reclaiming"; } -export function acquireIndexLock(indexPath: string, operation: IndexMutationOperation): IndexLockLease { +export function acquireIndexLock( + indexPath: string, + operation: IndexMutationOperation, + recoveryScope?: IndexLockRecoveryScope, +): IndexLockLease { mkdirSync(indexPath, { recursive: true }); const canonicalIndexPath = realpathSync.native(indexPath); const lockPath = path.join(canonicalIndexPath, "indexing.lock"); cleanupDeadPublicationCandidates(canonicalIndexPath); for (let attempt = 0; attempt < 6; attempt += 1) { - const owner = createOwner(operation); + const owner = recoveryScope === undefined + ? createOwner(operation) + : { + ...createOwner(operation), + projectRoot: recoveryScope.projectRoot, + scopedRoots: recoveryScope.scopedRoots, + }; if (publishJsonDirectory(lockPath, owner)) { const lease: IndexLockLease = { canonicalIndexPath, @@ -482,9 +507,9 @@ export async function withIndexLock( indexPath: string, operation: IndexMutationOperation, callback: (lease: IndexLockLease) => Promise | T, - options: { completeRecoveries?: boolean } = {}, + options: { completeRecoveries?: boolean; recoveryScope?: IndexLockRecoveryScope } = {}, ): Promise { - const lease = acquireIndexLock(indexPath, operation); + const lease = acquireIndexLock(indexPath, operation, options.recoveryScope); let result: T | undefined; let callbackError: unknown; let callbackFailed = false; diff --git a/src/indexer/index.ts b/src/indexer/index.ts index a7990da..f67cc05 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -1185,7 +1185,7 @@ export class Indexer { runtimeOptions: IndexerRuntimeOptions = {}, ) { this.projectRoot = projectRoot; - this.projectIdentityHash = hashContent(this.getCanonicalPath(projectRoot)).slice(0, 16); + this.projectIdentityHash = this.getProjectIdentityHash(projectRoot); this.materializedProjectRoot = runtimeOptions.materializedProjectRoot ?? projectRoot; this.branchNameOverride = runtimeOptions.branchName; this.catalogIdentityOverride = runtimeOptions.catalogIdentity; @@ -1333,6 +1333,10 @@ export class Indexer { } } + private getProjectIdentityHash(projectRoot: string): string { + return hashContent(this.getCanonicalPath(projectRoot)).slice(0, 16); + } + private isProjectOwnedIndexPath(): boolean { return isProjectIndexPathOwnedByProject(this.projectRoot, this.indexPath, this.host); } @@ -1375,7 +1379,10 @@ export class Indexer { callback: (recoveredOwners: readonly IndexLockOwner[]) => Promise, ): Promise { this.refreshBranchInfo(); - const lease = acquireIndexLock(this.indexPath, operation); + const lease = acquireIndexLock(this.indexPath, operation, { + projectRoot: this.projectRoot, + scopedRoots: this.getScopedRoots(), + }); this.indexPath = lease.canonicalIndexPath; this.refreshRuntimeArtifactPaths(); this.activeIndexLease = lease; @@ -1478,11 +1485,11 @@ export class Indexer { ); } - private getScopedRoots(): string[] { - const roots = new Set([this.getCanonicalPath(this.projectRoot)]); + private getScopedRoots(projectRoot = this.projectRoot): string[] { + const roots = new Set([this.getCanonicalPath(projectRoot)]); for (const kbRoot of this.config.knowledgeBases) { - roots.add(this.getCanonicalPath(path.resolve(this.projectRoot, kbRoot))); + roots.add(this.getCanonicalPath(path.resolve(projectRoot, kbRoot))); } return Array.from(roots); @@ -1584,20 +1591,20 @@ export class Indexer { return this.currentBranch || "default"; } - private getLegacyMigrationMetadataKey(): string { - return `index.globalBranchMigration.${this.projectIdentityHash}`; + private getLegacyMigrationMetadataKey(projectIdentityHash = this.projectIdentityHash): string { + return `index.globalBranchMigration.${projectIdentityHash}`; } - private getProjectEmbeddingStrategyMetadataKey(): string { - return `index.embeddingStrategyVersion.${this.projectIdentityHash}`; + private getProjectEmbeddingStrategyMetadataKey(projectIdentityHash = this.projectIdentityHash): string { + return `index.embeddingStrategyVersion.${projectIdentityHash}`; } - private getProjectForceReembedMetadataKey(): string { - return `index.forceReembed.${this.projectIdentityHash}`; + private getProjectForceReembedMetadataKey(projectIdentityHash = this.projectIdentityHash): string { + return `index.forceReembed.${projectIdentityHash}`; } - private getProjectMigrationFinalizedMetadataKey(): string { - return `index.migrationFinalized.${this.projectIdentityHash}`; + private getProjectMigrationFinalizedMetadataKey(projectIdentityHash = this.projectIdentityHash): string { + return `index.migrationFinalized.${projectIdentityHash}`; } private getBranchMigrationMetadataKey( @@ -1754,7 +1761,7 @@ export class Indexer { return primary === legacy ? [primary] : [primary, legacy]; } - private getProjectLocalScopedOwnershipIds(roots: string[]): { + private getProjectLocalScopedOwnershipIds(roots: string[], projectRoot = this.projectRoot): { chunkIds: Set; symbolIds: Set; } { @@ -1766,12 +1773,12 @@ export class Indexer { const projectLocalFilePaths = new Set([ ...Array.from(this.fileHashCache.keys()).filter( - (filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath) + (filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot) ), ...(this.store?.getAllMetadata() ?? []) .map(({ metadata }) => metadata.filePath) .filter( - (filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath) + (filePath) => this.isFileInCurrentScope(filePath, roots) && this.isFileInProjectRoot(filePath, projectRoot) ), ]); @@ -1788,7 +1795,11 @@ export class Indexer { return { chunkIds, symbolIds }; } - private getProjectScopedBranchCatalogCleanupKeys(projectChunkIds: string[], projectSymbolIds: string[]): string[] { + private getProjectScopedBranchCatalogCleanupKeys( + projectChunkIds: string[], + projectSymbolIds: string[], + projectRoot = this.projectRoot, + ): string[] { if (this.config.scope !== "global") { return this.getBranchCatalogCleanupKeys(); } @@ -1796,9 +1807,10 @@ export class Indexer { const keys = new Set(); const projectChunkIdSet = new Set(projectChunkIds); const projectSymbolIdSet = new Set(projectSymbolIds); + const projectIdentityHash = this.getProjectIdentityHash(projectRoot); for (const branchKey of this.database?.getAllBranches() ?? []) { - if (branchKey.startsWith(`${this.projectIdentityHash}:`)) { + if (branchKey.startsWith(`${projectIdentityHash}:`)) { keys.add(branchKey); continue; } @@ -1810,8 +1822,10 @@ export class Indexer { } } - for (const branchKey of this.getBranchCatalogCleanupKeys()) { - keys.add(branchKey); + if (projectRoot === this.projectRoot) { + for (const branchKey of this.getBranchCatalogCleanupKeys()) { + keys.add(branchKey); + } } return Array.from(keys); @@ -1822,10 +1836,10 @@ export class Indexer { return roots.some((root) => isPathWithinRoot(canonicalFilePath, root)); } - private isFileInProjectRoot(filePath: string): boolean { + private isFileInProjectRoot(filePath: string, projectRoot = this.projectRoot): boolean { return isPathWithinRoot( this.getCanonicalStoredFilePath(filePath), - this.getCanonicalPath(this.projectRoot), + this.getCanonicalPath(projectRoot), ); } @@ -1875,13 +1889,14 @@ export class Indexer { return false; } - private hasForeignScopedBranchData(): boolean { + private hasForeignScopedBranchData(projectRoot = this.projectRoot): boolean { if (!this.database || this.config.scope !== "global") { return false; } - const roots = this.getScopedRoots(); - const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots); + const roots = this.getScopedRoots(projectRoot); + const projectIdentityHash = this.getProjectIdentityHash(projectRoot); + const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots, projectRoot); return this.database.getAllBranches().some( (branchKey) => { @@ -1892,7 +1907,7 @@ export class Indexer { return false; } - if (branchKey.startsWith(`${this.projectIdentityHash}:`)) { + if (branchKey.startsWith(`${projectIdentityHash}:`)) { return false; } @@ -1907,7 +1922,8 @@ export class Indexer { store: VectorStore, invertedIndex: InvertedIndex, database: Database, - roots: string[] + roots: string[], + projectRoot = this.projectRoot, ): { removedChunkIds: string[]; hasForeignData: boolean } { const allMetadata = store.getAllMetadata(); const scopedEntries = allMetadata.filter(({ metadata }) => this.isFileInCurrentScope(metadata.filePath, roots)); @@ -1917,7 +1933,7 @@ export class Indexer { ]); const projectLocalFilePaths = new Set( - Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath)) + Array.from(filePaths).filter((filePath) => this.isFileInProjectRoot(filePath, projectRoot)) ); const removedChunkIds = new Set(scopedEntries.map(({ key }) => key)); @@ -1930,7 +1946,7 @@ export class Indexer { const projectLocalChunkIds = new Set( scopedEntries - .filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath)) + .filter(({ metadata }) => this.isFileInProjectRoot(metadata.filePath, projectRoot)) .map(({ key }) => key) ); for (const filePath of projectLocalFilePaths) { @@ -1953,6 +1969,7 @@ export class Indexer { const branchCleanupKeys = this.getProjectScopedBranchCatalogCleanupKeys( Array.from(projectLocalChunkIds), Array.from(projectLocalSymbolIds), + projectRoot, ); for (const branchKey of branchCleanupKeys) { database.deleteBranchChunksForBranch(branchKey, removedChunkIdList); @@ -2035,22 +2052,34 @@ export class Indexer { hostname: owner.hostname, operation: owner.operation, startedAt: owner.startedAt, + projectRoot: owner.projectRoot, }); } if (this.config.scope === "global") { - const shouldCompleteClear = owners.some( + const clearOwners = owners.filter( (owner) => owner.operation === "clear" || (owner.operation === "force-index" && this.hasForceIndexPhaseMarker()), ); - if (shouldCompleteClear) { - // Re-apply the clear decision: a global clear only wipes the whole - // shared index when no foreign project data is present, otherwise it - // stays scoped to the current project. - this.clearGlobalIndexUnlocked(); + if (clearOwners.length > 0) { + // The scoped clear purges the file-hash cache entries of the + // originating project: load the persisted cache first so the purge + // is written back instead of being lost on an empty in-memory map. + this.loadFileHashCache(); + } + for (const owner of clearOwners) { + // Re-apply the clear decision against the originating project scope: + // a global clear only wipes the whole shared index when no foreign + // project data is present, otherwise it stays scoped to the project + // that started the clear. Owners written before the recovery scope + // was persisted fall back to the current project. + this.clearGlobalIndexUnlocked( + owner.projectRoot ?? this.projectRoot, + owner.scopedRoots ?? this.getScopedRoots(), + ); } await this.healthCheckUnlocked(); this.logger.info( - shouldCompleteClear + clearOwners.length > 0 ? "Recovery complete, next index will rebuild all files" : "Recovery complete, next index will resume from the last checkpoint", ); @@ -2338,6 +2367,33 @@ export class Indexer { } } + private restoreMissingChunkRows(database: Database, chunks: readonly PendingChunk[]): void { + const missing: ChunkData[] = []; + for (const chunk of chunks) { + if (database.getChunk(chunk.id)) { + continue; + } + missing.push({ + chunkId: chunk.id, + contentHash: chunk.contentHash, + filePath: chunk.metadata.filePath, + startLine: chunk.metadata.startLine, + endLine: chunk.metadata.endLine, + nodeType: chunk.metadata.chunkType, + name: chunk.metadata.name, + language: chunk.metadata.language, + blameSha: chunk.metadata.blameSha, + blameAuthor: chunk.metadata.blameAuthor, + blameAuthorEmail: chunk.metadata.blameAuthorEmail, + blameCommittedAt: chunk.metadata.blameCommittedAt, + blameSummary: chunk.metadata.blameSummary, + }); + } + if (missing.length > 0) { + database.upsertChunksBatch(missing); + } + } + private getProviderRateLimits(provider: string): { concurrency: number; intervalMs: number; @@ -4486,6 +4542,12 @@ export class Indexer { retryableChunksWithExistingData.add(chunk.id); } } + // A failed chunk checkpointed before its embedding attempt has a + // committed SQLite row only when the parsing phase upserted it; a + // crash before that checkpoint can leave the row missing while the + // failed-batches record survives. Restore the row so the retry does + // not leave a branch reference without chunk metadata. + this.restoreMissingChunkRows(database, pendingChunks); stats.totalChunks += pendingChunks.length; onProgress?.({ phase: "embedding", @@ -5434,7 +5496,7 @@ export class Indexer { }); } - private clearGlobalIndexDataUnlocked(): void { + private clearGlobalIndexDataUnlocked(projectRoot = this.projectRoot): void { const { store, invertedIndex, database } = this.requireLoadedIndexState(); const clearedBranchKeys = database.getAllBranches(); store.clear(); @@ -5455,54 +5517,55 @@ export class Indexer { database.deleteMetadata("index.embeddingModel"); database.deleteMetadata("index.embeddingDimensions"); database.deleteMetadata("index.embeddingStrategyVersion"); - database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey()); - database.deleteMetadata(this.getProjectForceReembedMetadataKey()); - database.deleteMetadata(this.getLegacyMigrationMetadataKey()); + const projectIdentityHash = this.getProjectIdentityHash(projectRoot); + database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash)); + database.deleteMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash)); + database.deleteMetadata(this.getLegacyMigrationMetadataKey(projectIdentityHash)); database.deleteMetadata("index.createdAt"); database.deleteMetadata("index.updatedAt"); this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo!); } - private clearGlobalIndexUnlocked(): void { + private clearGlobalIndexUnlocked(projectRoot = this.projectRoot, roots = this.getScopedRoots()): void { const { store, invertedIndex, database } = this.requireLoadedIndexState(); store.load(); invertedIndex.load(); this.loadFileHashCache(); - const roots = this.getScopedRoots(); const compatibility = this.checkCompatibility(); const allMetadata = store.getAllMetadata(); const hasForeignData = allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || - this.hasForeignScopedBranchData() || + this.hasForeignScopedBranchData(projectRoot) || this.hasForeignScopedFileHashData(roots) || this.hasForeignScopedFailedBatches(roots); if (!compatibility.compatible && hasForeignData) { if (compatibility.code === IncompatibilityCode.EMBEDDING_STRATEGY_MISMATCH) { - this.clearSharedIndexProjectData(store, invertedIndex, database, roots); + this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot); this.clearScopedFileHashCache(roots); this.clearScopedFailedBatches(roots); - database.setMetadata(this.getProjectForceReembedMetadataKey(), "true"); - database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey()); - database.deleteMetadata(this.getProjectMigrationFinalizedMetadataKey()); + const projectIdentityHash = this.getProjectIdentityHash(projectRoot); + database.setMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash), "true"); + database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash)); + database.deleteMetadata(this.getProjectMigrationFinalizedMetadataKey(projectIdentityHash)); this.indexCompatibility = { compatible: true }; return; } throw new Error( `Global index compatibility reset is unsafe because the shared index contains files from other projects. ` + - `The current global index cannot be force-rebuilt for ${this.projectRoot} without deleting other repositories' indexed data. ` + + `The current global index cannot be force-rebuilt for ${projectRoot} without deleting other repositories' indexed data. ` + `Use scope="project" for isolated rebuilds, or manually delete the shared global index if you intend to rebuild all projects.` ); } if (!hasForeignData) { - this.clearGlobalIndexDataUnlocked(); + this.clearGlobalIndexDataUnlocked(projectRoot); return; } - this.clearSharedIndexProjectData(store, invertedIndex, database, roots); + this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot); this.clearScopedFileHashCache(roots); this.clearScopedFailedBatches(roots); this.indexCompatibility = compatibility; @@ -5710,6 +5773,10 @@ export class Indexer { )) { const chunks = retryBatch.map(({ chunk }) => chunk); const attemptCounts = new Map(retryBatch.map(({ chunk, attemptCount }) => [chunk.id, attemptCount])); + // Restore chunk rows that a recovery health check may have collected + // as orphans: a failed chunk checkpointed before its embedding has a + // committed SQLite row but no branch association until it succeeds. + this.restoreMissingChunkRows(database, chunks); const batchResult = await this.processPendingChunkBatch(chunks, { store, provider, diff --git a/tests/indexer-checkpoint-resume.test.ts b/tests/indexer-checkpoint-resume.test.ts index ca15bd7..c82ea7a 100644 --- a/tests/indexer-checkpoint-resume.test.ts +++ b/tests/indexer-checkpoint-resume.test.ts @@ -850,4 +850,177 @@ describe("indexer checkpoint resume", () => { const alphaOneOccurrences = (failedContent.match(/"name":"alphaOne"/g) ?? []).length; expect(alphaOneOccurrences).toBe(1); }); + + it("global lease recovery replays an interrupted clear against the originating project scope", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + // Simulate a crashed clear started by project A: the lease records the + // originating project scope so the recovery replays the clear against A + // instead of the project that reclaims the lease. + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "clear", + token: "11111111-1111-4111-8111-111111111111", + projectRoot: projectDir, + scopedRoots: [canonicalPath(projectDir), canonicalPath(kbDir)], + })); + + // Project B reclaims the lease. The recovery must clear A's data (and the + // shared knowledge base) while preserving B's checkpointed data. B's run + // only scans B and the knowledge base, so A's removal is observable in + // the shared database and hash cache, not in B's embedding calls. + const beforeCalls = fetchSpy.mock.calls.length; + const stats = await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + expect(stats.failedChunks).toBe(0); + const resumeInputs = fetchSpy.mock.calls.slice(beforeCalls).flatMap((call) => { + const body = JSON.parse(String(call[1]?.body ?? "{}")) as { input?: string[] }; + return body.input ?? []; + }); + // B's checkpointed data is preserved: B's files are not re-embedded, and + // the shared knowledge base (cleared with A's scope) is re-cataloged by + // B's run (reusing the cached embeddings, so no new embedding calls). + expect(resumeInputs.some((text) => text.includes("betaOne"))).toBe(false); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + expect(db.getChunksByName("alphaOne").length).toBe(0); + expect(db.getChunksByName("betaOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("sharedOne").length).toBeGreaterThan(0); + } finally { + db.close(); + } + const fileHashes = JSON.parse(fs.readFileSync(path.join(indexDir, "file-hashes.json"), "utf-8")) as Record; + expect(Object.keys(fileHashes).some((filePath) => filePath.includes("alpha.ts"))).toBe(false); + expect(Object.keys(fileHashes).some((filePath) => filePath.includes("beta.ts"))).toBe(true); + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("cross-project clear recovery preserves the reclaiming project branch catalog", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + // Simulate a crashed clear started by project A, reclaimed by project B + // through a retry run: the recovery must not remove the shared knowledge + // base entries from B's branch catalog, because B never re-scans them. + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "clear", + token: "11111111-1111-4111-8111-111111111111", + projectRoot: projectDir, + scopedRoots: [canonicalPath(projectDir), canonicalPath(kbDir)], + })); + + // B reclaims the lease through retryFailedBatches, which runs the + // recovery but never re-scans the knowledge base. + const retry = await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).retryFailedBatches(); + expect(retry.remaining).toBe(0); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + const branchKey = `${projectIdentityHash(projectBDir)}:default`; + const branchChunkIds = db.getBranchChunkIds(branchKey); + expect(branchChunkIds.length).toBeGreaterThan(0); + expect(db.getChunksByName("sharedOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("betaOne").length).toBeGreaterThan(0); + } finally { + db.close(); + } + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("recovery health check does not orphan checkpointed failed chunk rows", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + // Every project A file contains a failing chunk so whichever files are + // checkpointed before the interruption carry a persisted failure record + // with a committed SQLite chunk row. + const sourceFiles = [ + path.join(projectDir, "src", "alpha.ts"), + path.join(projectDir, "src", "beta.ts"), + path.join(projectDir, "src", "gamma.ts"), + ]; + for (const [fileIndex, filePath] of sourceFiles.entries()) { + writeSourceFile( + filePath, + ["first", "second", "triggerFailure", "fourth", "fifth", "sixth"].map((name) => `${name}${fileIndex}`), + ); + } + + failEmbeddingText = "triggerFailure"; + await expect(createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index((progress) => { + if (progress.phase === "embedding" && progress.filesProcessed === progress.totalFiles) { + throw new Error("simulated interruption after failed batch checkpoint"); + } + })).rejects.toThrow("simulated interruption after failed batch checkpoint"); + failEmbeddingText = null; + + // Simulate a crashed indexing session: the recovery health check runs + // gcOrphanChunks() and deletes the committed rows of failed chunks that + // have no branch association yet. + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "index", + token: "11111111-1111-4111-8111-111111111111", + })); + + // The dedicated retry must restore the missing chunk rows before + // re-embedding, so the branch catalog never references a chunk without + // metadata. + const retry = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).retryFailedBatches(); + expect(retry.remaining).toBe(0); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + const branchKey = `${projectIdentityHash(projectDir)}:default`; + const branchChunkIds = db.getBranchChunkIds(branchKey); + expect(branchChunkIds.length).toBeGreaterThan(0); + for (const chunkId of branchChunkIds) { + expect(db.getChunk(chunkId)).not.toBeNull(); + } + for (const filePath of ["src/alpha.ts", "src/beta.ts", "src/gamma.ts"]) { + expect(db.getChunksByFile(path.join(projectDir, filePath)).length).toBeGreaterThan(0); + } + } finally { + db.close(); + } + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); }); From 7a86c972f2908794884dc1dfbbef967863f76829 Mon Sep 17 00:00:00 2001 From: Nicolas COMPAIN Date: Mon, 17 Aug 2026 09:59:38 +0200 Subject: [PATCH 4/5] fix(indexer): bind interrupted clears to their recovery lease Persist the destructive clear phase, effective embedding configuration, and compatibility decision on the active lease owner. Recovery now rejects legacy or configuration-mismatched global clears before mutation, replays knowledge-base scopes and compatibility decisions from the originating lease, and ignores stale phase files for token-bound owners. Add regressions for legacy clear leases, cross-project configuration and compatibility recovery, stale force-index markers, and ambiguous legacy force-index phases. --- CHANGELOG.md | 2 +- src/indexer/index-lock.ts | 72 +++++++ src/indexer/index.ts | 173 +++++++++++++---- tests/indexer-checkpoint-resume.test.ts | 244 ++++++++++++++++++++++++ 4 files changed, 449 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 064f492..20f7099 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - **Interrupted indexing resume**: Interrupted indexing runs now persist incremental checkpoints (database, vectors, BM25, failed batches, and file hashes), so a subsequent run resumes incrementally instead of re-embedding the whole project. Interrupted global clears and force-indexes are completed on recovery (scoped to the current project when foreign data is present), and checkpointed failed batches and pending retries survive for later retry runs. Checkpoint persistence orders the BM25 index before the vector store so a crash between the two cannot orphan chunks from keyword search. Project-scope dead-lease recovery preserves checkpointed artifacts for incremental resume instead of resetting the local index. Pending retry chunks are no longer duplicated across multiple checkpoints, and finalization deduplicates failed-batch records by chunk ID (keeping the highest attempt count). -- **Interrupted recovery safety**: Recovery of an interrupted global clear or clearing-phase force-index now replays the clear against the originating project scope persisted with the lease, so a project reclaiming a dead lease can no longer delete another project's indexed data. Retry runs restore SQLite chunk rows that a recovery health check may have collected as orphans, so a branch reference can never point at a chunk without metadata. +- **Interrupted recovery safety**: Recovery of an interrupted global clear or clearing-phase force-index now replays the clear against the originating project scope persisted with the lease, so a project reclaiming a dead lease can no longer delete another project's indexed data. The destructive phase and effective embedding configuration are bound to the lease owner; legacy or configuration-mismatched clears fail closed and retain their recovery marker instead of applying the reclaiming project's settings. Retry runs restore SQLite chunk rows that a recovery health check may have collected as orphans, so a branch reference can never point at a chunk without metadata. - **Forced re-embed migration safety**: While a project-scoped forced re-embed is pending, checkpoints no longer stamp new embedding metadata or a compatibility certificate, unchanged scope files are re-embedded instead of skipped, and `retryFailedBatches` only clears the pending migration after the main run has committed its new metadata. This prevents an interrupted migration from resuming into a silent mix of old and new vector spaces. - **Conceptual context retrieval**: Prefer implementation paths over tests and documentation for code-focused `codebase_context` searches, without applying a hard source-only filter or changing documentation and test queries. - **Scoped definition lookup**: Allow explicit definition searches constrained to a file type or directory to return matching declarations in fixtures and test paths without weakening source-first ranking for ordinary searches. diff --git a/src/indexer/index-lock.ts b/src/indexer/index-lock.ts index c52735d..e5e27ae 100644 --- a/src/indexer/index-lock.ts +++ b/src/indexer/index-lock.ts @@ -28,10 +28,23 @@ export interface IndexLockOwner { startedAt: string; operation: IndexMutationOperation; token: string; + /** Recovery protocol written by owners that persist destructive phase state. */ + recoveryProtocolVersion?: 1; /** Originating project root for interrupted global clears (recovery scope). */ projectRoot?: string; /** Originating scoped roots for interrupted global clears (recovery scope). */ scopedRoots?: string[]; + /** Destructive clear state, present only while a clear is in progress. */ + clearRecovery?: IndexLockClearRecoveryState; +} + +export interface IndexLockClearRecoveryState { + phase: "clearing"; + embeddingProvider: string; + embeddingModel: string; + embeddingDimensions: number; + embeddingStrategyVersion: string; + compatibilityDecision: "compatible" | "embedding-strategy-mismatch" | "incompatible"; } export interface IndexLockRecovery { @@ -104,12 +117,37 @@ function parseOwner(value: unknown): IndexLockOwner | null { if (typeof candidate.startedAt !== "string" || Number.isNaN(Date.parse(candidate.startedAt))) return null; if (typeof candidate.operation !== "string" || !VALID_OPERATIONS.has(candidate.operation as IndexMutationOperation)) return null; if (typeof candidate.token !== "string" || !UUID_PATTERN.test(candidate.token)) return null; + if (candidate.recoveryProtocolVersion !== undefined && candidate.recoveryProtocolVersion !== 1) return null; if (candidate.projectRoot !== undefined && typeof candidate.projectRoot !== "string") return null; if (candidate.scopedRoots !== undefined) { if (!Array.isArray(candidate.scopedRoots) || candidate.scopedRoots.some((root) => typeof root !== "string")) { return null; } } + if (candidate.clearRecovery !== undefined) { + const recovery = candidate.clearRecovery as Partial; + if ( + typeof recovery !== "object" + || recovery === null + || recovery.phase !== "clearing" + || typeof recovery.embeddingProvider !== "string" + || recovery.embeddingProvider.length === 0 + || typeof recovery.embeddingModel !== "string" + || recovery.embeddingModel.length === 0 + || !Number.isInteger(recovery.embeddingDimensions) + || (recovery.embeddingDimensions ?? 0) <= 0 + || typeof recovery.embeddingStrategyVersion !== "string" + || recovery.embeddingStrategyVersion.length === 0 + || ( + recovery.compatibilityDecision !== "compatible" + && recovery.compatibilityDecision !== "embedding-strategy-mismatch" + && recovery.compatibilityDecision !== "incompatible" + ) + || (candidate.operation !== "clear" && candidate.operation !== "force-index") + ) { + return null; + } + } return candidate as IndexLockOwner; } @@ -425,6 +463,7 @@ export function acquireIndexLock( ? createOwner(operation) : { ...createOwner(operation), + recoveryProtocolVersion: 1 as const, projectRoot: recoveryScope.projectRoot, scopedRoots: recoveryScope.scopedRoots, }; @@ -503,6 +542,39 @@ export function releaseIndexLock(lease: IndexLockLease): boolean { return true; } +export function setIndexLockClearRecoveryState( + lease: IndexLockLease, + clearRecovery: IndexLockClearRecoveryState | null, +): void { + const currentOwner = readDirectoryOwner(lease.lockPath); + if (!currentOwner || !sameOwner(currentOwner, lease.owner)) { + throw new Error(`Lost ownership of index mutation lease ${lease.owner.token}`); + } + + const nextOwner: IndexLockOwner = { ...currentOwner }; + if (clearRecovery === null) { + delete nextOwner.clearRecovery; + } else { + nextOwner.clearRecovery = clearRecovery; + } + + const ownerPath = path.join(lease.lockPath, OWNER_FILE_NAME); + const temporaryPath = path.join( + lease.lockPath, + `${OWNER_FILE_NAME}.tmp.${lease.owner.pid}.${lease.owner.token}.${randomUUID()}`, + ); + try { + writeFileSync(temporaryPath, JSON.stringify(nextOwner), { + encoding: "utf-8", + flag: "wx", + mode: 0o600, + }); + retryTransientFilesystemOperation(() => renameSync(temporaryPath, ownerPath)); + } finally { + if (existsSync(temporaryPath)) rmSync(temporaryPath, { force: true }); + } +} + export async function withIndexLock( indexPath: string, operation: IndexMutationOperation, diff --git a/src/indexer/index.ts b/src/indexer/index.ts index f67cc05..bdd67ad 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -84,6 +84,8 @@ import { recoverLeaseArtifacts, releaseIndexLock, removeLeaseTemporaryPath, + setIndexLockClearRecoveryState, + type IndexLockClearRecoveryState, type IndexLockLease, type IndexLockOwner, type IndexMutationOperation, @@ -1889,12 +1891,14 @@ export class Indexer { return false; } - private hasForeignScopedBranchData(projectRoot = this.projectRoot): boolean { + private hasForeignScopedBranchData( + projectRoot = this.projectRoot, + roots = this.getScopedRoots(projectRoot), + ): boolean { if (!this.database || this.config.scope !== "global") { return false; } - const roots = this.getScopedRoots(projectRoot); const projectIdentityHash = this.getProjectIdentityHash(projectRoot); const { chunkIds: projectLocalChunkIds, symbolIds: projectLocalSymbolIds } = this.getProjectLocalScopedOwnershipIds(roots, projectRoot); @@ -2025,24 +2029,50 @@ export class Indexer { }; } - private getForceIndexPhaseMarkerPath(): string { - return path.join(this.indexPath, "force-index-phase"); + private getCurrentClearRecoveryState(): IndexLockClearRecoveryState { + if (!this.configuredProviderInfo) { + throw new Error("Cannot persist clear recovery state before the embedding provider is initialized"); + } + const compatibility = this.checkCompatibility(); + const compatibilityDecision = compatibility.compatible + ? "compatible" + : compatibility.code === IncompatibilityCode.EMBEDDING_STRATEGY_MISMATCH + ? "embedding-strategy-mismatch" + : "incompatible"; + return { + phase: "clearing", + embeddingProvider: this.configuredProviderInfo.provider, + embeddingModel: this.configuredProviderInfo.modelInfo.model, + embeddingDimensions: this.configuredProviderInfo.modelInfo.dimensions, + embeddingStrategyVersion: EMBEDDING_STRATEGY_VERSION, + compatibilityDecision, + }; } - private writeForceIndexPhaseMarker(): void { - writeFileSync(this.getForceIndexPhaseMarkerPath(), "clearing", { encoding: "utf-8" }); + private beginClearRecoveryState(): IndexLockClearRecoveryState { + const recovery = this.getCurrentClearRecoveryState(); + setIndexLockClearRecoveryState(this.requireActiveLease(), recovery); + return recovery; } - private removeForceIndexPhaseMarker(): void { - try { - unlinkSync(this.getForceIndexPhaseMarkerPath()); - } catch { - // Best-effort cleanup. - } + private finishClearRecoveryState(): void { + setIndexLockClearRecoveryState(this.requireActiveLease(), null); } - private hasForceIndexPhaseMarker(): boolean { - return existsSync(this.getForceIndexPhaseMarkerPath()); + private matchesCurrentClearRecoveryConfiguration(recovery: IndexLockClearRecoveryState): boolean { + const configuredProviderInfo = this.configuredProviderInfo; + return configuredProviderInfo !== null + && recovery.embeddingProvider === configuredProviderInfo.provider + && recovery.embeddingModel === configuredProviderInfo.modelInfo.model + && recovery.embeddingDimensions === configuredProviderInfo.modelInfo.dimensions + && recovery.embeddingStrategyVersion === EMBEDDING_STRATEGY_VERSION; + } + + private hasUnknownLegacyForceIndexClear(owner: IndexLockOwner): boolean { + return owner.operation === "force-index" + && owner.clearRecovery === undefined + && owner.recoveryProtocolVersion !== 1 + && existsSync(path.join(this.indexPath, "force-index-phase")); } private async recoverFromInterruptedIndexingUnlocked(owners: readonly IndexLockOwner[]): Promise { @@ -2057,29 +2087,64 @@ export class Indexer { } if (this.config.scope === "global") { - const clearOwners = owners.filter( - (owner) => owner.operation === "clear" || (owner.operation === "force-index" && this.hasForceIndexPhaseMarker()), - ); - if (clearOwners.length > 0) { + const clearScopes: Array<{ + projectRoot: string; + scopedRoots: string[]; + compatibilityDecision: IndexLockClearRecoveryState["compatibilityDecision"]; + }> = []; + for (const owner of owners) { + if (this.hasUnknownLegacyForceIndexClear(owner)) { + throw new Error( + `Cannot automatically recover interrupted force-index ${owner.token}: ` + + "the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection." + ); + } + if ( + owner.operation === "clear" + && owner.clearRecovery === undefined + && owner.recoveryProtocolVersion !== 1 + ) { + throw new Error( + `Cannot automatically recover interrupted global clear ${owner.token}: ` + + "the originating recovery state is unknown. The recovery marker was retained for manual inspection." + ); + } + if (owner.clearRecovery === undefined) continue; + if (!owner.projectRoot || !owner.scopedRoots || owner.scopedRoots.length === 0) { + throw new Error( + `Cannot automatically recover interrupted global clear ${owner.token}: ` + + "the originating project scope is unknown. The recovery marker was retained for manual inspection." + ); + } + if (!this.matchesCurrentClearRecoveryConfiguration(owner.clearRecovery)) { + throw new Error( + `Cannot automatically recover interrupted global clear ${owner.token}: ` + + "the current embedding configuration does not match the originating lease. " + + "The recovery marker was retained; retry from the originating project with matching settings." + ); + } + clearScopes.push({ + projectRoot: owner.projectRoot, + scopedRoots: owner.scopedRoots, + compatibilityDecision: owner.clearRecovery.compatibilityDecision, + }); + } + if (clearScopes.length > 0) { // The scoped clear purges the file-hash cache entries of the // originating project: load the persisted cache first so the purge // is written back instead of being lost on an empty in-memory map. this.loadFileHashCache(); } - for (const owner of clearOwners) { + for (const { projectRoot, scopedRoots, compatibilityDecision } of clearScopes) { // Re-apply the clear decision against the originating project scope: // a global clear only wipes the whole shared index when no foreign // project data is present, otherwise it stays scoped to the project - // that started the clear. Owners written before the recovery scope - // was persisted fall back to the current project. - this.clearGlobalIndexUnlocked( - owner.projectRoot ?? this.projectRoot, - owner.scopedRoots ?? this.getScopedRoots(), - ); + // that started the clear. + this.clearGlobalIndexUnlocked(projectRoot, scopedRoots, compatibilityDecision); } await this.healthCheckUnlocked(); this.logger.info( - clearOwners.length > 0 + clearScopes.length > 0 ? "Recovery complete, next index will rebuild all files" : "Recovery complete, next index will resume from the last checkpoint", ); @@ -3247,8 +3312,18 @@ export class Indexer { ]); } if (recoveredOwners.length > 0 && this.config.scope === "project") { + const unknownLegacyForceIndex = recoveredOwners.find( + (owner) => this.hasUnknownLegacyForceIndexClear(owner), + ); + if (unknownLegacyForceIndex) { + throw new Error( + `Cannot automatically recover interrupted force-index ${unknownLegacyForceIndex.token}: ` + + "the legacy clearing phase ownership is unknown. The recovery marker was retained for manual inspection." + ); + } const shouldReset = recoveredOwners.some( - (owner) => owner.operation === "clear" || (owner.operation === "force-index" && this.hasForceIndexPhaseMarker()), + (owner) => owner.clearRecovery !== undefined + || (owner.operation === "clear" && owner.recoveryProtocolVersion !== 1), ); if (shouldReset) { await this.resetLocalIndexArtifacts(); @@ -5480,11 +5555,9 @@ export class Indexer { async forceIndex(onProgress?: ProgressCallback): Promise { return this.withIndexMutationLease("force-index", async (recoveredOwners) => { await this.ensureInitializedUnlocked(recoveredOwners); - // Mark the clearing phase so recovery can distinguish a crash before - // the clear from a crash during indexing. - this.writeForceIndexPhaseMarker(); - await this.clearIndexUnlocked(); - this.removeForceIndexPhaseMarker(); + const recovery = this.beginClearRecoveryState(); + await this.clearIndexUnlocked(recovery.compatibilityDecision); + this.finishClearRecoveryState(); return this.indexUnlocked(onProgress, [], true); }); } @@ -5492,7 +5565,8 @@ export class Indexer { async clearIndex(): Promise { await this.withIndexMutationLease("clear", async (recoveredOwners) => { await this.ensureInitializedUnlocked(recoveredOwners); - await this.clearIndexUnlocked(); + const recovery = this.beginClearRecoveryState(); + await this.clearIndexUnlocked(recovery.compatibilityDecision); }); } @@ -5527,21 +5601,32 @@ export class Indexer { this.indexCompatibility = this.validateIndexCompatibility(this.configuredProviderInfo!); } - private clearGlobalIndexUnlocked(projectRoot = this.projectRoot, roots = this.getScopedRoots()): void { + private clearGlobalIndexUnlocked( + projectRoot = this.projectRoot, + roots = this.getScopedRoots(), + recoveryDecision?: IndexLockClearRecoveryState["compatibilityDecision"], + ): void { const { store, invertedIndex, database } = this.requireLoadedIndexState(); store.load(); invertedIndex.load(); this.loadFileHashCache(); const compatibility = this.checkCompatibility(); + const compatibilityDecision = recoveryDecision ?? ( + compatibility.compatible + ? "compatible" + : compatibility.code === IncompatibilityCode.EMBEDDING_STRATEGY_MISMATCH + ? "embedding-strategy-mismatch" + : "incompatible" + ); const allMetadata = store.getAllMetadata(); const hasForeignData = allMetadata.some(({ metadata }) => !this.isFileInCurrentScope(metadata.filePath, roots)) || - this.hasForeignScopedBranchData(projectRoot) || + this.hasForeignScopedBranchData(projectRoot, roots) || this.hasForeignScopedFileHashData(roots) || this.hasForeignScopedFailedBatches(roots); - if (!compatibility.compatible && hasForeignData) { - if (compatibility.code === IncompatibilityCode.EMBEDDING_STRATEGY_MISMATCH) { + if (compatibilityDecision !== "compatible" && hasForeignData) { + if (compatibilityDecision === "embedding-strategy-mismatch") { this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot); this.clearScopedFileHashCache(roots); this.clearScopedFailedBatches(roots); @@ -5549,7 +5634,9 @@ export class Indexer { database.setMetadata(this.getProjectForceReembedMetadataKey(projectIdentityHash), "true"); database.deleteMetadata(this.getProjectEmbeddingStrategyMetadataKey(projectIdentityHash)); database.deleteMetadata(this.getProjectMigrationFinalizedMetadataKey(projectIdentityHash)); - this.indexCompatibility = { compatible: true }; + if (projectRoot === this.projectRoot) { + this.indexCompatibility = { compatible: true }; + } return; } @@ -5568,14 +5655,18 @@ export class Indexer { this.clearSharedIndexProjectData(store, invertedIndex, database, roots, projectRoot); this.clearScopedFileHashCache(roots); this.clearScopedFailedBatches(roots); - this.indexCompatibility = compatibility; + if (projectRoot === this.projectRoot) { + this.indexCompatibility = compatibility; + } } - private async clearIndexUnlocked(): Promise { + private async clearIndexUnlocked( + recoveryDecision?: IndexLockClearRecoveryState["compatibilityDecision"], + ): Promise { const { store, invertedIndex, database } = this.requireLoadedIndexState(); if (this.config.scope === "global") { - this.clearGlobalIndexUnlocked(); + this.clearGlobalIndexUnlocked(this.projectRoot, this.getScopedRoots(), recoveryDecision); return; } diff --git a/tests/indexer-checkpoint-resume.test.ts b/tests/indexer-checkpoint-resume.test.ts index c82ea7a..2d6f64d 100644 --- a/tests/indexer-checkpoint-resume.test.ts +++ b/tests/indexer-checkpoint-resume.test.ts @@ -115,6 +115,27 @@ describe("indexer checkpoint resume", () => { return { projectBDir, kbDir }; } + function getClearRecoveryState( + model = "checkpoint-test-model", + compatibilityDecision: "compatible" | "embedding-strategy-mismatch" | "incompatible" = "compatible", + ): { + phase: "clearing"; + embeddingProvider: "custom"; + embeddingModel: string; + embeddingDimensions: number; + embeddingStrategyVersion: string; + compatibilityDecision: "compatible" | "embedding-strategy-mismatch" | "incompatible"; + } { + return { + phase: "clearing", + embeddingProvider: "custom", + embeddingModel: model, + embeddingDimensions: 8, + embeddingStrategyVersion: "2", + compatibilityDecision, + }; + } + beforeEach(() => { failEmbeddingText = null; projectDir = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-project-")); @@ -432,6 +453,10 @@ describe("indexer checkpoint resume", () => { startedAt: new Date().toISOString(), operation: "clear", token: "11111111-1111-4111-8111-111111111111", + recoveryProtocolVersion: 1, + projectRoot: projectDir, + scopedRoots: [canonicalPath(projectDir), canonicalPath(kbDir)], + clearRecovery: getClearRecoveryState(), })); const fileHashesPath = path.join(indexDir, "file-hashes.json"); @@ -872,8 +897,10 @@ describe("indexer checkpoint resume", () => { startedAt: new Date().toISOString(), operation: "clear", token: "11111111-1111-4111-8111-111111111111", + recoveryProtocolVersion: 1, projectRoot: projectDir, scopedRoots: [canonicalPath(projectDir), canonicalPath(kbDir)], + clearRecovery: getClearRecoveryState(), })); // Project B reclaims the lease. The recovery must clear A's data (and the @@ -930,8 +957,10 @@ describe("indexer checkpoint resume", () => { startedAt: new Date().toISOString(), operation: "clear", token: "11111111-1111-4111-8111-111111111111", + recoveryProtocolVersion: 1, projectRoot: projectDir, scopedRoots: [canonicalPath(projectDir), canonicalPath(kbDir)], + clearRecovery: getClearRecoveryState(), })); // B reclaims the lease through retryFailedBatches, which runs the @@ -1023,4 +1052,219 @@ describe("indexer checkpoint resume", () => { fs.rmSync(kbDir, { recursive: true, force: true }); fs.rmSync(tempHome, { recursive: true, force: true }); }); + + it("global recovery retains legacy clear markers when the originating state is unknown", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const token = "22222222-2222-4222-8222-222222222222"; + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "clear", + token, + })); + + await expect( + createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).retryFailedBatches(), + ).rejects.toThrow("originating recovery state is unknown"); + + expect(fs.existsSync(path.join(indexDir, `indexing.lock.recovery.${token}`))).toBe(true); + expect(fs.existsSync(lockPath)).toBe(false); + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + expect(db.getChunksByName("alphaOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("betaOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("sharedOne").length).toBeGreaterThan(0); + } finally { + db.close(); + } + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("cross-project clear recovery rejects a different embedding configuration", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const token = "33333333-3333-4333-8333-333333333333"; + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "clear", + token, + recoveryProtocolVersion: 1, + projectRoot: projectDir, + scopedRoots: [canonicalPath(projectDir), canonicalPath(kbDir)], + clearRecovery: getClearRecoveryState("originating-model"), + })); + + await expect( + createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).retryFailedBatches(), + ).rejects.toThrow("embedding configuration does not match the originating lease"); + + expect(fs.existsSync(path.join(indexDir, `indexing.lock.recovery.${token}`))).toBe(true); + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + expect(db.getChunksByName("alphaOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("betaOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("sharedOne").length).toBeGreaterThan(0); + } finally { + db.close(); + } + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("cross-project clear recovery preserves the originating compatibility decision", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).index(); + + const projectAHash = projectIdentityHash(projectDir); + const writableDb = new Database(path.join(indexDir, "codebase.db")); + writableDb.setMetadata(`index.embeddingStrategyVersion.${projectAHash}`, "1"); + writableDb.close(); + + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "clear", + token: "66666666-6666-4666-8666-666666666666", + recoveryProtocolVersion: 1, + projectRoot: projectDir, + scopedRoots: [canonicalPath(projectDir), canonicalPath(kbDir)], + clearRecovery: getClearRecoveryState("checkpoint-test-model", "embedding-strategy-mismatch"), + })); + + const retry = await createGlobalIndexer(projectBDir, "checkpoint-test-model", kbDir).retryFailedBatches(); + expect(retry.remaining).toBe(0); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + expect(db.getMetadata(`index.forceReembed.${projectAHash}`)).toBe("true"); + expect(db.getMetadata(`index.embeddingStrategyVersion.${projectAHash}`)).toBeNull(); + expect(db.getChunksByName("alphaOne").length).toBe(0); + expect(db.getChunksByName("betaOne").length).toBeGreaterThan(0); + } finally { + db.close(); + } + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("stale force-index phase markers cannot clear another recovered owner", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + const beforeCalls = fetchSpy.mock.calls.length; + + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "force-index", + token: "44444444-4444-4444-8444-444444444444", + recoveryProtocolVersion: 1, + projectRoot: projectDir, + scopedRoots: [canonicalPath(projectDir), canonicalPath(kbDir)], + })); + fs.writeFileSync(path.join(indexDir, "force-index-phase"), JSON.stringify({ + phase: "clearing", + ownerToken: "55555555-5555-4555-8555-555555555555", + })); + + const stats = await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + expect(stats.failedChunks).toBe(0); + expect(countEmbeddedTexts(fetchSpy, beforeCalls)).toBe(0); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + expect(db.getChunksByName("alphaOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("sharedOne").length).toBeGreaterThan(0); + } finally { + db.close(); + } + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); + + it("legacy force-index phase markers fail closed when ownership is unknown", async () => { + const tempHome = fs.mkdtempSync(path.join(os.tmpdir(), "checkpoint-resume-home-")); + vi.stubEnv("HOME", tempHome); + vi.stubEnv("USERPROFILE", tempHome); + const { projectBDir, kbDir } = setupGlobalScope(); + + await createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(); + + const deadProcess = spawnSync(process.execPath, ["-e", "process.exit(0)"]); + const token = "77777777-7777-4777-8777-777777777777"; + const lockPath = path.join(indexDir, "indexing.lock"); + fs.mkdirSync(lockPath, { recursive: true }); + fs.writeFileSync(path.join(lockPath, "owner.json"), JSON.stringify({ + pid: deadProcess.pid!, + hostname: os.hostname(), + startedAt: new Date().toISOString(), + operation: "force-index", + token, + })); + fs.writeFileSync(path.join(indexDir, "force-index-phase"), "clearing"); + + await expect( + createGlobalIndexer(projectDir, "checkpoint-test-model", kbDir).index(), + ).rejects.toThrow("legacy clearing phase ownership is unknown"); + + expect(fs.existsSync(path.join(indexDir, `indexing.lock.recovery.${token}`))).toBe(true); + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + expect(db.getChunksByName("alphaOne").length).toBeGreaterThan(0); + expect(db.getChunksByName("sharedOne").length).toBeGreaterThan(0); + } finally { + db.close(); + } + + fs.rmSync(projectBDir, { recursive: true, force: true }); + fs.rmSync(kbDir, { recursive: true, force: true }); + fs.rmSync(tempHome, { recursive: true, force: true }); + }); }); From 19f10548727954b7af80d616ebff5eb8cfe08533 Mon Sep 17 00:00:00 2001 From: Nicolas COMPAIN Date: Mon, 17 Aug 2026 12:24:05 +0200 Subject: [PATCH 5/5] fix(indexer): publish empty failed-batch checkpoints Publish filtered failed-batch state before file hashes at checkpoints and final index exits. Persist an atomic empty state when stale retries are discarded, and cover crash recovery with and without an intermediate checkpoint. --- CHANGELOG.md | 2 +- src/indexer/index.ts | 88 +++++++++++++++---------- tests/indexer-checkpoint-resume.test.ts | 52 +++++++++++++++ 3 files changed, 105 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20f7099..39355b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- **Interrupted indexing resume**: Interrupted indexing runs now persist incremental checkpoints (database, vectors, BM25, failed batches, and file hashes), so a subsequent run resumes incrementally instead of re-embedding the whole project. Interrupted global clears and force-indexes are completed on recovery (scoped to the current project when foreign data is present), and checkpointed failed batches and pending retries survive for later retry runs. Checkpoint persistence orders the BM25 index before the vector store so a crash between the two cannot orphan chunks from keyword search. Project-scope dead-lease recovery preserves checkpointed artifacts for incremental resume instead of resetting the local index. Pending retry chunks are no longer duplicated across multiple checkpoints, and finalization deduplicates failed-batch records by chunk ID (keeping the highest attempt count). +- **Interrupted indexing resume**: Interrupted indexing runs now persist incremental checkpoints (database, vectors, BM25, failed batches, and file hashes), so a subsequent run resumes incrementally instead of re-embedding the whole project. Interrupted global clears and force-indexes are completed on recovery (scoped to the current project when foreign data is present), and checkpointed failed batches and pending retries survive for later retry runs. Checkpoint persistence orders the BM25 index before the vector store so a crash between the two cannot orphan chunks from keyword search. Project-scope dead-lease recovery preserves checkpointed artifacts for incremental resume instead of resetting the local index. Pending retry chunks are no longer duplicated across multiple checkpoints, empty failed-batch state is published before new file hashes, and finalization deduplicates failed-batch records by chunk ID (keeping the highest attempt count). - **Interrupted recovery safety**: Recovery of an interrupted global clear or clearing-phase force-index now replays the clear against the originating project scope persisted with the lease, so a project reclaiming a dead lease can no longer delete another project's indexed data. The destructive phase and effective embedding configuration are bound to the lease owner; legacy or configuration-mismatched clears fail closed and retain their recovery marker instead of applying the reclaiming project's settings. Retry runs restore SQLite chunk rows that a recovery health check may have collected as orphans, so a branch reference can never point at a chunk without metadata. - **Forced re-embed migration safety**: While a project-scoped forced re-embed is pending, checkpoints no longer stamp new embedding metadata or a compatibility certificate, unchanged scope files are re-embedded instead of skipped, and `retryFailedBatches` only clears the pending migration after the main run has committed its new metadata. This prevents an interrupted migration from resuming into a silent mix of old and new vector spaces. - **Conceptual context retrieval**: Prefer implementation paths over tests and documentation for code-focused `codebase_context` searches, without applying a hard source-only filter or changing documentation and test queries. diff --git a/src/indexer/index.ts b/src/indexer/index.ts index bdd67ad..1f165be 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -504,6 +504,13 @@ interface FailedBatchWriteState { recordsWritten: number; } +interface FailedBatchProcessingState { + state: FailedBatchWriteState; + latestById: Map; + materializedRetryIds: Set; + discardedExistingRecords: boolean; +} + interface EmbeddingRateLimitState { backoffMs: number; } @@ -2227,12 +2234,13 @@ export class Indexer { if (retained.length > 0) { writeFailedBatchRecords(this.failedBatchesPath, retained); } else { + writeFailedBatchRecords(this.failedBatchesPath, []); this.clearFailedBatchState(); } return; } - state.writer.cleanup(); + state.writer.commit(); this.clearFailedBatchState(); } @@ -2247,10 +2255,10 @@ export class Indexer { database: Database, store: VectorStore, invertedIndex: InvertedIndex, - failedProcessing: { state: FailedBatchWriteState; latestById: Map; materializedRetryIds: Set }, + failedProcessing: FailedBatchProcessingState, + resolvedRetryChunkIds: ReadonlySet, currentFileHashes: Map, committedFilePaths: Set, - unchangedFilePaths: Set, scopedRoots: string[] | null, configuredProviderInfo: ConfiguredProviderInfo, ): void { @@ -2266,7 +2274,11 @@ export class Indexer { // chunks whose vectors are already durable. this.saveInvertedIndex(invertedIndex); store.save(); - if (failedProcessing.state.recordsWritten > 0 || failedProcessing.latestById.size > 0) { + if ( + failedProcessing.state.recordsWritten > 0 + || failedProcessing.latestById.size > 0 + || failedProcessing.discardedExistingRecords + ) { // Persist the pending retries alongside the written records so a crash // after this checkpoint cannot lose them. for (const metadata of failedProcessing.latestById.values()) { @@ -2288,20 +2300,17 @@ export class Indexer { } } } - failedProcessing.state.writer.commit(); + this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds); failedProcessing.state = this.createFailedBatchWriteState(); + failedProcessing.discardedExistingRecords = false; // Preserve the committed records (including out-of-scope projects' - // failed batches) so finalization never deletes them, but drop retries - // that were already completed during this run. + // failed batches) so finalization never deletes them. for (const record of this.loadSerializedFailedBatches()) { for (const rawChunk of record.chunks) { const chunkId = getPendingChunkId(rawChunk); - const filePath = getPendingChunkFilePath(rawChunk); - const inScope = scopedRoots === null || (filePath !== null && this.isFileInCurrentScope(filePath, scopedRoots)); - const isPendingRetry = chunkId !== null && failedProcessing.latestById.has(chunkId); - const isCompletedRetry = !isPendingRetry && inScope && filePath !== null && unchangedFilePaths.has(filePath); - if (!isCompletedRetry) { - this.writeFailedBatchRecord(failedProcessing.state, { ...record, chunks: [rawChunk] }); + this.writeFailedBatchRecord(failedProcessing.state, { ...record, chunks: [rawChunk] }); + if (chunkId !== null) { + failedProcessing.materializedRetryIds.add(chunkId); } } } @@ -2350,9 +2359,10 @@ export class Indexer { private prepareFailedBatchProcessing( roots: string[] | null, shouldProcess: (filePath: string | null) => boolean, - ): { state: FailedBatchWriteState; latestById: Map; materializedRetryIds: Set } { + ): FailedBatchProcessingState { const state = this.createFailedBatchWriteState(); const latestById = new Map(); + let discardedExistingRecords = false; try { for (const batch of this.loadSerializedFailedBatches()) { @@ -2364,11 +2374,13 @@ export class Indexer { continue; } if (!shouldProcess(filePath)) { + discardedExistingRecords = true; continue; } const chunkId = getPendingChunkId(rawChunk); if (!chunkId) { + discardedExistingRecords = true; continue; } const existing = latestById.get(chunkId); @@ -2382,7 +2394,12 @@ export class Indexer { } } } - return { state, latestById, materializedRetryIds: new Set() }; + return { + state, + latestById, + materializedRetryIds: new Set(), + discardedExistingRecords, + }; } catch (error) { state.writer.cleanup(); throw error; @@ -4315,6 +4332,7 @@ export class Indexer { let processedChangedFiles = 0; let lastCheckpointChunks = 0; const committedFilePaths = new Set(unchangedFilePaths); + const resolvedRetryChunkIds = new Set(); for (const descriptorBatch of iterateOrderedFileBatches( changedFileDescriptors, (descriptor) => descriptor.sourceBytes, @@ -4588,16 +4606,15 @@ export class Indexer { store, invertedIndex, failedProcessing, + resolvedRetryChunkIds, currentFileHashes, committedFilePaths, - unchangedFilePaths, scopedRoots, configuredProviderInfo, ); } } - const resolvedRetryChunkIds = new Set(); const retryableFailedChunks = this.iterateLatestFailedChunks( failedProcessing.latestById, scopedRoots, @@ -4679,9 +4696,9 @@ export class Indexer { store, invertedIndex, failedProcessing, + resolvedRetryChunkIds, currentFileHashes, committedFilePaths, - unchangedFilePaths, scopedRoots, configuredProviderInfo, ); @@ -4727,13 +4744,6 @@ export class Indexer { if (removedStoredChunks) { this.saveInvertedIndex(invertedIndex); } - if (scopedRoots) { - this.replaceScopedFileHashCache(currentFileHashes, scopedRoots); - } else { - this.fileHashCache = currentFileHashes; - this.saveFileHashCache(); - } - this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds); database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION); database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION); database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION); @@ -4742,6 +4752,13 @@ export class Indexer { this.indexCompatibility = { compatible: true }; database.commitWriteTransaction(); writeTransactionActive = false; + this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds); + if (scopedRoots) { + this.replaceScopedFileHashCache(currentFileHashes, scopedRoots); + } else { + this.fileHashCache = currentFileHashes; + this.saveFileHashCache(); + } stats.durationMs = Date.now() - startTime; onProgress?.({ phase: "complete", @@ -4766,13 +4783,6 @@ export class Indexer { ); store.save(); this.saveInvertedIndex(invertedIndex); - if (scopedRoots) { - this.replaceScopedFileHashCache(currentFileHashes, scopedRoots); - } else { - this.fileHashCache = currentFileHashes; - this.saveFileHashCache(); - } - this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds); database.setMetadata(swiftParserMetadataKey, SWIFT_PARSER_VERSION); database.setMetadata(metalParserMetadataKey, METAL_PARSER_VERSION); database.setMetadata(symbolExtractorMetadataKey, SYMBOL_EXTRACTOR_VERSION); @@ -4781,6 +4791,13 @@ export class Indexer { this.indexCompatibility = { compatible: true }; database.commitWriteTransaction(); writeTransactionActive = false; + this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds); + if (scopedRoots) { + this.replaceScopedFileHashCache(currentFileHashes, scopedRoots); + } else { + this.fileHashCache = currentFileHashes; + this.saveFileHashCache(); + } stats.durationMs = Date.now() - startTime; onProgress?.({ phase: "complete", @@ -4818,16 +4835,15 @@ export class Indexer { store.save(); this.saveInvertedIndex(invertedIndex); + database.commitWriteTransaction(); + writeTransactionActive = false; + this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds); if (scopedRoots) { this.replaceScopedFileHashCache(currentFileHashes, scopedRoots); } else { this.fileHashCache = currentFileHashes; this.saveFileHashCache(); } - this.finalizeFailedBatchWriteState(failedProcessing.state, resolvedRetryChunkIds); - - database.commitWriteTransaction(); - writeTransactionActive = false; if (this.config.indexing.autoGc && stats.removedChunks > 0) { const gcReset = await this.maybeRunOrphanGc(); diff --git a/tests/indexer-checkpoint-resume.test.ts b/tests/indexer-checkpoint-resume.test.ts index 2d6f64d..c94f2a8 100644 --- a/tests/indexer-checkpoint-resume.test.ts +++ b/tests/indexer-checkpoint-resume.test.ts @@ -849,6 +849,58 @@ describe("indexer checkpoint resume", () => { expect(failedContent2).not.toContain("alphaOne"); }); + async function verifyEmptyFailureStateBeforeHash(checkpointIntervalChunks?: number): Promise { + const alphaFile = path.join(projectDir, "src", "alpha.ts"); + const failedBatchesPath = path.join(indexDir, "failed-batches.json"); + writeSourceFile(alphaFile, ["alphaOne"]); + + failEmbeddingText = "alphaOne"; + await createIndexer(checkpointIntervalChunks).index(); + expect(fs.readFileSync(failedBatchesPath, "utf-8")).toContain("alphaOne"); + failEmbeddingText = null; + + writeSourceFile(alphaFile, ["alphaReplacement"]); + const prototype = Indexer.prototype as unknown as { + saveFileHashCache(this: Indexer): void; + }; + const saveFileHashCache = prototype.saveFileHashCache; + const hashSave = vi.spyOn(prototype, "saveFileHashCache").mockImplementation(function (this: Indexer) { + saveFileHashCache.call(this); + throw new Error("simulated interruption after empty failed-batch checkpoint"); + }); + try { + await expect(createIndexer(checkpointIntervalChunks).index()).rejects.toThrow( + "simulated interruption after empty failed-batch checkpoint", + ); + } finally { + hashSave.mockRestore(); + } + + expect(fs.existsSync(failedBatchesPath)).toBe(false); + const beforeResumeCalls = fetchSpy.mock.calls.length; + await createIndexer(checkpointIntervalChunks).index(); + expect(countEmbeddedTexts(fetchSpy, beforeResumeCalls)).toBe(0); + + const db = Database.openReadOnly(path.join(indexDir, "codebase.db")); + try { + const branchChunkIds = new Set(db.getBranchChunkIds("default")); + const staleChunks = db.getChunksByName("alphaOne"); + const currentChunks = db.getChunksByName("alphaReplacement"); + expect(staleChunks.every((chunk) => !branchChunkIds.has(chunk.chunkId))).toBe(true); + expect(currentChunks.some((chunk) => branchChunkIds.has(chunk.chunkId))).toBe(true); + } finally { + db.close(); + } + } + + it("empty checkpoint state removes stale failures before persisting the new file hash", async () => { + await verifyEmptyFailureStateBeforeHash(1); + }); + + it("final empty failed-batch state is published before the new file hash", async () => { + await verifyEmptyFailureStateBeforeHash(); + }); + it("checkpoint does not duplicate pending retry chunks across multiple checkpoints", async () => { const alphaFile = path.join(projectDir, "src", "alpha.ts"); const betaFile = path.join(projectDir, "src", "beta.ts");