diff --git a/CHANGELOG.md b/CHANGELOG.md index 13464a1..93bb2ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `--embedding-model` to the cross-repository benchmark runner, preserving `nomic-embed-text` as the default while recording the selected local Ollama model in its artifacts. - Expanded the frozen mixed-intent cross-repository cohort to 100 reviewed queries across JavaScript, Python, Go, Rust, Java, C#, PHP, and Ruby, with additional hard Ruby and C# cases. - Added a deterministic pull-request CI gate that clones each pinned cohort revision and validates every evidence path plus definition symbol without invoking embeddings. +- Added a `dryRun` option to `index_codebase` and an `index --dry-run` CLI flag: a read-only preflight that parses the real file set and reports the exact embedding token total (files, source chunks, tokens) without requesting embeddings or writing to the index. The total serves as a fixed, monotonic denominator for live indexing progress (a force index climbs to ~100%; an incremental index tops out below 100% because cached chunks are counted but not re-embedded) and as a cost preview before committing GPU or time. ### Changed diff --git a/docs/installation.md b/docs/installation.md index 9ab2523..7baebcc 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -189,6 +189,7 @@ opencode-codebase-index-mcp index --project /path/to/repo --estimate-only When using `index`: - `--estimate-only` prints the estimate directly and exits. +- `--dry-run` parses the file set and reports the exact embedding token total (files, source chunks, tokens) without indexing; read-only, no embedding requests. - `--force` bypasses stale-index checks. - `--verbose` includes detailed final index statistics. - `--config` loads and parses that file, then initializes the runtime from it before indexing. diff --git a/docs/tools.md b/docs/tools.md index 8035b39..6ce8ffa 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -118,7 +118,7 @@ Reports readiness, chunk counts, compatibility, current provider/model, and inde ### `index_codebase` -Creates or updates the index. Incremental indexing is the default. Use `force: true` only for a required full rebuild. `estimateOnly` reports estimated embedding work without indexing. +Creates or updates the index. Incremental indexing is the default. Use `force: true` only for a required full rebuild. `estimateOnly` reports estimated embedding work without indexing. `dryRun` parses the real file set and reports the exact embedding token total (files, source chunks, tokens) without requesting embeddings or writing to the index — a read-only preflight. ### `index_health_check` diff --git a/src/adapters/mcp/cli.ts b/src/adapters/mcp/cli.ts index 4f44ec4..9e7d675 100644 --- a/src/adapters/mcp/cli.ts +++ b/src/adapters/mcp/cli.ts @@ -33,6 +33,7 @@ export interface CliIndexArgs { config?: string; force: boolean; estimateOnly: boolean; + dryRun: boolean; verbose: boolean; } @@ -69,6 +70,7 @@ export function parseIndexArgs(argv: string[], cwd: string): CliIndexArgs { let config: string | undefined; let force = false; let estimateOnly = false; + let dryRun = false; let verbose = false; for (let i = 0; i < argv.length; i += 1) { @@ -111,13 +113,16 @@ export function parseIndexArgs(argv: string[], cwd: string): CliIndexArgs { continue; } - if (arg === "--force" || arg === "--estimate-only" || arg === "--verbose") { + if (arg === "--force" || arg === "--estimate-only" || arg === "--dry-run" || arg === "--verbose") { if (arg === "--force") { force = true; } if (arg === "--estimate-only") { estimateOnly = true; } + if (arg === "--dry-run") { + dryRun = true; + } if (arg === "--verbose") { verbose = true; } @@ -131,7 +136,7 @@ export function parseIndexArgs(argv: string[], cwd: string): CliIndexArgs { throw new Error(`Unknown index option: ${arg}`); } - return { project, host, config, force, estimateOnly, verbose }; + return { project, host, config, force, estimateOnly, dryRun, verbose }; } export function loadCliRawConfig(args: CliArgs): unknown { @@ -149,6 +154,7 @@ Options: --config Explicit JSON config path --force Rebuild index even if already up to date --estimate-only Estimate indexing cost only + --dry-run Parse only; report the exact embedding token total without indexing --verbose Include detailed final index statistics --help Show this message @@ -391,6 +397,7 @@ export async function handleIndexCommand( const indexArgs: SharedIndexCodebaseArgs = { force: parsedArgs.force, estimateOnly: parsedArgs.estimateOnly, + dryRun: parsedArgs.dryRun, verbose: parsedArgs.verbose, }; diff --git a/src/adapters/mcp/register-tools.ts b/src/adapters/mcp/register-tools.ts index cab75fd..a2c93a9 100644 --- a/src/adapters/mcp/register-tools.ts +++ b/src/adapters/mcp/register-tools.ts @@ -200,6 +200,7 @@ export function registerMcpTools(server: McpServer, runtime: McpServerRuntime): { force: allowNullAsUndefined(z.boolean().optional().default(false)).describe("Force reindex even if already indexed"), estimateOnly: allowNullAsUndefined(z.boolean().optional().default(false)).describe("Only show cost estimate without indexing"), + dryRun: allowNullAsUndefined(z.boolean().optional().default(false)).describe("Parse the file set and report the exact embedding token total without indexing. Read-only; the index is not changed. The total is the value 'Tokens used' climbs to for a force index (and an upper bound for an incremental)."), verbose: allowNullAsUndefined(z.boolean().optional().default(false)).describe("Show detailed info about skipped files and parsing failures"), }, async (args) => { diff --git a/src/adapters/opencode/tools.ts b/src/adapters/opencode/tools.ts index 9b52f6e..0c3b158 100644 --- a/src/adapters/opencode/tools.ts +++ b/src/adapters/opencode/tools.ts @@ -192,6 +192,7 @@ export const index_codebase: ToolDefinition = tool({ args: { force: z.boolean().optional().default(false).describe("Force reindex even if already indexed"), estimateOnly: z.boolean().optional().default(false).describe("Only show cost estimate without indexing"), + dryRun: z.boolean().optional().default(false).describe("Parse the file set and report the exact embedding token total without indexing. Read-only; the index is not changed. The total is the value 'Tokens used' climbs to for a force index (and an upper bound for an incremental)."), verbose: z.boolean().optional().default(false).describe("Show detailed info about skipped files and parsing failures"), }, async execute(args, context) { diff --git a/src/adapters/pi/extension.ts b/src/adapters/pi/extension.ts index c7f46c5..96c8453 100644 --- a/src/adapters/pi/extension.ts +++ b/src/adapters/pi/extension.ts @@ -3,7 +3,7 @@ import { Type } from "typebox"; import { parseConfig } from "../../config/schema.js"; import { loadMergedConfig } from "../../config/merger.js"; -import { formatCostEstimate } from "../../utils/cost.js"; +import { formatCostEstimate, formatDryRunEstimate } from "../../utils/cost.js"; import { formatPrImpact } from "../../tools/format-pr-impact.js"; import { formatCodeCommunities } from "../../tools/format-communities.js"; import { @@ -274,12 +274,14 @@ export default function codebaseIndexPiExtension(pi: ExtensionAPI): void { parameters: Type.Object({ force: Type.Optional(Type.Boolean({ default: false })), estimateOnly: Type.Optional(Type.Boolean({ default: false })), + dryRun: Type.Optional(Type.Boolean({ default: false })), verbose: Type.Optional(Type.Boolean({ default: false })), }), async execute(_toolCallId, params, _signal, _onUpdate, ctx) { try { const result = await runIndexCodebase(projectRoot(ctx), HOST, params); if (result.kind === "estimate") return text(formatCostEstimate(result.estimate), result.estimate); + if (result.kind === "dryrun") return text(formatDryRunEstimate(result.dryrun), result.dryrun); if (result.kind === "busy") return text(result.text, { code: "INDEX_BUSY" }); if (result.kind === "message") return text(result.text); return text(formatIndexStats(result.stats, params.verbose ?? false), result.stats); diff --git a/src/indexer/index.ts b/src/indexer/index.ts index 6c3b8b0..605959f 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -14,7 +14,7 @@ import { CustomProviderNonRetryableError, } from "../embeddings/provider.js"; import { collectFiles, SkippedFile } from "../utils/files.js"; -import { createCostEstimate, CostEstimate } from "../utils/cost.js"; +import { createCostEstimate, CostEstimate, DryRunEstimate } from "../utils/cost.js"; import { Logger, initializeLogger } from "../utils/logger.js"; import { VectorStore, @@ -4014,6 +4014,84 @@ export class Indexer { return createCostEstimate(files, configuredProviderInfo); } + // Dry-run counterpart to index()/forceIndex(): parse the real file set and sum + // estimateTokens over the embedding text of every indexable chunk, without + // calling the embedding provider or writing to the index. Read-only and + // lock-free (mirrors estimateCost). The token sum is the exact value "Tokens + // used" climbs to for a force index (cache bypassed); for an incremental it is + // an upper bound because cached chunks are counted here but not re-embedded. + // Used by index_codebase(dryRun:true) to give a stable, monotonic progress + // denominator that matches the live "Tokens used" basis. + async dryRunCost(): Promise { + const { configuredProviderInfo } = await this.ensureInitialized(); + const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo); + const includePatterns = [...this.config.include, ...this.config.additionalInclude]; + const { files } = await collectFiles( + this.materializedProjectRoot, + includePatterns, + this.config.exclude, + this.config.indexing.maxFileSize, + this.getMaterializedKnowledgeBases(), + { maxDepth: this.config.indexing.maxDepth, maxFilesPerDirectory: this.config.indexing.maxFilesPerDirectory }, + ); + + let filesCount = 0; + let chunksCount = 0; + let tokensToEmbed = 0; + // Parse in the same ordered batches as index() (fileBatchLimits) so the + // memory profile matches a real run. For each file: read, parse, apply the + // same fallback-to-text + maxChunksPerFile cap + selectIndexableChunks path, + // then sum estimateTokens over the embedding text (createEmbeddingTexts) + // — the identical basis the provider reports as "Tokens used". + for (const batch of iterateOrderedFileBatches(files, (f) => f.size, this.fileBatchLimits)) { + const loadedFiles = await Promise.all(batch.map(async (f) => { + try { + return { + path: this.toStoredFilePath(f.path), + content: await fsPromises.readFile(f.path, "utf-8"), + }; + } catch { + // Unreadable file: index() records a parse failure and skips it. + return null; + } + })); + const readable = loadedFiles.filter( + (f): f is { path: string; content: string } => f !== null, + ); + filesCount += readable.length; + const contentByPath = new Map(readable.map((f) => [f.path, f.content])); + const parsedFiles = parseFiles(readable, this.config.indexing.linesPerChunk); + for (const parsed of parsedFiles) { + let chunksToProcess = parsed.chunks; + if ( + this.config.indexing.fallbackToTextOnMaxChunks && + chunksToProcess.length > this.config.indexing.maxChunksPerFile + ) { + const content = contentByPath.get(parsed.path); + if (content !== undefined) { + chunksToProcess = parseFileAsText(parsed.path, content, this.config.indexing.linesPerChunk); + } + } + chunksToProcess = selectIndexableChunks( + chunksToProcess, + this.config.indexing.maxChunksPerFile, + this.config.indexing.semanticOnly, + ); + for (const chunk of chunksToProcess) { + const texts = createEmbeddingTexts(chunk, parsed.path, maxChunkTokens); + // Count one per source chunk (matches stats.indexedChunks); a chunk may + // split into multiple embedding texts, which are summed into tokensToEmbed. + chunksCount += 1; + for (const text of texts) { + tokensToEmbed += estimateTokens(text); + } + } + } + } + + return { filesCount, chunksCount, tokensToEmbed }; + } + async index(onProgress?: ProgressCallback): Promise { return this.withIndexMutationLease("index", async (recoveredOwners) => { return this.indexUnlocked(onProgress, recoveredOwners); diff --git a/src/tools/contracts.ts b/src/tools/contracts.ts index 1caa19b..a0169b0 100644 --- a/src/tools/contracts.ts +++ b/src/tools/contracts.ts @@ -63,6 +63,7 @@ export const DEFAULT_CODEBASE_EDIT_CONTEXT_EDGE_LIMIT = 5 as const; export interface SharedIndexCodebaseArgs { force?: boolean; estimateOnly?: boolean; + dryRun?: boolean; verbose?: boolean; } diff --git a/src/tools/execute-common.ts b/src/tools/execute-common.ts index 90c0bee..8eeca68 100644 --- a/src/tools/execute-common.ts +++ b/src/tools/execute-common.ts @@ -21,7 +21,7 @@ import { runIndexCodebase, runIndexHealthCheck, } from "./operations.js"; -import { formatCostEstimate } from "../utils/cost.js"; +import { formatCostEstimate, formatDryRunEstimate } from "../utils/cost.js"; import { resolveCodebaseEditContext } from "./edit-context.js"; import { resolveCodebaseContext } from "./context.js"; import { @@ -67,6 +67,7 @@ export async function executeIndexCodebase( ): Promise { const result = await runIndexCodebase(projectRoot, host, args, onProgress); if (result.kind === "estimate") return { text: formatCostEstimate(result.estimate) }; + if (result.kind === "dryrun") return { text: formatDryRunEstimate(result.dryrun) }; if (result.kind === "busy") return { text: result.text, isError: true }; if (result.kind === "message") return { text: result.text }; return { text: formatIndexStats(result.stats, args.verbose ?? false) }; diff --git a/src/tools/operations.ts b/src/tools/operations.ts index 2c1c6f5..4578cef 100644 --- a/src/tools/operations.ts +++ b/src/tools/operations.ts @@ -20,7 +20,7 @@ import type { SharedCodeCommunitiesArgs } from "./contracts.js"; import { calculatePercentage, formatProgressTitle, formatStatus } from "./utils.js"; import type { LogLevel } from "../config/schema.js"; import type { LogEntry } from "../utils/logger.js"; -import type { CostEstimate } from "../utils/cost.js"; +import type { CostEstimate, DryRunEstimate } from "../utils/cost.js"; import type { AutoIndexStatusSnapshot } from "../utils/auto-index.js"; import type { SearchTrace } from "../indexer/index.js"; import { @@ -425,10 +425,11 @@ export async function getCallGraphPath( export async function runIndexCodebase( projectRoot: string | undefined, host: HostMode, - args: { force?: boolean; estimateOnly?: boolean; verbose?: boolean }, + args: { force?: boolean; estimateOnly?: boolean; dryRun?: boolean; verbose?: boolean }, onProgress?: ProgressCb, ): Promise< | { kind: "estimate"; estimate: CostEstimate } + | { kind: "dryrun"; dryrun: DryRunEstimate } | { kind: "stats"; stats: IndexStats } | IndexMessageResult | IndexBusyResult @@ -441,6 +442,10 @@ export async function runIndexCodebase( return { kind: "estimate", estimate: await indexer.estimateCost() }; } + if (args.dryRun) { + return { kind: "dryrun", dryrun: await indexer.dryRunCost() }; + } + const coordinated = runCoordinatedIndex(root, host, args.force ?? false, (progress) => { if (onProgress) { void onProgress(formatProgressTitle(progress), { diff --git a/src/utils/cost.ts b/src/utils/cost.ts index e7f51de..80639e7 100644 --- a/src/utils/cost.ts +++ b/src/utils/cost.ts @@ -12,6 +12,29 @@ export interface CostEstimate { isFree: boolean; } +// Result of a dry-run index_codebase pass: parse the real file set, build the +// embedding text for every indexable chunk, and sum estimateTokens over those +// texts without requesting embeddings or writing to the index. Provider +// detection may still run as part of normal initialization (for ollama this +// contacts /api/tags and /api/show); no embeddings are requested and no writes +// occur. +// +// The token sum uses the local estimate (estimateTokens = ceil(len/4)). It +// equals the live "Tokens used" counter only for providers that report usage +// on the same basis (ollama counts ceil(len/4)); for providers that report a +// server tokenizer count (OpenAI, Gemini, custom) it is only an estimate. +// +// For a matching provider and a project-scoped force index, the force pass +// clears its own cached embeddings, so the live counter climbs to this sum. A +// force index on a shared global index can reuse cached embeddings from other +// projects, and an incremental index counts cached chunks that are not +// re-embedded; in both cases the dry-run value is an upper bound. +export interface DryRunEstimate { + filesCount: number; + chunksCount: number; + tokensToEmbed: number; +} + export function estimateTokens(text: string): number { return Math.ceil(text.length / 4); } @@ -85,6 +108,27 @@ export function formatCostEstimate(estimate: CostEstimate): string { `; } +export function formatDryRunEstimate(estimate: DryRunEstimate): string { + return `Dry run: parsed the file set to measure the embedding workload. No embedding requests were made and the index was not changed. + + Files to embed: ${estimate.filesCount.toLocaleString()} + Chunks to embed: ${estimate.chunksCount.toLocaleString()} + Tokens to embed: ${estimate.tokensToEmbed.toLocaleString()} + +The "Tokens to embed" value uses the local estimateTokens(text) = ceil(len/4). It +matches the live "Tokens used" counter only for providers that report usage on the +same basis (ollama); for providers that report a server tokenizer count (OpenAI, +Gemini, custom) it is only an estimate. + +For a matching provider and a project-scoped force index, the force pass clears its +own cached embeddings, so the live counter climbs to this number. A force index on a +shared global index can reuse cached embeddings from other projects, and an +incremental index counts cached chunks that are not re-embedded; in both cases this +number is an upper bound on the live counter, so a progress percent against this +total tops out below 100%. +`; +} + export function formatBytes(bytes: number): string { if (bytes === 0) return "0 B"; const k = 1024; diff --git a/tests/cost.test.ts b/tests/cost.test.ts index 92baa9f..5d0487b 100644 --- a/tests/cost.test.ts +++ b/tests/cost.test.ts @@ -5,9 +5,11 @@ import { estimateChunksFromFiles, estimateCost, formatCostEstimate, + formatDryRunEstimate, parseConfirmationResponse, formatConfirmationPrompt, CostEstimate, + DryRunEstimate, } from "../src/utils/cost.js"; describe("cost utilities", () => { @@ -224,6 +226,40 @@ describe("cost utilities", () => { }); }); + describe("formatDryRunEstimate", () => { + it("should format the file/chunk/token totals with locale grouping", () => { + const estimate: DryRunEstimate = { + filesCount: 970, + chunksCount: 72488, + tokensToEmbed: 76509389, + }; + + const formatted = formatDryRunEstimate(estimate); + + expect(formatted).toContain("Files to embed: 970"); + expect(formatted).toContain("Chunks to embed: 72,488"); + expect(formatted).toContain("Tokens to embed: 76,509,389"); + }); + + it("should state the dry-run did not write embeddings or change the index", () => { + const formatted = formatDryRunEstimate({ filesCount: 1, chunksCount: 1, tokensToEmbed: 1 }); + + expect(formatted).toContain("No embedding requests were made"); + expect(formatted).toContain("index was not changed"); + // The percent-denominator basis note ci relies on: local estimate basis, + // scoped to estimate-based providers (ollama). + expect(formatted).toContain("estimateTokens(text)"); + expect(formatted).toContain("ollama"); + }); + + it("should distinguish force-index exactness from incremental upper bound", () => { + const formatted = formatDryRunEstimate({ filesCount: 0, chunksCount: 0, tokensToEmbed: 0 }); + + expect(formatted).toContain("force index"); + expect(formatted).toContain("upper bound"); + }); + }); + describe("formatConfirmationPrompt", () => { it("should return prompt with all options", () => { const prompt = formatConfirmationPrompt(); diff --git a/tests/dryrun-index.test.ts b/tests/dryrun-index.test.ts new file mode 100644 index 0000000..71c25ed --- /dev/null +++ b/tests/dryrun-index.test.ts @@ -0,0 +1,155 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { parseConfig } from "../src/config/schema.js"; +import { Indexer } from "../src/indexer/index.js"; + +// dryRunCost() is the parse-only counterpart of index()/forceIndex(): it parses +// the real file set with the same chunking path (parseFiles + fallback-to-text +// + maxChunksPerFile cap + selectIndexableChunks + createEmbeddingTexts) and +// sums estimateTokens over the embedding text of every indexable chunk WITHOUT +// calling the embedding provider or writing to the index. These tests pin the +// three guarantees the feature relies on: no embedding provider call and no +// indexed result; the parse path mirrors forceIndex exactly so the token total +// matches a real force index for an estimate-based provider (ollama counts +// ceil(len/4) for every embedded text, the same basis dryRunCost sums); and the +// fallback-to-text + maxChunksPerFile cap path is exercised. +describe("indexer dryRunCost", () => { + let tempDir: string; + let fetchSpy: ReturnType; + let embeddingCalls: string[] = []; + let _indexers: Indexer[] = []; + + beforeEach(() => { + embeddingCalls = []; + fetchSpy = vi.spyOn(globalThis, "fetch"); + fetchSpy.mockImplementation(async (url, init) => { + if (String(url).endsWith("/api/tags")) { + return new Response(JSON.stringify({ models: [{ name: "nomic-embed-text" }] }), { status: 200 }); + } + const body = JSON.parse(String(init?.body ?? "{}")) as { prompt?: string; input?: string[] }; + // ollama single-text /api/embeddings path (legacy / single-text request). + if (body.prompt !== undefined) { + embeddingCalls.push(body.prompt); + return new Response(JSON.stringify({ embedding: Array.from({ length: 768 }, () => 0.1) }), { status: 200 }); + } + // ollama batched /api/embed path (PR #300): input is an array of texts. + // Record each text so the force-index call count is observable, and return + // one embedding vector per input text. + if (Array.isArray(body.input)) { + for (const text of body.input) embeddingCalls.push(text); + const embeddings = body.input.map(() => Array.from({ length: 768 }, () => 0.1)); + return new Response(JSON.stringify({ embeddings }), { status: 200 }); + } + return new Response(JSON.stringify({ error: "unexpected request" }), { status: 400 }); + }); + + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "dryrun-indexer-")); + fs.mkdirSync(path.join(tempDir, "src"), { recursive: true }); + // AST-parsed file: one function chunk. + fs.writeFileSync(path.join(tempDir, "src", "alpha.ts"), "export function alpha() { return 1; }\n", "utf-8"); + // More AST chunks than maxChunksPerFile so the fallback-to-text path runs + // (parseFileAsText -> chunk_by_lines) and selectIndexableChunks caps it. + fs.writeFileSync( + path.join(tempDir, "src", "big.ts"), + [ + "export function a() { return 1; }", + "export function b() { return 2; }", + "export function c() { return 3; }", + "export function d() { return 4; }", + "", + ].join("\n"), + "utf-8", + ); + }); + + afterEach(async () => { + await Promise.all(_indexers.map((i) => i.close())); + _indexers = []; + fetchSpy.mockRestore(); + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + function createIndexer(): Indexer { + const config = parseConfig({ + embeddingProvider: "ollama", + embeddingModel: "nomic-embed-text", + indexing: { + watchFiles: false, + retries: 0, + retryDelayMs: 1, + maxChunksPerFile: 2, + fallbackToTextOnMaxChunks: true, + linesPerChunk: 3, + }, + }); + const indexer = new Indexer(tempDir, config, "opencode"); + _indexers.push(indexer); + return indexer; + } + + it("does not call the embedding provider and leaves the index unindexed", async () => { + const indexer = createIndexer(); + const dryRun = await indexer.dryRunCost(); + + expect(embeddingCalls.length).toBe(0); + expect(dryRun.filesCount).toBeGreaterThanOrEqual(2); + expect(dryRun.chunksCount).toBeGreaterThan(0); + expect(dryRun.tokensToEmbed).toBeGreaterThan(0); + + const status = await indexer.getStatus(); + expect(status.indexed).toBe(false); + }); + + it("matches a force-index token total for an estimate-based provider (ollama)", async () => { + const indexer = createIndexer(); + const dryRun = await indexer.dryRunCost(); + + const stats = await indexer.forceIndex(); + + expect(embeddingCalls.length).toBeGreaterThan(0); + // ollama reports ceil(len/4) per embedded text, the same basis dryRunCost + // sums, so a force index (cache cleared) climbs to exactly this total. + expect(stats.tokensUsed).toBe(dryRun.tokensToEmbed); + expect(stats.indexedChunks).toBe(dryRun.chunksCount); + }); + + it("is idempotent and writes no embeddings across repeated calls", async () => { + const indexer = createIndexer(); + const first = await indexer.dryRunCost(); + const second = await indexer.dryRunCost(); + + expect(second).toEqual(first); + expect(embeddingCalls.length).toBe(0); + + const status = await indexer.getStatus(); + expect(status.indexed).toBe(false); + }); + + it("counts a source chunk once even when it splits into multiple embedding texts", async () => { + // One exported function whose body is a >6100-char string: a single source + // chunk whose embedding text exceeds nomic-embed-text's maxChunkTokens + // (1536 -> maxContentChars ~6100), so createEmbeddingTexts splits it into + // multiple embedding texts. + fs.writeFileSync( + path.join(tempDir, "src", "huge.ts"), + `export function huge() { return "${"x".repeat(7000)}"; }\n`, + "utf-8", + ); + const indexer = createIndexer(); + const dryRun = await indexer.dryRunCost(); + + const stats = await indexer.forceIndex(); + + // chunksCount counts source chunks (like indexedChunks), not embedding texts, + // so it still matches after a chunk splits. + expect(stats.indexedChunks).toBe(dryRun.chunksCount); + // tokens are summed per embedding text, so the token totals still match. + expect(stats.tokensUsed).toBe(dryRun.tokensToEmbed); + // The oversized chunk split: more embedding requests than source chunks. + expect(embeddingCalls.length).toBeGreaterThan(dryRun.chunksCount); + }); +}); \ No newline at end of file diff --git a/tests/mcp-cli-index.test.ts b/tests/mcp-cli-index.test.ts index 47a8503..db646aa 100644 --- a/tests/mcp-cli-index.test.ts +++ b/tests/mcp-cli-index.test.ts @@ -24,6 +24,7 @@ describe("mcp cli index arg parsing", () => { config: undefined, force: false, estimateOnly: false, + dryRun: false, verbose: false, }); }); @@ -50,10 +51,25 @@ describe("mcp cli index arg parsing", () => { config: path.join(tempDir, "my.config.json"), force: true, estimateOnly: true, + dryRun: false, verbose: true, }); }); + it("parses --dry-run as a parse-only flag", () => { + const result = parseIndexArgs(["--dry-run"], tempDir); + + expect(result).toEqual({ + project: tempDir, + host: "opencode", + config: undefined, + force: false, + estimateOnly: false, + dryRun: true, + verbose: false, + }); + }); + it("rejects unknown index options", () => { expect(() => parseIndexArgs(["--bad-option"], tempDir)).toThrow("Unknown index option: --bad-option"); }); @@ -111,6 +127,7 @@ describe("mcp cli index command execution", () => { expect(runIndex).toHaveBeenCalledWith(tempDir, "opencode", { force: true, estimateOnly: true, + dryRun: false, verbose: true, }, expect.any(Function)); }); @@ -194,7 +211,7 @@ describe("mcp cli index command execution", () => { expect(exitCode).toBe(0); expect(initialized).toEqual([{ projectRoot: tempDir, host: "opencode", provider: "custom" }]); - expect(runIndex).toHaveBeenCalledWith(tempDir, "opencode", { force: false, estimateOnly: false, verbose: false }, expect.any(Function)); + expect(runIndex).toHaveBeenCalledWith(tempDir, "opencode", { force: false, estimateOnly: false, dryRun: false, verbose: false }, expect.any(Function)); expect(stdout).toEqual(["ok"]); expect(stderr).toEqual([]); });