From efff5e1c5194faaddf9d7bdf8552d27ead4c93a0 Mon Sep 17 00:00:00 2001 From: Dmitri Khokhlov Date: Mon, 17 Aug 2026 14:03:38 -0700 Subject: [PATCH 1/3] feat(indexing): add dryRun mode to index_codebase for parse-only token totals index_codebase now accepts dryRun:true (CLI: --dry-run) to parse the file set and sum estimateTokens over the real chunk embedding text WITHOUT embedding or writing to the index. The token total is the exact value "Tokens used" climbs to for a force index (cache bypassed) and a stable upper bound for an incremental, so it serves as a fixed, monotonic percent denominator for live progress reporting (e.g. the ~/bin/ci wrapper's preflight). Implemented as a standalone read-only Indexer.dryRunCost() that reuses the exact index() pipeline (collectFiles + parseFiles + fallbackToTextOnMaxChunks + selectIndexableChunks + createEmbeddingTexts + estimateTokens); no DB writes, no ollama call, no lock. Plumbed through operations.ts (result union kind:"dryrun"), execute-common.ts, contracts.ts (SharedIndexCodebaseArgs), the MCP/opencode/pi tool schemas, and the CLI (parseIndexArgs + --dry-run + usage). Formatted by formatDryRunEstimate in utils/cost.ts. Tests: parseIndexArgs --dry-run case + the dryRun:false field in the existing CLI arg/execution assertions; formatDryRunEstimate coverage in cost.test.ts. --- src/adapters/mcp/cli.ts | 11 ++++- src/adapters/mcp/register-tools.ts | 1 + src/adapters/opencode/tools.ts | 1 + src/adapters/pi/extension.ts | 4 +- src/indexer/index.ts | 78 +++++++++++++++++++++++++++++- src/tools/contracts.ts | 1 + src/tools/execute-common.ts | 3 +- src/tools/operations.ts | 9 +++- src/utils/cost.ts | 29 +++++++++++ tests/cost.test.ts | 34 +++++++++++++ tests/mcp-cli-index.test.ts | 19 +++++++- 11 files changed, 182 insertions(+), 8 deletions(-) diff --git a/src/adapters/mcp/cli.ts b/src/adapters/mcp/cli.ts index 4f44ec4c..9e7d675d 100644 --- a/src/adapters/mcp/cli.ts +++ b/src/adapters/mcp/cli.ts @@ -33,6 +33,7 @@ export interface CliIndexArgs { config?: string; force: boolean; estimateOnly: boolean; + dryRun: boolean; verbose: boolean; } @@ -69,6 +70,7 @@ export function parseIndexArgs(argv: string[], cwd: string): CliIndexArgs { let config: string | undefined; let force = false; let estimateOnly = false; + let dryRun = false; let verbose = false; for (let i = 0; i < argv.length; i += 1) { @@ -111,13 +113,16 @@ export function parseIndexArgs(argv: string[], cwd: string): CliIndexArgs { continue; } - if (arg === "--force" || arg === "--estimate-only" || arg === "--verbose") { + if (arg === "--force" || arg === "--estimate-only" || arg === "--dry-run" || arg === "--verbose") { if (arg === "--force") { force = true; } if (arg === "--estimate-only") { estimateOnly = true; } + if (arg === "--dry-run") { + dryRun = true; + } if (arg === "--verbose") { verbose = true; } @@ -131,7 +136,7 @@ export function parseIndexArgs(argv: string[], cwd: string): CliIndexArgs { throw new Error(`Unknown index option: ${arg}`); } - return { project, host, config, force, estimateOnly, verbose }; + return { project, host, config, force, estimateOnly, dryRun, verbose }; } export function loadCliRawConfig(args: CliArgs): unknown { @@ -149,6 +154,7 @@ Options: --config Explicit JSON config path --force Rebuild index even if already up to date --estimate-only Estimate indexing cost only + --dry-run Parse only; report the exact embedding token total without indexing --verbose Include detailed final index statistics --help Show this message @@ -391,6 +397,7 @@ export async function handleIndexCommand( const indexArgs: SharedIndexCodebaseArgs = { force: parsedArgs.force, estimateOnly: parsedArgs.estimateOnly, + dryRun: parsedArgs.dryRun, verbose: parsedArgs.verbose, }; diff --git a/src/adapters/mcp/register-tools.ts b/src/adapters/mcp/register-tools.ts index cab75fde..a2c93a91 100644 --- a/src/adapters/mcp/register-tools.ts +++ b/src/adapters/mcp/register-tools.ts @@ -200,6 +200,7 @@ export function registerMcpTools(server: McpServer, runtime: McpServerRuntime): { force: allowNullAsUndefined(z.boolean().optional().default(false)).describe("Force reindex even if already indexed"), estimateOnly: allowNullAsUndefined(z.boolean().optional().default(false)).describe("Only show cost estimate without indexing"), + dryRun: allowNullAsUndefined(z.boolean().optional().default(false)).describe("Parse the file set and report the exact embedding token total without indexing. Read-only; the index is not changed. The total is the value 'Tokens used' climbs to for a force index (and an upper bound for an incremental)."), verbose: allowNullAsUndefined(z.boolean().optional().default(false)).describe("Show detailed info about skipped files and parsing failures"), }, async (args) => { diff --git a/src/adapters/opencode/tools.ts b/src/adapters/opencode/tools.ts index 9b52f6ea..0c3b158a 100644 --- a/src/adapters/opencode/tools.ts +++ b/src/adapters/opencode/tools.ts @@ -192,6 +192,7 @@ export const index_codebase: ToolDefinition = tool({ args: { force: z.boolean().optional().default(false).describe("Force reindex even if already indexed"), estimateOnly: z.boolean().optional().default(false).describe("Only show cost estimate without indexing"), + dryRun: z.boolean().optional().default(false).describe("Parse the file set and report the exact embedding token total without indexing. Read-only; the index is not changed. The total is the value 'Tokens used' climbs to for a force index (and an upper bound for an incremental)."), verbose: z.boolean().optional().default(false).describe("Show detailed info about skipped files and parsing failures"), }, async execute(args, context) { diff --git a/src/adapters/pi/extension.ts b/src/adapters/pi/extension.ts index c7f46c57..96c84537 100644 --- a/src/adapters/pi/extension.ts +++ b/src/adapters/pi/extension.ts @@ -3,7 +3,7 @@ import { Type } from "typebox"; import { parseConfig } from "../../config/schema.js"; import { loadMergedConfig } from "../../config/merger.js"; -import { formatCostEstimate } from "../../utils/cost.js"; +import { formatCostEstimate, formatDryRunEstimate } from "../../utils/cost.js"; import { formatPrImpact } from "../../tools/format-pr-impact.js"; import { formatCodeCommunities } from "../../tools/format-communities.js"; import { @@ -274,12 +274,14 @@ export default function codebaseIndexPiExtension(pi: ExtensionAPI): void { parameters: Type.Object({ force: Type.Optional(Type.Boolean({ default: false })), estimateOnly: Type.Optional(Type.Boolean({ default: false })), + dryRun: Type.Optional(Type.Boolean({ default: false })), verbose: Type.Optional(Type.Boolean({ default: false })), }), async execute(_toolCallId, params, _signal, _onUpdate, ctx) { try { const result = await runIndexCodebase(projectRoot(ctx), HOST, params); if (result.kind === "estimate") return text(formatCostEstimate(result.estimate), result.estimate); + if (result.kind === "dryrun") return text(formatDryRunEstimate(result.dryrun), result.dryrun); if (result.kind === "busy") return text(result.text, { code: "INDEX_BUSY" }); if (result.kind === "message") return text(result.text); return text(formatIndexStats(result.stats, params.verbose ?? false), result.stats); diff --git a/src/indexer/index.ts b/src/indexer/index.ts index 9f1ee0c0..26de43af 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -14,7 +14,7 @@ import { CustomProviderNonRetryableError, } from "../embeddings/provider.js"; import { collectFiles, SkippedFile } from "../utils/files.js"; -import { createCostEstimate, CostEstimate } from "../utils/cost.js"; +import { createCostEstimate, CostEstimate, DryRunEstimate } from "../utils/cost.js"; import { Logger, initializeLogger } from "../utils/logger.js"; import { VectorStore, @@ -4014,6 +4014,82 @@ export class Indexer { return createCostEstimate(files, configuredProviderInfo); } + // Dry-run counterpart to index()/forceIndex(): parse the real file set and sum + // estimateTokens over the embedding text of every indexable chunk, without + // calling the embedding provider or writing to the index. Read-only and + // lock-free (mirrors estimateCost). The token sum is the exact value "Tokens + // used" climbs to for a force index (cache bypassed); for an incremental it is + // an upper bound because cached chunks are counted here but not re-embedded. + // Used by index_codebase(dryRun:true) to give a stable, monotonic progress + // denominator that matches the live "Tokens used" basis. + async dryRunCost(): Promise { + const { configuredProviderInfo } = await this.ensureInitialized(); + const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo); + const includePatterns = [...this.config.include, ...this.config.additionalInclude]; + const { files } = await collectFiles( + this.materializedProjectRoot, + includePatterns, + this.config.exclude, + this.config.indexing.maxFileSize, + this.getMaterializedKnowledgeBases(), + { maxDepth: this.config.indexing.maxDepth, maxFilesPerDirectory: this.config.indexing.maxFilesPerDirectory }, + ); + + let filesCount = 0; + let chunksCount = 0; + let tokensToEmbed = 0; + // Parse in the same ordered batches as index() (fileBatchLimits) so the + // memory profile matches a real run. For each file: read, parse, apply the + // same fallback-to-text + maxChunksPerFile cap + selectIndexableChunks path, + // then sum estimateTokens over the embedding text (createEmbeddingTexts) + // — the identical basis the provider reports as "Tokens used". + for (const batch of iterateOrderedFileBatches(files, (f) => f.size, this.fileBatchLimits)) { + const loadedFiles = await Promise.all(batch.map(async (f) => { + try { + return { + path: this.toStoredFilePath(f.path), + content: await fsPromises.readFile(f.path, "utf-8"), + }; + } catch { + // Unreadable file: index() records a parse failure and skips it. + return null; + } + })); + const readable = loadedFiles.filter( + (f): f is { path: string; content: string } => f !== null, + ); + filesCount += readable.length; + const contentByPath = new Map(readable.map((f) => [f.path, f.content])); + const parsedFiles = parseFiles(readable, this.config.indexing.linesPerChunk); + for (const parsed of parsedFiles) { + let chunksToProcess = parsed.chunks; + if ( + this.config.indexing.fallbackToTextOnMaxChunks && + chunksToProcess.length > this.config.indexing.maxChunksPerFile + ) { + const content = contentByPath.get(parsed.path); + if (content !== undefined) { + chunksToProcess = parseFileAsText(parsed.path, content, this.config.indexing.linesPerChunk); + } + } + chunksToProcess = selectIndexableChunks( + chunksToProcess, + this.config.indexing.maxChunksPerFile, + this.config.indexing.semanticOnly, + ); + for (const chunk of chunksToProcess) { + const texts = createEmbeddingTexts(chunk, parsed.path, maxChunkTokens); + for (const text of texts) { + chunksCount += 1; + tokensToEmbed += estimateTokens(text); + } + } + } + } + + return { filesCount, chunksCount, tokensToEmbed }; + } + async index(onProgress?: ProgressCallback): Promise { return this.withIndexMutationLease("index", async (recoveredOwners) => { return this.indexUnlocked(onProgress, recoveredOwners); diff --git a/src/tools/contracts.ts b/src/tools/contracts.ts index 1caa19b6..a0169b0a 100644 --- a/src/tools/contracts.ts +++ b/src/tools/contracts.ts @@ -63,6 +63,7 @@ export const DEFAULT_CODEBASE_EDIT_CONTEXT_EDGE_LIMIT = 5 as const; export interface SharedIndexCodebaseArgs { force?: boolean; estimateOnly?: boolean; + dryRun?: boolean; verbose?: boolean; } diff --git a/src/tools/execute-common.ts b/src/tools/execute-common.ts index 90c0bee0..8eeca685 100644 --- a/src/tools/execute-common.ts +++ b/src/tools/execute-common.ts @@ -21,7 +21,7 @@ import { runIndexCodebase, runIndexHealthCheck, } from "./operations.js"; -import { formatCostEstimate } from "../utils/cost.js"; +import { formatCostEstimate, formatDryRunEstimate } from "../utils/cost.js"; import { resolveCodebaseEditContext } from "./edit-context.js"; import { resolveCodebaseContext } from "./context.js"; import { @@ -67,6 +67,7 @@ export async function executeIndexCodebase( ): Promise { const result = await runIndexCodebase(projectRoot, host, args, onProgress); if (result.kind === "estimate") return { text: formatCostEstimate(result.estimate) }; + if (result.kind === "dryrun") return { text: formatDryRunEstimate(result.dryrun) }; if (result.kind === "busy") return { text: result.text, isError: true }; if (result.kind === "message") return { text: result.text }; return { text: formatIndexStats(result.stats, args.verbose ?? false) }; diff --git a/src/tools/operations.ts b/src/tools/operations.ts index 2c1c6f5e..4578cef8 100644 --- a/src/tools/operations.ts +++ b/src/tools/operations.ts @@ -20,7 +20,7 @@ import type { SharedCodeCommunitiesArgs } from "./contracts.js"; import { calculatePercentage, formatProgressTitle, formatStatus } from "./utils.js"; import type { LogLevel } from "../config/schema.js"; import type { LogEntry } from "../utils/logger.js"; -import type { CostEstimate } from "../utils/cost.js"; +import type { CostEstimate, DryRunEstimate } from "../utils/cost.js"; import type { AutoIndexStatusSnapshot } from "../utils/auto-index.js"; import type { SearchTrace } from "../indexer/index.js"; import { @@ -425,10 +425,11 @@ export async function getCallGraphPath( export async function runIndexCodebase( projectRoot: string | undefined, host: HostMode, - args: { force?: boolean; estimateOnly?: boolean; verbose?: boolean }, + args: { force?: boolean; estimateOnly?: boolean; dryRun?: boolean; verbose?: boolean }, onProgress?: ProgressCb, ): Promise< | { kind: "estimate"; estimate: CostEstimate } + | { kind: "dryrun"; dryrun: DryRunEstimate } | { kind: "stats"; stats: IndexStats } | IndexMessageResult | IndexBusyResult @@ -441,6 +442,10 @@ export async function runIndexCodebase( return { kind: "estimate", estimate: await indexer.estimateCost() }; } + if (args.dryRun) { + return { kind: "dryrun", dryrun: await indexer.dryRunCost() }; + } + const coordinated = runCoordinatedIndex(root, host, args.force ?? false, (progress) => { if (onProgress) { void onProgress(formatProgressTitle(progress), { diff --git a/src/utils/cost.ts b/src/utils/cost.ts index e7f51de4..d268861b 100644 --- a/src/utils/cost.ts +++ b/src/utils/cost.ts @@ -12,6 +12,20 @@ export interface CostEstimate { isFree: boolean; } +// Result of a dry-run index_codebase pass: parse the real file set, build the +// embedding text for every indexable chunk, and sum estimateTokens over those +// texts without calling the embedding provider or writing to the index. The +// token sum uses the exact same basis (estimateTokens = ceil(len/4)) the +// provider reports as "Tokens used", so for a force index (cache bypassed) it +// is the precise value "Tokens used" climbs to. For an incremental index it is +// an upper bound: chunks already in the embedding cache are counted here but +// are not re-embedded. +export interface DryRunEstimate { + filesCount: number; + chunksCount: number; + tokensToEmbed: number; +} + export function estimateTokens(text: string): number { return Math.ceil(text.length / 4); } @@ -85,6 +99,21 @@ export function formatCostEstimate(estimate: CostEstimate): string { `; } +export function formatDryRunEstimate(estimate: DryRunEstimate): string { + return `Dry run: parsed the file set to measure the embedding workload. No embeddings were written and the index was not changed. + + Files to embed: ${estimate.filesCount.toLocaleString()} + Chunks to embed: ${estimate.chunksCount.toLocaleString()} + Tokens to embed: ${estimate.tokensToEmbed.toLocaleString()} + +The "Tokens to embed" value uses the same estimateTokens(text) basis the embedding +provider reports as "Tokens used", so a force index (cache bypassed) climbs to exactly +this number. For an incremental index this is an upper bound: chunks already in the +embedding cache are counted here but are not re-embedded, so "Tokens used" climbs to a +lower value and a progress percent against this total tops out below 100%. +`; +} + export function formatBytes(bytes: number): string { if (bytes === 0) return "0 B"; const k = 1024; diff --git a/tests/cost.test.ts b/tests/cost.test.ts index 92baa9f6..63c4dce9 100644 --- a/tests/cost.test.ts +++ b/tests/cost.test.ts @@ -5,9 +5,11 @@ import { estimateChunksFromFiles, estimateCost, formatCostEstimate, + formatDryRunEstimate, parseConfirmationResponse, formatConfirmationPrompt, CostEstimate, + DryRunEstimate, } from "../src/utils/cost.js"; describe("cost utilities", () => { @@ -224,6 +226,38 @@ describe("cost utilities", () => { }); }); + describe("formatDryRunEstimate", () => { + it("should format the file/chunk/token totals with locale grouping", () => { + const estimate: DryRunEstimate = { + filesCount: 970, + chunksCount: 72488, + tokensToEmbed: 76509389, + }; + + const formatted = formatDryRunEstimate(estimate); + + expect(formatted).toContain("Files to embed: 970"); + expect(formatted).toContain("Chunks to embed: 72,488"); + expect(formatted).toContain("Tokens to embed: 76,509,389"); + }); + + it("should state the dry-run did not write embeddings or change the index", () => { + const formatted = formatDryRunEstimate({ filesCount: 1, chunksCount: 1, tokensToEmbed: 1 }); + + expect(formatted).toContain("No embeddings were written"); + expect(formatted).toContain("index was not changed"); + // The percent-denominator basis note ci relies on. + expect(formatted).toContain("estimateTokens(text) basis"); + }); + + it("should distinguish force-index exactness from incremental upper bound", () => { + const formatted = formatDryRunEstimate({ filesCount: 0, chunksCount: 0, tokensToEmbed: 0 }); + + expect(formatted).toContain("force index"); + expect(formatted).toContain("upper bound"); + }); + }); + describe("formatConfirmationPrompt", () => { it("should return prompt with all options", () => { const prompt = formatConfirmationPrompt(); diff --git a/tests/mcp-cli-index.test.ts b/tests/mcp-cli-index.test.ts index 47a8503f..db646aae 100644 --- a/tests/mcp-cli-index.test.ts +++ b/tests/mcp-cli-index.test.ts @@ -24,6 +24,7 @@ describe("mcp cli index arg parsing", () => { config: undefined, force: false, estimateOnly: false, + dryRun: false, verbose: false, }); }); @@ -50,10 +51,25 @@ describe("mcp cli index arg parsing", () => { config: path.join(tempDir, "my.config.json"), force: true, estimateOnly: true, + dryRun: false, verbose: true, }); }); + it("parses --dry-run as a parse-only flag", () => { + const result = parseIndexArgs(["--dry-run"], tempDir); + + expect(result).toEqual({ + project: tempDir, + host: "opencode", + config: undefined, + force: false, + estimateOnly: false, + dryRun: true, + verbose: false, + }); + }); + it("rejects unknown index options", () => { expect(() => parseIndexArgs(["--bad-option"], tempDir)).toThrow("Unknown index option: --bad-option"); }); @@ -111,6 +127,7 @@ describe("mcp cli index command execution", () => { expect(runIndex).toHaveBeenCalledWith(tempDir, "opencode", { force: true, estimateOnly: true, + dryRun: false, verbose: true, }, expect.any(Function)); }); @@ -194,7 +211,7 @@ describe("mcp cli index command execution", () => { expect(exitCode).toBe(0); expect(initialized).toEqual([{ projectRoot: tempDir, host: "opencode", provider: "custom" }]); - expect(runIndex).toHaveBeenCalledWith(tempDir, "opencode", { force: false, estimateOnly: false, verbose: false }, expect.any(Function)); + expect(runIndex).toHaveBeenCalledWith(tempDir, "opencode", { force: false, estimateOnly: false, dryRun: false, verbose: false }, expect.any(Function)); expect(stdout).toEqual(["ok"]); expect(stderr).toEqual([]); }); From 041fac5c2c5532837fbf91c33335cd54f1bb0e39 Mon Sep 17 00:00:00 2001 From: Dmitri Khokhlov Date: Mon, 17 Aug 2026 23:46:46 -0700 Subject: [PATCH 2/3] test(dryRun): scope the token-total claim and add an integration test Address review (codex) of the dryRun feature. The "exact force-index token total" claim was overstated. dryRunCost() sums the local estimate (estimateTokens = ceil(len/4)); the live "Tokens used" counter is provider-reported. They match only for providers that report usage on the same basis (ollama counts ceil(len/4) per embedded text). For providers that report a server tokenizer count (OpenAI, Gemini, custom) the dry-run value is an estimate, not an exact match. Reword the DryRunEstimate docstring and the formatDryRunEstimate output so the claim is provider-conditional, and note that a force index on a shared global index can reuse cached embeddings from other projects, so the dry-run value is an upper bound there (as it is for any incremental index). Update the formatDryRunEstimate unit assertions to the scoped wording. Add tests/dryrun-index.test.ts: an integration test that runs dryRunCost() against a temp project (AST chunking + fallback-to-text + maxChunksPerFile cap) with a mocked ollama provider and asserts (1) no embedding provider call and the index stays unindexed, (2) the dry-run total equals a force-index tokensUsed for an estimate-based provider and the chunk counts match, and (3) dryRunCost is idempotent and writes no embeddings across repeated calls. --- src/utils/cost.ts | 34 ++++++---- tests/cost.test.ts | 6 +- tests/dryrun-index.test.ts | 131 +++++++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 13 deletions(-) create mode 100644 tests/dryrun-index.test.ts diff --git a/src/utils/cost.ts b/src/utils/cost.ts index d268861b..2cfd60ad 100644 --- a/src/utils/cost.ts +++ b/src/utils/cost.ts @@ -14,12 +14,18 @@ export interface CostEstimate { // Result of a dry-run index_codebase pass: parse the real file set, build the // embedding text for every indexable chunk, and sum estimateTokens over those -// texts without calling the embedding provider or writing to the index. The -// token sum uses the exact same basis (estimateTokens = ceil(len/4)) the -// provider reports as "Tokens used", so for a force index (cache bypassed) it -// is the precise value "Tokens used" climbs to. For an incremental index it is -// an upper bound: chunks already in the embedding cache are counted here but -// are not re-embedded. +// texts without calling the embedding provider or writing to the index. +// +// The token sum uses the local estimate (estimateTokens = ceil(len/4)). It +// equals the live "Tokens used" counter only for providers that report usage +// on the same basis (ollama counts ceil(len/4)); for providers that report a +// server tokenizer count (OpenAI, Gemini, custom) it is only an estimate. +// +// For a matching provider and a project-scoped force index, the force pass +// clears its own cached embeddings, so the live counter climbs to this sum. A +// force index on a shared global index can reuse cached embeddings from other +// projects, and an incremental index counts cached chunks that are not +// re-embedded; in both cases the dry-run value is an upper bound. export interface DryRunEstimate { filesCount: number; chunksCount: number; @@ -106,11 +112,17 @@ export function formatDryRunEstimate(estimate: DryRunEstimate): string { Chunks to embed: ${estimate.chunksCount.toLocaleString()} Tokens to embed: ${estimate.tokensToEmbed.toLocaleString()} -The "Tokens to embed" value uses the same estimateTokens(text) basis the embedding -provider reports as "Tokens used", so a force index (cache bypassed) climbs to exactly -this number. For an incremental index this is an upper bound: chunks already in the -embedding cache are counted here but are not re-embedded, so "Tokens used" climbs to a -lower value and a progress percent against this total tops out below 100%. +The "Tokens to embed" value uses the local estimateTokens(text) = ceil(len/4). It +matches the live "Tokens used" counter only for providers that report usage on the +same basis (ollama); for providers that report a server tokenizer count (OpenAI, +Gemini, custom) it is only an estimate. + +For a matching provider and a project-scoped force index, the force pass clears its +own cached embeddings, so the live counter climbs to this number. A force index on a +shared global index can reuse cached embeddings from other projects, and an +incremental index counts cached chunks that are not re-embedded; in both cases this +number is an upper bound on the live counter, so a progress percent against this +total tops out below 100%. `; } diff --git a/tests/cost.test.ts b/tests/cost.test.ts index 63c4dce9..0609fbee 100644 --- a/tests/cost.test.ts +++ b/tests/cost.test.ts @@ -246,8 +246,10 @@ describe("cost utilities", () => { expect(formatted).toContain("No embeddings were written"); expect(formatted).toContain("index was not changed"); - // The percent-denominator basis note ci relies on. - expect(formatted).toContain("estimateTokens(text) basis"); + // The percent-denominator basis note ci relies on: local estimate basis, + // scoped to estimate-based providers (ollama). + expect(formatted).toContain("estimateTokens(text)"); + expect(formatted).toContain("ollama"); }); it("should distinguish force-index exactness from incremental upper bound", () => { diff --git a/tests/dryrun-index.test.ts b/tests/dryrun-index.test.ts new file mode 100644 index 00000000..167ef174 --- /dev/null +++ b/tests/dryrun-index.test.ts @@ -0,0 +1,131 @@ +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { parseConfig } from "../src/config/schema.js"; +import { Indexer } from "../src/indexer/index.js"; + +// dryRunCost() is the parse-only counterpart of index()/forceIndex(): it parses +// the real file set with the same chunking path (parseFiles + fallback-to-text +// + maxChunksPerFile cap + selectIndexableChunks + createEmbeddingTexts) and +// sums estimateTokens over the embedding text of every indexable chunk WITHOUT +// calling the embedding provider or writing to the index. These tests pin the +// three guarantees the feature relies on: no embedding provider call and no +// indexed result; the parse path mirrors forceIndex exactly so the token total +// matches a real force index for an estimate-based provider (ollama counts +// ceil(len/4) for every embedded text, the same basis dryRunCost sums); and the +// fallback-to-text + maxChunksPerFile cap path is exercised. +describe("indexer dryRunCost", () => { + let tempDir: string; + let fetchSpy: ReturnType; + let embeddingCalls: string[] = []; + let _indexers: Indexer[] = []; + + beforeEach(() => { + embeddingCalls = []; + fetchSpy = vi.spyOn(globalThis, "fetch"); + fetchSpy.mockImplementation(async (url, init) => { + if (String(url).endsWith("/api/tags")) { + return new Response(JSON.stringify({ models: [{ name: "nomic-embed-text" }] }), { status: 200 }); + } + const body = JSON.parse(String(init?.body ?? "{}")) as { prompt?: string; input?: string[] }; + // ollama single-text /api/embeddings path (legacy / single-text request). + if (body.prompt !== undefined) { + embeddingCalls.push(body.prompt); + return new Response(JSON.stringify({ embedding: Array.from({ length: 768 }, () => 0.1) }), { status: 200 }); + } + // ollama batched /api/embed path (PR #300): input is an array of texts. + // Record each text so the force-index call count is observable, and return + // one embedding vector per input text. + if (Array.isArray(body.input)) { + for (const text of body.input) embeddingCalls.push(text); + const embeddings = body.input.map(() => Array.from({ length: 768 }, () => 0.1)); + return new Response(JSON.stringify({ embeddings }), { status: 200 }); + } + return new Response(JSON.stringify({ error: "unexpected request" }), { status: 400 }); + }); + + tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "dryrun-indexer-")); + fs.mkdirSync(path.join(tempDir, "src"), { recursive: true }); + // AST-parsed file: one function chunk. + fs.writeFileSync(path.join(tempDir, "src", "alpha.ts"), "export function alpha() { return 1; }\n", "utf-8"); + // More AST chunks than maxChunksPerFile so the fallback-to-text path runs + // (parseFileAsText -> chunk_by_lines) and selectIndexableChunks caps it. + fs.writeFileSync( + path.join(tempDir, "src", "big.ts"), + [ + "export function a() { return 1; }", + "export function b() { return 2; }", + "export function c() { return 3; }", + "export function d() { return 4; }", + "", + ].join("\n"), + "utf-8", + ); + }); + + afterEach(async () => { + await Promise.all(_indexers.map((i) => i.close())); + _indexers = []; + fetchSpy.mockRestore(); + fs.rmSync(tempDir, { recursive: true, force: true }); + }); + + function createIndexer(): Indexer { + const config = parseConfig({ + embeddingProvider: "ollama", + embeddingModel: "nomic-embed-text", + indexing: { + watchFiles: false, + retries: 0, + retryDelayMs: 1, + maxChunksPerFile: 2, + fallbackToTextOnMaxChunks: true, + linesPerChunk: 3, + }, + }); + const indexer = new Indexer(tempDir, config, "opencode"); + _indexers.push(indexer); + return indexer; + } + + it("does not call the embedding provider and leaves the index unindexed", async () => { + const indexer = createIndexer(); + const dryRun = await indexer.dryRunCost(); + + expect(embeddingCalls.length).toBe(0); + expect(dryRun.filesCount).toBeGreaterThanOrEqual(2); + expect(dryRun.chunksCount).toBeGreaterThan(0); + expect(dryRun.tokensToEmbed).toBeGreaterThan(0); + + const status = await indexer.getStatus(); + expect(status.indexed).toBe(false); + }); + + it("matches a force-index token total for an estimate-based provider (ollama)", async () => { + const indexer = createIndexer(); + const dryRun = await indexer.dryRunCost(); + + const stats = await indexer.forceIndex(); + + expect(embeddingCalls.length).toBeGreaterThan(0); + // ollama reports ceil(len/4) per embedded text, the same basis dryRunCost + // sums, so a force index (cache cleared) climbs to exactly this total. + expect(stats.tokensUsed).toBe(dryRun.tokensToEmbed); + expect(stats.indexedChunks).toBe(dryRun.chunksCount); + }); + + it("is idempotent and writes no embeddings across repeated calls", async () => { + const indexer = createIndexer(); + const first = await indexer.dryRunCost(); + const second = await indexer.dryRunCost(); + + expect(second).toEqual(first); + expect(embeddingCalls.length).toBe(0); + + const status = await indexer.getStatus(); + expect(status.indexed).toBe(false); + }); +}); \ No newline at end of file From 8e4e5422f476e0f5b9ff737f4b68cda8a530bfb6 Mon Sep 17 00:00:00 2001 From: Dmitri Khokhlov Date: Tue, 18 Aug 2026 06:57:23 -0700 Subject: [PATCH 3/3] docs+test: address PR #305 review Address the four review items on the dryRun PR. 1. Narrow the read-only guarantee to "no embedding requests" (not "no provider contact"): dryRunCost still runs provider detection as part of normal initialization (for ollama this contacts /api/tags and /api/show) because it needs maxTokens to cap chunk size; no embeddings are requested and no writes occur. Update the DryRunEstimate docstring, formatDryRunEstimate output, and the cost test assertion. 2. Count chunksCount once per source chunk (matching forceIndex indexedChunks), not once per embedding text. tokensToEmbed still sums per embedding text, so it matches tokensUsed. Add a split-chunk regression test: one oversized source chunk that splits into multiple embedding texts still keeps chunksCount == indexedChunks and tokensToEmbed == tokensUsed, with more embedding requests than chunks. 3. Document dryRun in docs/tools.md, docs/installation.md, and CHANGELOG. The PR description is updated separately via gh pr edit. --- CHANGELOG.md | 1 + docs/installation.md | 1 + docs/tools.md | 2 +- src/indexer/index.ts | 4 +++- src/utils/cost.ts | 7 +++++-- tests/cost.test.ts | 2 +- tests/dryrun-index.test.ts | 24 ++++++++++++++++++++++++ 7 files changed, 36 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39355b49..d6660cb0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `--embedding-model` to the cross-repository benchmark runner, preserving `nomic-embed-text` as the default while recording the selected local Ollama model in its artifacts. - Expanded the frozen mixed-intent cross-repository cohort to 100 reviewed queries across JavaScript, Python, Go, Rust, Java, C#, PHP, and Ruby, with additional hard Ruby and C# cases. - Added a deterministic pull-request CI gate that clones each pinned cohort revision and validates every evidence path plus definition symbol without invoking embeddings. +- Added a `dryRun` option to `index_codebase` and an `index --dry-run` CLI flag: a read-only preflight that parses the real file set and reports the exact embedding token total (files, source chunks, tokens) without requesting embeddings or writing to the index. The total serves as a fixed, monotonic denominator for live indexing progress (a force index climbs to ~100%; an incremental index tops out below 100% because cached chunks are counted but not re-embedded) and as a cost preview before committing GPU or time. ### Changed diff --git a/docs/installation.md b/docs/installation.md index 9ab25234..7baebcca 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -189,6 +189,7 @@ opencode-codebase-index-mcp index --project /path/to/repo --estimate-only When using `index`: - `--estimate-only` prints the estimate directly and exits. +- `--dry-run` parses the file set and reports the exact embedding token total (files, source chunks, tokens) without indexing; read-only, no embedding requests. - `--force` bypasses stale-index checks. - `--verbose` includes detailed final index statistics. - `--config` loads and parses that file, then initializes the runtime from it before indexing. diff --git a/docs/tools.md b/docs/tools.md index 8035b39a..6ce8ffa2 100644 --- a/docs/tools.md +++ b/docs/tools.md @@ -118,7 +118,7 @@ Reports readiness, chunk counts, compatibility, current provider/model, and inde ### `index_codebase` -Creates or updates the index. Incremental indexing is the default. Use `force: true` only for a required full rebuild. `estimateOnly` reports estimated embedding work without indexing. +Creates or updates the index. Incremental indexing is the default. Use `force: true` only for a required full rebuild. `estimateOnly` reports estimated embedding work without indexing. `dryRun` parses the real file set and reports the exact embedding token total (files, source chunks, tokens) without requesting embeddings or writing to the index — a read-only preflight. ### `index_health_check` diff --git a/src/indexer/index.ts b/src/indexer/index.ts index 26de43af..acf217b5 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -4079,8 +4079,10 @@ export class Indexer { ); for (const chunk of chunksToProcess) { const texts = createEmbeddingTexts(chunk, parsed.path, maxChunkTokens); + // Count one per source chunk (matches stats.indexedChunks); a chunk may + // split into multiple embedding texts, which are summed into tokensToEmbed. + chunksCount += 1; for (const text of texts) { - chunksCount += 1; tokensToEmbed += estimateTokens(text); } } diff --git a/src/utils/cost.ts b/src/utils/cost.ts index 2cfd60ad..80639e7d 100644 --- a/src/utils/cost.ts +++ b/src/utils/cost.ts @@ -14,7 +14,10 @@ export interface CostEstimate { // Result of a dry-run index_codebase pass: parse the real file set, build the // embedding text for every indexable chunk, and sum estimateTokens over those -// texts without calling the embedding provider or writing to the index. +// texts without requesting embeddings or writing to the index. Provider +// detection may still run as part of normal initialization (for ollama this +// contacts /api/tags and /api/show); no embeddings are requested and no writes +// occur. // // The token sum uses the local estimate (estimateTokens = ceil(len/4)). It // equals the live "Tokens used" counter only for providers that report usage @@ -106,7 +109,7 @@ export function formatCostEstimate(estimate: CostEstimate): string { } export function formatDryRunEstimate(estimate: DryRunEstimate): string { - return `Dry run: parsed the file set to measure the embedding workload. No embeddings were written and the index was not changed. + return `Dry run: parsed the file set to measure the embedding workload. No embedding requests were made and the index was not changed. Files to embed: ${estimate.filesCount.toLocaleString()} Chunks to embed: ${estimate.chunksCount.toLocaleString()} diff --git a/tests/cost.test.ts b/tests/cost.test.ts index 0609fbee..5d0487b3 100644 --- a/tests/cost.test.ts +++ b/tests/cost.test.ts @@ -244,7 +244,7 @@ describe("cost utilities", () => { it("should state the dry-run did not write embeddings or change the index", () => { const formatted = formatDryRunEstimate({ filesCount: 1, chunksCount: 1, tokensToEmbed: 1 }); - expect(formatted).toContain("No embeddings were written"); + expect(formatted).toContain("No embedding requests were made"); expect(formatted).toContain("index was not changed"); // The percent-denominator basis note ci relies on: local estimate basis, // scoped to estimate-based providers (ollama). diff --git a/tests/dryrun-index.test.ts b/tests/dryrun-index.test.ts index 167ef174..71c25ed0 100644 --- a/tests/dryrun-index.test.ts +++ b/tests/dryrun-index.test.ts @@ -128,4 +128,28 @@ describe("indexer dryRunCost", () => { const status = await indexer.getStatus(); expect(status.indexed).toBe(false); }); + + it("counts a source chunk once even when it splits into multiple embedding texts", async () => { + // One exported function whose body is a >6100-char string: a single source + // chunk whose embedding text exceeds nomic-embed-text's maxChunkTokens + // (1536 -> maxContentChars ~6100), so createEmbeddingTexts splits it into + // multiple embedding texts. + fs.writeFileSync( + path.join(tempDir, "src", "huge.ts"), + `export function huge() { return "${"x".repeat(7000)}"; }\n`, + "utf-8", + ); + const indexer = createIndexer(); + const dryRun = await indexer.dryRunCost(); + + const stats = await indexer.forceIndex(); + + // chunksCount counts source chunks (like indexedChunks), not embedding texts, + // so it still matches after a chunk splits. + expect(stats.indexedChunks).toBe(dryRun.chunksCount); + // tokens are summed per embedding text, so the token totals still match. + expect(stats.tokensUsed).toBe(dryRun.tokensToEmbed); + // The oversized chunk split: more embedding requests than source chunks. + expect(embeddingCalls.length).toBeGreaterThan(dryRun.chunksCount); + }); }); \ No newline at end of file