feat(indexing): add dryRun mode to index_codebase for parse-only token totals

CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added `--embedding-model` to the cross-repository benchmark runner, preserving `nomic-embed-text` as the default while recording the selected local Ollama model in its artifacts.
 - Expanded the frozen mixed-intent cross-repository cohort to 100 reviewed queries across JavaScript, Python, Go, Rust, Java, C#, PHP, and Ruby, with additional hard Ruby and C# cases.
 - Added a deterministic pull-request CI gate that clones each pinned cohort revision and validates every evidence path plus definition symbol without invoking embeddings.
+- Added a `dryRun` option to `index_codebase` and an `index --dry-run` CLI flag: a read-only preflight that parses the real file set and reports the exact embedding token total (files, source chunks, tokens) without requesting embeddings or writing to the index. The total serves as a fixed, monotonic denominator for live indexing progress (a force index climbs to ~100%; an incremental index tops out below 100% because cached chunks are counted but not re-embedded) and as a cost preview before committing GPU or time.
 
 ### Changed
 

docs/installation.md

@@ -189,6 +189,7 @@ opencode-codebase-index-mcp index --project /path/to/repo --estimate-only
 When using `index`:
 
 - `--estimate-only` prints the estimate directly and exits.
+- `--dry-run` parses the file set and reports the exact embedding token total (files, source chunks, tokens) without indexing; read-only, no embedding requests.
 - `--force` bypasses stale-index checks.
 - `--verbose` includes detailed final index statistics.
 - `--config` loads and parses that file, then initializes the runtime from it before indexing.

docs/tools.md

@@ -118,7 +118,7 @@ Reports readiness, chunk counts, compatibility, current provider/model, and inde
 
 ### `index_codebase`
 
-Creates or updates the index. Incremental indexing is the default. Use `force: true` only for a required full rebuild. `estimateOnly` reports estimated embedding work without indexing.
+Creates or updates the index. Incremental indexing is the default. Use `force: true` only for a required full rebuild. `estimateOnly` reports estimated embedding work without indexing. `dryRun` parses the real file set and reports the exact embedding token total (files, source chunks, tokens) without requesting embeddings or writing to the index — a read-only preflight.
 
 ### `index_health_check`
 

src/adapters/mcp/cli.ts

@@ -33,6 +33,7 @@ export interface CliIndexArgs {
   config?: string;
   force: boolean;
   estimateOnly: boolean;
+  dryRun: boolean;
   verbose: boolean;
 }
 
@@ -69,6 +70,7 @@ export function parseIndexArgs(argv: string[], cwd: string): CliIndexArgs {
   let config: string | undefined;
   let force = false;
   let estimateOnly = false;
+  let dryRun = false;
   let verbose = false;
 
   for (let i = 0; i < argv.length; i += 1) {
@@ -111,13 +113,16 @@ export function parseIndexArgs(argv: string[], cwd: string): CliIndexArgs {
       continue;
     }
 
-    if (arg === "--force" || arg === "--estimate-only" || arg === "--verbose") {
+    if (arg === "--force" || arg === "--estimate-only" || arg === "--dry-run" || arg === "--verbose") {
       if (arg === "--force") {
         force = true;
       }
       if (arg === "--estimate-only") {
         estimateOnly = true;
       }
+      if (arg === "--dry-run") {
+        dryRun = true;
+      }
       if (arg === "--verbose") {
         verbose = true;
       }
@@ -131,7 +136,7 @@ export function parseIndexArgs(argv: string[], cwd: string): CliIndexArgs {
     throw new Error(`Unknown index option: ${arg}`);
   }
 
-  return { project, host, config, force, estimateOnly, verbose };
+  return { project, host, config, force, estimateOnly, dryRun, verbose };
 }
 
 export function loadCliRawConfig(args: CliArgs): unknown {
@@ -149,6 +154,7 @@ Options:
   --config <path>        Explicit JSON config path
   --force                Rebuild index even if already up to date
   --estimate-only        Estimate indexing cost only
+  --dry-run              Parse only; report the exact embedding token total without indexing
   --verbose              Include detailed final index statistics
   --help                 Show this message
 
@@ -391,6 +397,7 @@ export async function handleIndexCommand(
     const indexArgs: SharedIndexCodebaseArgs = {
       force: parsedArgs.force,
       estimateOnly: parsedArgs.estimateOnly,
+      dryRun: parsedArgs.dryRun,
       verbose: parsedArgs.verbose,
     };
 

src/adapters/mcp/register-tools.ts

@@ -200,6 +200,7 @@ export function registerMcpTools(server: McpServer, runtime: McpServerRuntime):
     {
       force: allowNullAsUndefined(z.boolean().optional().default(false)).describe("Force reindex even if already indexed"),
       estimateOnly: allowNullAsUndefined(z.boolean().optional().default(false)).describe("Only show cost estimate without indexing"),
+      dryRun: allowNullAsUndefined(z.boolean().optional().default(false)).describe("Parse the file set and report the exact embedding token total without indexing. Read-only; the index is not changed. The total is the value 'Tokens used' climbs to for a force index (and an upper bound for an incremental)."),
       verbose: allowNullAsUndefined(z.boolean().optional().default(false)).describe("Show detailed info about skipped files and parsing failures"),
     },
     async (args) => {

src/adapters/opencode/tools.ts

@@ -192,6 +192,7 @@ export const index_codebase: ToolDefinition = tool({
   args: {
     force: z.boolean().optional().default(false).describe("Force reindex even if already indexed"),
     estimateOnly: z.boolean().optional().default(false).describe("Only show cost estimate without indexing"),
+    dryRun: z.boolean().optional().default(false).describe("Parse the file set and report the exact embedding token total without indexing. Read-only; the index is not changed. The total is the value 'Tokens used' climbs to for a force index (and an upper bound for an incremental)."),
     verbose: z.boolean().optional().default(false).describe("Show detailed info about skipped files and parsing failures"),
   },
   async execute(args, context) {

src/adapters/pi/extension.ts

@@ -3,7 +3,7 @@ import { Type } from "typebox";
 
 import { parseConfig } from "../../config/schema.js";
 import { loadMergedConfig } from "../../config/merger.js";
-import { formatCostEstimate } from "../../utils/cost.js";
+import { formatCostEstimate, formatDryRunEstimate } from "../../utils/cost.js";
 import { formatPrImpact } from "../../tools/format-pr-impact.js";
 import { formatCodeCommunities } from "../../tools/format-communities.js";
 import {
@@ -274,12 +274,14 @@ export default function codebaseIndexPiExtension(pi: ExtensionAPI): void {
     parameters: Type.Object({
       force: Type.Optional(Type.Boolean({ default: false })),
       estimateOnly: Type.Optional(Type.Boolean({ default: false })),
+      dryRun: Type.Optional(Type.Boolean({ default: false })),
       verbose: Type.Optional(Type.Boolean({ default: false })),
     }),
     async execute(_toolCallId, params, _signal, _onUpdate, ctx) {
       try {
         const result = await runIndexCodebase(projectRoot(ctx), HOST, params);
         if (result.kind === "estimate") return text(formatCostEstimate(result.estimate), result.estimate);
+        if (result.kind === "dryrun") return text(formatDryRunEstimate(result.dryrun), result.dryrun);
         if (result.kind === "busy") return text(result.text, { code: "INDEX_BUSY" });
         if (result.kind === "message") return text(result.text);
         return text(formatIndexStats(result.stats, params.verbose ?? false), result.stats);

src/indexer/index.ts

@@ -14,7 +14,7 @@ import {
   CustomProviderNonRetryableError,
 } from "../embeddings/provider.js";
 import { collectFiles, SkippedFile } from "../utils/files.js";
-import { createCostEstimate, CostEstimate } from "../utils/cost.js";
+import { createCostEstimate, CostEstimate, DryRunEstimate } from "../utils/cost.js";
 import { Logger, initializeLogger } from "../utils/logger.js";
 import {
   VectorStore,
@@ -4014,6 +4014,84 @@ export class Indexer {
     return createCostEstimate(files, configuredProviderInfo);
   }
 
+  // Dry-run counterpart to index()/forceIndex(): parse the real file set and sum
+  // estimateTokens over the embedding text of every indexable chunk, without
+  // calling the embedding provider or writing to the index. Read-only and
+  // lock-free (mirrors estimateCost). The token sum is the exact value "Tokens
+  // used" climbs to for a force index (cache bypassed); for an incremental it is
+  // an upper bound because cached chunks are counted here but not re-embedded.
+  // Used by index_codebase(dryRun:true) to give a stable, monotonic progress
+  // denominator that matches the live "Tokens used" basis.
+  async dryRunCost(): Promise<DryRunEstimate> {
+    const { configuredProviderInfo } = await this.ensureInitialized();
+    const maxChunkTokens = getSafeEmbeddingChunkTokenLimit(configuredProviderInfo);
+    const includePatterns = [...this.config.include, ...this.config.additionalInclude];
+    const { files } = await collectFiles(
+      this.materializedProjectRoot,
+      includePatterns,
+      this.config.exclude,
+      this.config.indexing.maxFileSize,
+      this.getMaterializedKnowledgeBases(),
+      { maxDepth: this.config.indexing.maxDepth, maxFilesPerDirectory: this.config.indexing.maxFilesPerDirectory },
+    );
+
+    let filesCount = 0;
+    let chunksCount = 0;
+    let tokensToEmbed = 0;
+    // Parse in the same ordered batches as index() (fileBatchLimits) so the
+    // memory profile matches a real run. For each file: read, parse, apply the
+    // same fallback-to-text + maxChunksPerFile cap + selectIndexableChunks path,
+    // then sum estimateTokens over the embedding text (createEmbeddingTexts)
+    // — the identical basis the provider reports as "Tokens used".
+    for (const batch of iterateOrderedFileBatches(files, (f) => f.size, this.fileBatchLimits)) {
+      const loadedFiles = await Promise.all(batch.map(async (f) => {
+        try {
+          return {
+            path: this.toStoredFilePath(f.path),
+            content: await fsPromises.readFile(f.path, "utf-8"),
+          };
+        } catch {
+          // Unreadable file: index() records a parse failure and skips it.
+          return null;
+        }
+      }));
+      const readable = loadedFiles.filter(
+        (f): f is { path: string; content: string } => f !== null,
+      );
+      filesCount += readable.length;
+      const contentByPath = new Map(readable.map((f) => [f.path, f.content]));
+      const parsedFiles = parseFiles(readable, this.config.indexing.linesPerChunk);
+      for (const parsed of parsedFiles) {
+        let chunksToProcess = parsed.chunks;
+        if (
+          this.config.indexing.fallbackToTextOnMaxChunks &&
+          chunksToProcess.length > this.config.indexing.maxChunksPerFile
+        ) {
+          const content = contentByPath.get(parsed.path);
+          if (content !== undefined) {
+            chunksToProcess = parseFileAsText(parsed.path, content, this.config.indexing.linesPerChunk);
+          }
+        }
+        chunksToProcess = selectIndexableChunks(
+          chunksToProcess,
+          this.config.indexing.maxChunksPerFile,
+          this.config.indexing.semanticOnly,
+        );
+        for (const chunk of chunksToProcess) {
+          const texts = createEmbeddingTexts(chunk, parsed.path, maxChunkTokens);
+          // Count one per source chunk (matches stats.indexedChunks); a chunk may
+          // split into multiple embedding texts, which are summed into tokensToEmbed.
+          chunksCount += 1;
+          for (const text of texts) {
+            tokensToEmbed += estimateTokens(text);
+          }
+        }
+      }
+    }
+
+    return { filesCount, chunksCount, tokensToEmbed };
+  }
+
   async index(onProgress?: ProgressCallback): Promise<IndexStats> {
     return this.withIndexMutationLease("index", async (recoveredOwners) => {
       return this.indexUnlocked(onProgress, recoveredOwners);

src/tools/contracts.ts

@@ -63,6 +63,7 @@ export const DEFAULT_CODEBASE_EDIT_CONTEXT_EDGE_LIMIT = 5 as const;
 export interface SharedIndexCodebaseArgs {
   force?: boolean;
   estimateOnly?: boolean;
+  dryRun?: boolean;
   verbose?: boolean;
 }
 

src/tools/execute-common.ts

@@ -21,7 +21,7 @@ import {
   runIndexCodebase,
   runIndexHealthCheck,
 } from "./operations.js";
-import { formatCostEstimate } from "../utils/cost.js";
+import { formatCostEstimate, formatDryRunEstimate } from "../utils/cost.js";
 import { resolveCodebaseEditContext } from "./edit-context.js";
 import { resolveCodebaseContext } from "./context.js";
 import {
@@ -67,6 +67,7 @@ export async function executeIndexCodebase(
 ): Promise<ExecutionResult> {
   const result = await runIndexCodebase(projectRoot, host, args, onProgress);
   if (result.kind === "estimate") return { text: formatCostEstimate(result.estimate) };
+  if (result.kind === "dryrun") return { text: formatDryRunEstimate(result.dryrun) };
   if (result.kind === "busy") return { text: result.text, isError: true };
   if (result.kind === "message") return { text: result.text };
   return { text: formatIndexStats(result.stats, args.verbose ?? false) };

src/tools/operations.ts

@@ -20,7 +20,7 @@ import type { SharedCodeCommunitiesArgs } from "./contracts.js";
 import { calculatePercentage, formatProgressTitle, formatStatus } from "./utils.js";
 import type { LogLevel } from "../config/schema.js";
 import type { LogEntry } from "../utils/logger.js";
-import type { CostEstimate } from "../utils/cost.js";
+import type { CostEstimate, DryRunEstimate } from "../utils/cost.js";
 import type { AutoIndexStatusSnapshot } from "../utils/auto-index.js";
 import type { SearchTrace } from "../indexer/index.js";
 import {
@@ -425,10 +425,11 @@ export async function getCallGraphPath(
 export async function runIndexCodebase(
   projectRoot: string | undefined,
   host: HostMode,
-  args: { force?: boolean; estimateOnly?: boolean; verbose?: boolean },
+  args: { force?: boolean; estimateOnly?: boolean; dryRun?: boolean; verbose?: boolean },
   onProgress?: ProgressCb,
 ): Promise<
   | { kind: "estimate"; estimate: CostEstimate }
+  | { kind: "dryrun"; dryrun: DryRunEstimate }
   | { kind: "stats"; stats: IndexStats }
   | IndexMessageResult
   | IndexBusyResult
@@ -441,6 +442,10 @@ export async function runIndexCodebase(
       return { kind: "estimate", estimate: await indexer.estimateCost() };
     }
 
+    if (args.dryRun) {
+      return { kind: "dryrun", dryrun: await indexer.dryRunCost() };
+    }
+
     const coordinated = runCoordinatedIndex(root, host, args.force ?? false, (progress) => {
       if (onProgress) {
         void onProgress(formatProgressTitle(progress), {

src/utils/cost.ts

@@ -12,6 +12,29 @@ export interface CostEstimate {
   isFree: boolean;
 }
 
+// Result of a dry-run index_codebase pass: parse the real file set, build the
+// embedding text for every indexable chunk, and sum estimateTokens over those
+// texts without requesting embeddings or writing to the index. Provider
+// detection may still run as part of normal initialization (for ollama this
+// contacts /api/tags and /api/show); no embeddings are requested and no writes
+// occur.
+//
+// The token sum uses the local estimate (estimateTokens = ceil(len/4)). It
+// equals the live "Tokens used" counter only for providers that report usage
+// on the same basis (ollama counts ceil(len/4)); for providers that report a
+// server tokenizer count (OpenAI, Gemini, custom) it is only an estimate.
+//
+// For a matching provider and a project-scoped force index, the force pass
+// clears its own cached embeddings, so the live counter climbs to this sum. A
+// force index on a shared global index can reuse cached embeddings from other
+// projects, and an incremental index counts cached chunks that are not
+// re-embedded; in both cases the dry-run value is an upper bound.
+export interface DryRunEstimate {
+  filesCount: number;
+  chunksCount: number;
+  tokensToEmbed: number;
+}
+
 export function estimateTokens(text: string): number {
   return Math.ceil(text.length / 4);
 }
@@ -85,6 +108,27 @@ export function formatCostEstimate(estimate: CostEstimate): string {
 `;
 }
 
+export function formatDryRunEstimate(estimate: DryRunEstimate): string {
+  return `Dry run: parsed the file set to measure the embedding workload. No embedding requests were made and the index was not changed.
+
+  Files to embed:   ${estimate.filesCount.toLocaleString()}
+  Chunks to embed:  ${estimate.chunksCount.toLocaleString()}
+  Tokens to embed:  ${estimate.tokensToEmbed.toLocaleString()}
+
+The "Tokens to embed" value uses the local estimateTokens(text) = ceil(len/4). It
+matches the live "Tokens used" counter only for providers that report usage on the
+same basis (ollama); for providers that report a server tokenizer count (OpenAI,
+Gemini, custom) it is only an estimate.
+
+For a matching provider and a project-scoped force index, the force pass clears its
+own cached embeddings, so the live counter climbs to this number. A force index on a
+shared global index can reuse cached embeddings from other projects, and an
+incremental index counts cached chunks that are not re-embedded; in both cases this
+number is an upper bound on the live counter, so a progress percent against this
+total tops out below 100%.
+`;
+}
+
 export function formatBytes(bytes: number): string {
   if (bytes === 0) return "0 B";
   const k = 1024;

tests/cost.test.ts

@@ -5,9 +5,11 @@ import {
   estimateChunksFromFiles,
   estimateCost,
   formatCostEstimate,
+  formatDryRunEstimate,
   parseConfirmationResponse,
   formatConfirmationPrompt,
   CostEstimate,
+  DryRunEstimate,
 } from "../src/utils/cost.js";
 
 describe("cost utilities", () => {
@@ -224,6 +226,40 @@ describe("cost utilities", () => {
     });
   });
 
+  describe("formatDryRunEstimate", () => {
+    it("should format the file/chunk/token totals with locale grouping", () => {
+      const estimate: DryRunEstimate = {
+        filesCount: 970,
+        chunksCount: 72488,
+        tokensToEmbed: 76509389,
+      };
+
+      const formatted = formatDryRunEstimate(estimate);
+
+      expect(formatted).toContain("Files to embed:   970");
+      expect(formatted).toContain("Chunks to embed:  72,488");
+      expect(formatted).toContain("Tokens to embed:  76,509,389");
+    });
+
+    it("should state the dry-run did not write embeddings or change the index", () => {
+      const formatted = formatDryRunEstimate({ filesCount: 1, chunksCount: 1, tokensToEmbed: 1 });
+
+      expect(formatted).toContain("No embedding requests were made");
+      expect(formatted).toContain("index was not changed");
+      // The percent-denominator basis note ci relies on: local estimate basis,
+      // scoped to estimate-based providers (ollama).
+      expect(formatted).toContain("estimateTokens(text)");
+      expect(formatted).toContain("ollama");
+    });
+
+    it("should distinguish force-index exactness from incremental upper bound", () => {
+      const formatted = formatDryRunEstimate({ filesCount: 0, chunksCount: 0, tokensToEmbed: 0 });
+
+      expect(formatted).toContain("force index");
+      expect(formatted).toContain("upper bound");
+    });
+  });
+
   describe("formatConfirmationPrompt", () => {
     it("should return prompt with all options", () => {
       const prompt = formatConfirmationPrompt();