From 171439854d5ee394805b14181925d199c0a4590e Mon Sep 17 00:00:00 2001 From: Dmitri Khokhlov Date: Mon, 17 Aug 2026 00:33:49 -0700 Subject: [PATCH 1/5] feat(indexing): make lines_per_chunk configurable for line-based chunking The line-based chunking path (chunk_by_lines) hardcodes a 30-line window with a fixed 3-line overlap. This path is used for .jsonl, .txt, unknown extensions, and the AST fallback. For line-delimited files where one line is one record (e.g. Claude .jsonl session transcripts), a 30-line window yields coarse retrieval (a hit returns a 30-record window) and dilutes the embedding signal via mean-pooling over ~2500 tokens. Expose the window size as `indexing.linesPerChunk` (default 30, so existing behavior is unchanged). Thread it through the napi entry points (parse_file / parse_file_as_text / parse_files) into chunk_by_lines as an Option (undefined -> None -> 30), keeping the TS wrappers' signatures optional so all existing callers compile unchanged. Auto-cap the overlap at one quarter of the window (overlap = min(OVERLAP_LINES, lines_per_chunk / 4)) so smaller opt-in windows shrink the overlap too, preventing near-duplicate chunks. At the default 30 the overlap is min(3, 7) = 3 -- bit-identical to before. Only the line-based path is affected; AST-parsed languages (extract_chunks) are unchanged regardless of the knob. Tests: - Rust: chunk_by_lines default-unchanged, custom size (5 -> step 4, overlap 1), and size-1 (no overlap, one line per chunk). - TS: parseFile/parseFiles honor linesPerChunk on .jsonl/.txt. - Config: default 30, explicit override honored, non-number/<1 coerced. --- docs/configuration.md | 1 + native/src/lib.rs | 22 ++++-- native/src/parser.rs | 174 ++++++++++++++++++++++++++++++----------- src/config/defaults.ts | 1 + src/config/schema.ts | 11 +++ src/indexer/index.ts | 4 +- src/native/parsing.ts | 12 +-- tests/config.test.ts | 10 +++ tests/native.test.ts | 37 +++++++++ 9 files changed, 213 insertions(+), 59 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 9989bcb..aa5f0c4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -169,6 +169,7 @@ Changing provider, model, dimensions, or embedding strategy can make an existing | `maxDepth` | `5` | Directory traversal depth; `-1` is unlimited | | `maxFilesPerDirectory` | `100` | Per-directory file cap | | `fallbackToTextOnMaxChunks` | `true` | Fall back to line chunks when the semantic cap is reached | +| `linesPerChunk` | `30` | Max lines per chunk for line-based parsing (`.jsonl`, `.txt`, unknown extensions, and the AST fallback). Lower it for finer-grained retrieval on line-delimited files. Only the line-based path is affected; AST-parsed languages are unchanged | | `gitBlame.enabled` | `false` | Store git blame metadata for filtering | Example: diff --git a/native/src/lib.rs b/native/src/lib.rs index c0e8e13..db1bcde 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -24,19 +24,29 @@ pub use store::*; pub use types::*; #[napi] -pub fn parse_file(file_path: String, content: String) -> Result> { - parser::parse_file_internal(&file_path, &content).map_err(|e| Error::from_reason(e.to_string())) +pub fn parse_file( + file_path: String, + content: String, + lines_per_chunk: Option, +) -> Result> { + parser::parse_file_internal(&file_path, &content, lines_per_chunk.unwrap_or(30) as usize) + .map_err(|e| Error::from_reason(e.to_string())) } #[napi] -pub fn parse_file_as_text(file_path: String, content: String) -> Result> { - parser::parse_file_as_text_internal(&file_path, &content) +pub fn parse_file_as_text( + file_path: String, + content: String, + lines_per_chunk: Option, +) -> Result> { + parser::parse_file_as_text_internal(&file_path, &content, lines_per_chunk.unwrap_or(30) as usize) .map_err(|e| Error::from_reason(e.to_string())) } #[napi] -pub fn parse_files(files: Vec) -> Result> { - parser::parse_files_parallel(files).map_err(|e| Error::from_reason(e.to_string())) +pub fn parse_files(files: Vec, lines_per_chunk: Option) -> Result> { + parser::parse_files_parallel(files, lines_per_chunk.unwrap_or(30) as usize) + .map_err(|e| Error::from_reason(e.to_string())) } #[napi] diff --git a/native/src/parser.rs b/native/src/parser.rs index 4b3d06d..cb2aefd 100644 --- a/native/src/parser.rs +++ b/native/src/parser.rs @@ -14,7 +14,11 @@ const MAX_CHUNK_SIZE: usize = 2000; const TARGET_CHUNK_SIZE: usize = 500; const OVERLAP_LINES: usize = 3; -pub fn parse_file_internal(file_path: &str, content: &str) -> Result> { +pub fn parse_file_internal( + file_path: &str, + content: &str, + lines_per_chunk: usize, +) -> Result> { let ext = Path::new(file_path) .extension() .and_then(|e| e.to_str()) @@ -23,7 +27,7 @@ pub fn parse_file_internal(file_path: &str, content: &str) -> Result Result tree_sitter_gdscript::LANGUAGE.into(), Language::Matlab => tree_sitter_matlab::LANGUAGE.into(), Language::Apex => tree_sitter_sfapex::apex::LANGUAGE.into(), - _ => return Ok(chunk_by_lines(content, &language)), + _ => return Ok(chunk_by_lines(content, &language, lines_per_chunk)), }; parser.set_language(&ts_language)?; @@ -61,25 +65,33 @@ pub fn parse_file_internal(file_path: &str, content: &str) -> Result Result> { +pub fn parse_file_as_text_internal( + file_path: &str, + content: &str, + lines_per_chunk: usize, +) -> Result> { let ext = Path::new(file_path) .extension() .and_then(|e| e.to_str()) .unwrap_or(""); let language = Language::from_extension(ext); - Ok(chunk_by_lines(content, &language)) + Ok(chunk_by_lines(content, &language, lines_per_chunk)) } -pub fn parse_files_parallel(files: Vec) -> Result> { +pub fn parse_files_parallel( + files: Vec, + lines_per_chunk: usize, +) -> Result> { let results: Vec = files .par_iter() .filter_map(|file| { let (chunks, symbols) = - parse_file_with_symbols_internal(&file.path, &file.content).ok()?; + parse_file_with_symbols_internal(&file.path, &file.content, lines_per_chunk) + .ok()?; let hash = crate::hasher::xxhash_content(&file.content); Some(ParsedFile { path: file.path.clone(), @@ -96,6 +108,7 @@ pub fn parse_files_parallel(files: Vec) -> Result> { fn parse_file_with_symbols_internal( file_path: &str, content: &str, + lines_per_chunk: usize, ) -> Result<(Vec, Vec)> { let ext = Path::new(file_path) .extension() @@ -104,7 +117,7 @@ fn parse_file_with_symbols_internal( let language = Language::from_extension(ext); if language == Language::Text { - return Ok((chunk_by_lines(content, &language), Vec::new())); + return Ok((chunk_by_lines(content, &language, lines_per_chunk), Vec::new())); } let mut parser = Parser::new(); @@ -132,14 +145,14 @@ fn parse_file_with_symbols_internal( Language::Gdscript => tree_sitter_gdscript::LANGUAGE.into(), Language::Matlab => tree_sitter_matlab::LANGUAGE.into(), Language::Apex => tree_sitter_sfapex::apex::LANGUAGE.into(), - _ => return Ok((chunk_by_lines(content, &language), Vec::new())), + _ => return Ok((chunk_by_lines(content, &language, lines_per_chunk), Vec::new())), }; parser.set_language(&ts_language)?; let tree = parser .parse(content, None) .ok_or_else(|| anyhow!("Failed to parse file: {}", file_path))?; - let chunks = extract_chunks(&tree, content, &language)?; + let chunks = extract_chunks(&tree, content, &language, lines_per_chunk)?; let symbols = extract_symbols(&tree, content, &language); Ok((chunks, symbols)) } @@ -187,7 +200,12 @@ fn extract_symbol_nodes( } } -fn extract_chunks(tree: &Tree, source: &str, language: &Language) -> Result> { +fn extract_chunks( + tree: &Tree, + source: &str, + language: &Language, + lines_per_chunk: usize, +) -> Result> { let mut chunks = Vec::new(); let root = tree.root_node(); let mut cursor = root.walk(); @@ -195,7 +213,7 @@ fn extract_chunks(tree: &Tree, source: &str, language: &Language) -> Result) { *chunks = merged; } -fn chunk_by_lines(content: &str, language: &Language) -> Vec { +fn chunk_by_lines(content: &str, language: &Language, lines_per_chunk: usize) -> Vec { let lines: Vec<&str> = content.lines().collect(); let total_lines = lines.len(); @@ -1162,12 +1180,12 @@ fn chunk_by_lines(content: &str, language: &Language) -> Vec { return Vec::new(); } - let lines_per_chunk = 30; - let step_size = if lines_per_chunk > OVERLAP_LINES { - lines_per_chunk - OVERLAP_LINES - } else { - lines_per_chunk - }; + // Cap the overlap so it stays at or below ~25% of the window. At the default + // lines_per_chunk (30) the overlap is min(3, 7) = 3, which is bit-identical to + // the previous hardcoded behavior. Smaller opt-in windows shrink the overlap so + // the chunks do not collapse into near-duplicates. + let overlap = OVERLAP_LINES.min(lines_per_chunk / 4); + let step_size = lines_per_chunk.saturating_sub(overlap); let mut chunks = Vec::new(); let mut start = 0; @@ -1389,7 +1407,7 @@ class Greeter { } "#; - let chunks = parse_file_internal("test.ts", content).unwrap(); + let chunks = parse_file_internal("test.ts", content, 30).unwrap(); assert!(!chunks.is_empty()); } @@ -1404,7 +1422,7 @@ class Greeter { const values = [1, 2].map(item => item * 2); "#; - let (_chunks, symbols) = parse_file_with_symbols_internal("arrows.ts", content) + let (_chunks, symbols) = parse_file_with_symbols_internal("arrows.ts", content, 30) .expect("should parse TypeScript arrow functions"); let arrow_names: Vec = symbols @@ -1434,7 +1452,7 @@ class Greeter { } "#; - let (_chunks, symbols) = parse_file_with_symbols_internal("animal.ts", content) + let (_chunks, symbols) = parse_file_with_symbols_internal("animal.ts", content, 30) .expect("should parse exported abstract class"); assert!(symbols.iter().any(|symbol| { @@ -1465,7 +1483,7 @@ class Greeter: return f"Hello, {self.name}!" "#; - let chunks = parse_file_internal("test.py", content).unwrap(); + let chunks = parse_file_internal("test.py", content, 30).unwrap(); assert!(!chunks.is_empty()); } @@ -1479,7 +1497,7 @@ class Greeter: #[test] fn test_parse_php_8_semantic_chunks_have_names() { let content = include_str!("../../tests/fixtures/call-graph/php-8-features.php"); - let chunks = parse_file_internal("php-8-features.php", content).unwrap(); + let chunks = parse_file_internal("php-8-features.php", content, 30).unwrap(); assert!(chunks.iter().any(|chunk| { chunk.chunk_type == "class_declaration" && chunk.name.as_deref() == Some("Job") @@ -1506,7 +1524,7 @@ class Greeter: .collect(); let content = lines.join("\n"); - let chunks = chunk_by_lines(&content, &Language::Text); + let chunks = chunk_by_lines(&content, &Language::Text, 30); assert!(chunks.len() >= 2, "Should have multiple chunks"); @@ -1522,6 +1540,72 @@ class Greeter: } } + #[test] + fn test_chunk_by_lines_default_unchanged() { + // The default window (30) must keep the historical 30-line chunks and 3-line + // overlap (step 27), so existing behavior is bit-identical when the knob is + // left at its default. + let lines: Vec = (0..100) + .map(|i| format!("line {} content here", i)) + .collect(); + let content = lines.join("\n"); + + let chunks = chunk_by_lines(&content, &Language::Text, 30); + + assert!(!chunks.is_empty(), "Should produce chunks"); + let first_span = chunks[0].end_line - chunks[0].start_line + 1; + assert_eq!(first_span, 30, "First chunk should span 30 lines"); + if chunks.len() >= 2 { + let step = chunks[1].start_line - chunks[0].start_line; + assert_eq!(step, 27, "Default step should be 27 (30 - overlap 3)"); + } + } + + #[test] + fn test_chunk_by_lines_custom_size() { + // A smaller opt-in window (5) must shrink both the chunk size and the + // overlap. overlap = min(3, 5/4 = 1) = 1, so step = 5 - 1 = 4. + let lines: Vec = (0..40) + .map(|i| format!("line {} content here", i)) + .collect(); + let content = lines.join("\n"); + + let chunks = chunk_by_lines(&content, &Language::Text, 5); + + assert!(chunks.len() >= 2, "Should produce multiple small chunks"); + for chunk in &chunks { + let span = chunk.end_line - chunk.start_line + 1; + assert!( + span <= 5, + "Each chunk should span at most 5 lines, got {}", + span + ); + } + if chunks.len() >= 2 { + let step = chunks[1].start_line - chunks[0].start_line; + assert_eq!(step, 4, "Step should be 4 (window 5 - overlap 1)"); + // Overlap of 1 line: chunk 1 starts where chunk 0 is still in range. + assert!( + chunks[1].start_line <= chunks[0].end_line, + "Chunks should overlap by 1 line" + ); + } + } + + #[test] + fn test_chunk_by_lines_size_one_no_overlap() { + // A window of 1 line must not overlap itself (overlap = min(3, 0) = 0), + // producing one chunk per non-empty line. + let content = "a\nb\nc"; + let chunks = chunk_by_lines(content, &Language::Text, 1); + + assert_eq!(chunks.len(), 3, "One line per chunk"); + for chunk in &chunks { + let span = chunk.end_line - chunk.start_line + 1; + assert_eq!(span, 1, "Each chunk should span exactly 1 line"); + } + } + #[test] fn test_jsdoc_extraction() { let content = r#" @@ -1535,7 +1619,7 @@ function validateEmail(email: string): boolean { } "#; - let chunks = parse_file_internal("test.ts", content).unwrap(); + let chunks = parse_file_internal("test.ts", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have at least one chunk"); let chunk = &chunks[0]; @@ -1561,7 +1645,7 @@ fn factorial(n: u64) -> Option { } "#; - let chunks = parse_file_internal("test.rs", content).unwrap(); + let chunks = parse_file_internal("test.rs", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have at least one chunk"); let chunk = &chunks[0]; @@ -1596,7 +1680,7 @@ public enum Operation { } "#; - let chunks = parse_file_internal("Calculator.java", content).unwrap(); + let chunks = parse_file_internal("Calculator.java", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for Java"); let has_class = chunks.iter().any(|c| c.chunk_type == "class_declaration"); @@ -1633,7 +1717,7 @@ public struct Point } "#; - let chunks = parse_file_internal("Person.cs", content).unwrap(); + let chunks = parse_file_internal("Person.cs", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for C#"); } @@ -1661,7 +1745,7 @@ module Utils end "#; - let chunks = parse_file_internal("greeter.rb", content).unwrap(); + let chunks = parse_file_internal("greeter.rb", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for Ruby"); let has_class = chunks.iter().any(|c| c.chunk_type == "class"); @@ -1682,7 +1766,7 @@ module Rack end "#; - let (_, symbols) = parse_file_with_symbols_internal("rack/protection.rb", content) + let (_, symbols) = parse_file_with_symbols_internal("rack/protection.rb", content, 30) .expect("should parse nested Ruby modules"); assert!( symbols @@ -1691,7 +1775,7 @@ end "Expected nested Ruby class symbol named ContentSecurityPolicy" ); - let chunks = parse_file_internal("rack/protection.rb", content) + let chunks = parse_file_internal("rack/protection.rb", content, 30) .expect("should produce semantic chunks for nested Ruby class"); assert!( chunks.iter().any(|chunk| { @@ -1721,7 +1805,7 @@ function add() { greet "World" "#; - let chunks = parse_file_internal("script.sh", content).unwrap(); + let chunks = parse_file_internal("script.sh", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for Bash"); let has_function = chunks.iter().any(|c| c.chunk_type == "function_definition"); @@ -1762,7 +1846,7 @@ void greet(const char* name) { } "#; - let chunks = parse_file_internal("main.c", content).unwrap(); + let chunks = parse_file_internal("main.c", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for C"); let has_function = chunks.iter().any(|c| c.chunk_type == "function_definition"); @@ -1808,7 +1892,7 @@ int main() { } "#; - let chunks = parse_file_internal("main.cpp", content).unwrap(); + let chunks = parse_file_internal("main.cpp", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for C++"); let has_class = chunks.iter().any(|c| c.chunk_type == "class_specifier"); @@ -1829,7 +1913,7 @@ int main() { #[test] fn test_parse_cpp_preserves_small_type_symbols() { - let chunks = parse_file_internal("small.cpp", "class Tag {};\nstruct Point {};\n").unwrap(); + let chunks = parse_file_internal("small.cpp", "class Tag {};\nstruct Point {};\n", 30).unwrap(); assert!(chunks.iter().any(|chunk| { chunk.chunk_type == "class_specifier" && chunk.name.as_deref() == Some("Tag") @@ -1847,7 +1931,7 @@ inline T scaled_value(T value, constant float& scale) { } "#; - let chunks = parse_file_internal("shader.metal", content).unwrap(); + let chunks = parse_file_internal("shader.metal", content, 30).unwrap(); let function = chunks .iter() .find(|chunk| chunk.name.as_deref() == Some("scaled_value")) @@ -1884,7 +1968,7 @@ lto = true opt-level = 3 "#; - let chunks = parse_file_internal("Cargo.toml", content).unwrap(); + let chunks = parse_file_internal("Cargo.toml", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for TOML"); let has_table = chunks.iter().any(|c| c.chunk_type == "table"); @@ -1922,7 +2006,7 @@ spec: - containerPort: 8080 "#; - let chunks = parse_file_internal("deployment.yaml", content).unwrap(); + let chunks = parse_file_internal("deployment.yaml", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for YAML"); } @@ -1953,7 +2037,7 @@ myFunction(); Please read CONTRIBUTING.md for details. "#; - let chunks = parse_file_internal("README.md", content).unwrap(); + let chunks = parse_file_internal("README.md", content, 30).unwrap(); // Markdown falls back to line-based chunking assert!(!chunks.is_empty(), "Should have chunks for Markdown"); // Should be block type since we use line-based chunking @@ -1979,7 +2063,7 @@ public with sharing class AccountService { } "#; - let chunks = parse_file_internal("AccountService.cls", content).unwrap(); + let chunks = parse_file_internal("AccountService.cls", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for Apex"); let has_class = chunks.iter().any(|c| c.chunk_type == "class_declaration"); @@ -1996,7 +2080,7 @@ trigger AccountTrigger on Account (before insert, before update, after delete) { } "#; - let chunks = parse_file_internal("AccountTrigger.trigger", content).unwrap(); + let chunks = parse_file_internal("AccountTrigger.trigger", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for Apex trigger"); let has_trigger = chunks.iter().any(|c| c.chunk_type == "trigger_declaration"); @@ -2029,7 +2113,7 @@ class InnerThing: pass "#; - let chunks = parse_file_internal("player.gd", content).unwrap(); + let chunks = parse_file_internal("player.gd", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for GDScript"); let chunk_types: Vec<&str> = chunks.iter().map(|c| c.chunk_type.as_str()).collect(); @@ -2071,7 +2155,7 @@ public class AccountService { } "#; - let chunks = parse_file_internal("AccountService.cls", content).unwrap(); + let chunks = parse_file_internal("AccountService.cls", content, 30).unwrap(); let class_chunk = chunks.iter().find(|c| c.chunk_type == "class_declaration"); assert!(class_chunk.is_some(), "Should find class_declaration"); assert!( diff --git a/src/config/defaults.ts b/src/config/defaults.ts index c096a4c..96d9e37 100644 --- a/src/config/defaults.ts +++ b/src/config/defaults.ts @@ -25,6 +25,7 @@ export function getDefaultIndexingConfig(): IndexingConfig { maxDepth: 5, maxFilesPerDirectory: 100, fallbackToTextOnMaxChunks: true, + linesPerChunk: 30, gitBlame: { enabled: false }, }; } diff --git a/src/config/schema.ts b/src/config/schema.ts index e09d52f..f76f688 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -65,6 +65,16 @@ export interface IndexingConfig { * instead of skipping the rest of the file. Default: true */ fallbackToTextOnMaxChunks: boolean; + /** + * Max lines per chunk for line-based parsing (.jsonl, .txt, unknown extensions, + * and the AST fallback path). Each chunk is a sliding window of this many lines. + * Default: 30. Lower it for finer-grained retrieval on line-delimited files (for + * example Claude .jsonl session transcripts, where one line is one message). The + * overlap between neighboring chunks is auto-capped at one quarter of this value, + * so smaller windows shrink the overlap too. Only the line-based path is affected; + * AST-parsed languages are unchanged. + */ + linesPerChunk: number; gitBlame: { enabled: boolean; }; @@ -218,6 +228,7 @@ export function parseConfig(raw: unknown): ParsedCodebaseIndexConfig { maxDepth: typeof rawIndexing.maxDepth === "number" ? (rawIndexing.maxDepth < -1 ? -1 : rawIndexing.maxDepth) : defaultIndexing.maxDepth, maxFilesPerDirectory: typeof rawIndexing.maxFilesPerDirectory === "number" ? Math.max(1, rawIndexing.maxFilesPerDirectory) : defaultIndexing.maxFilesPerDirectory, fallbackToTextOnMaxChunks: typeof rawIndexing.fallbackToTextOnMaxChunks === "boolean" ? rawIndexing.fallbackToTextOnMaxChunks : defaultIndexing.fallbackToTextOnMaxChunks, + linesPerChunk: typeof rawIndexing.linesPerChunk === "number" ? Math.max(1, Math.floor(rawIndexing.linesPerChunk)) : defaultIndexing.linesPerChunk, gitBlame: { enabled: rawIndexing.gitBlame && typeof rawIndexing.gitBlame === "object" && typeof (rawIndexing.gitBlame as Record).enabled === "boolean" ? (rawIndexing.gitBlame as { enabled: boolean }).enabled diff --git a/src/indexer/index.ts b/src/indexer/index.ts index 1f165be..e6e64ef 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -4346,7 +4346,7 @@ export class Indexer { const loadedByPath = new Map(loadedFiles.map((file) => [file.path, file])); const descriptorByPath = new Map(descriptorBatch.map((descriptor) => [descriptor.storedPath, descriptor])); const parseStartTime = performance.now(); - const parsedFiles = parseFiles(loadedFiles); + const parsedFiles = parseFiles(loadedFiles, this.config.indexing.linesPerChunk); const parseMs = performance.now() - parseStartTime; this.logger.recordFilesParsed(parsedFiles.length); this.logger.recordParseDuration(parseMs); @@ -4378,7 +4378,7 @@ export class Indexer { this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile ) { - chunksToProcess = parseFileAsText(parsed.path, loadedFile.content); + chunksToProcess = parseFileAsText(parsed.path, loadedFile.content, this.config.indexing.linesPerChunk); } chunksToProcess = selectIndexableChunks( chunksToProcess, diff --git a/src/native/parsing.ts b/src/native/parsing.ts index fdcc4b3..2c39391 100644 --- a/src/native/parsing.ts +++ b/src/native/parsing.ts @@ -1,18 +1,18 @@ import type { CallSiteData, CodeChunk, FileInput, ParsedFile, ParsedSymbol, ChunkType } from "./types.js"; import { native } from "./binding.js"; -export function parseFile(filePath: string, content: string): CodeChunk[] { - const result = native.parseFile(filePath, content); +export function parseFile(filePath: string, content: string, linesPerChunk?: number): CodeChunk[] { + const result = native.parseFile(filePath, content, linesPerChunk); return result.map(mapChunk); } -export function parseFileAsText(filePath: string, content: string): CodeChunk[] { - const result = native.parseFileAsText(filePath, content); +export function parseFileAsText(filePath: string, content: string, linesPerChunk?: number): CodeChunk[] { + const result = native.parseFileAsText(filePath, content, linesPerChunk); return result.map(mapChunk); } -export function parseFiles(files: FileInput[]): ParsedFile[] { - const result = native.parseFiles(files); +export function parseFiles(files: FileInput[], linesPerChunk?: number): ParsedFile[] { + const result = native.parseFiles(files, linesPerChunk); return result.map((f: any) => ({ path: f.path, chunks: f.chunks.map(mapChunk), diff --git a/tests/config.test.ts b/tests/config.test.ts index fb24394..279b5d8 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -264,6 +264,16 @@ describe("config schema", () => { expect(parseConfig({ indexing: { maxChunksPerFile: -5 } }).indexing.maxChunksPerFile).toBe(1); }); + it("should default linesPerChunk to 30 and honor explicit overrides", () => { + expect(parseConfig({}).indexing.linesPerChunk).toBe(30); + expect(parseConfig({ indexing: { linesPerChunk: 10 } }).indexing.linesPerChunk).toBe(10); + // Non-number falls back to the default. + expect(parseConfig({ indexing: { linesPerChunk: "small" } }).indexing.linesPerChunk).toBe(30); + // Values below 1 are clamped to 1, and floats are floored. + expect(parseConfig({ indexing: { linesPerChunk: 0 } }).indexing.linesPerChunk).toBe(1); + expect(parseConfig({ indexing: { linesPerChunk: 7.9 } }).indexing.linesPerChunk).toBe(7); + }); + it("should enforce minimum of 1 for gcIntervalDays", () => { expect(parseConfig({ indexing: { gcIntervalDays: 0 } }).indexing.gcIntervalDays).toBe(1); expect(parseConfig({ indexing: { gcIntervalDays: -1 } }).indexing.gcIntervalDays).toBe(1); diff --git a/tests/native.test.ts b/tests/native.test.ts index 99c3c4e..f428ad1 100644 --- a/tests/native.test.ts +++ b/tests/native.test.ts @@ -133,6 +133,25 @@ const add = (a, b) => a + b; expect(chunks[0]?.chunkType).toBe("block"); }); + it("honors linesPerChunk for line-based (.jsonl) files", () => { + const lines = Array.from({ length: 20 }, (_, i) => `{"i":${i}}`); + const content = lines.join("\n"); + + const defaultChunks = parseFile("session.jsonl", content, 30); + const smallChunks = parseFile("session.jsonl", content, 5); + + // Default window covers all 20 lines in one chunk. + expect(defaultChunks.length).toBe(1); + expect(defaultChunks[0].endLine - defaultChunks[0].startLine + 1).toBe(20); + + // Window of 5 with overlap 1 (min(3, 5/4)) -> step 4 -> chunks at starts 1, 5, 9, 13, 17. + expect(smallChunks.length).toBe(5); + for (const chunk of smallChunks) { + expect(chunk.endLine - chunk.startLine + 1).toBeLessThanOrEqual(5); + } + expect(smallChunks[1].startLine - smallChunks[0].startLine).toBe(4); + }); + it("should parse PHP files", () => { const content = ` item * 2); ]), ); }); + + it("honors linesPerChunk across a batch of line-based files", () => { + const lines = Array.from({ length: 12 }, (_, i) => `line ${i}`); + const content = lines.join("\n"); + const files = [ + { path: "a.txt", content }, + { path: "b.jsonl", content }, + ]; + + const [a, b] = parseFiles(files, 4); + + // Window of 4 with overlap 1 (min(3, 4/4)) -> step 3 -> starts 1, 4, 7, 10. + expect(a.chunks.length).toBe(4); + expect(b.chunks.length).toBe(4); + for (const chunk of a.chunks) { + expect(chunk.endLine - chunk.startLine + 1).toBeLessThanOrEqual(4); + } + }); }); describe("hashContent", () => { From ac5ccd806fd23f2cc9f15e7bbf71f8b019473389 Mon Sep 17 00:00:00 2001 From: Dmitri Khokhlov Date: Mon, 17 Aug 2026 00:48:21 -0700 Subject: [PATCH 2/5] fix(indexing): clamp lines_per_chunk >= 1 and reject non-finite config Code review (codex + claude) found that chunk_by_lines hangs when lines_per_chunk == 0: overlap = min(3, 0) = 0 and step_size = 0, so the window loop never advances. The config layer already clamps to >= 1, but chunk_by_lines is a pub fn reachable through the napi API with an explicit 0 (native.parseFile(path, content, 0) -> Some(0) survives unwrap_or(30)), and a direct caller has no protection. Codex reproduced the hang with a bounded probe. Clamp at the layer that owns the invariant: lines_per_chunk.max(1) at the top of chunk_by_lines. A window of 0 now behaves like 1 (one line per chunk) instead of hanging. This also covers the AST paths that forward the value straight through. The config coercion let NaN and Infinity through the typeof "number" gate (Math.floor(NaN) = NaN, Math.floor(Infinity) = Infinity); at the napi boundary V8 ToUint32 maps both to 0, feeding the same hang. Tighten the guard to also require Number.isFinite. JSON cannot express these, so file configs were already safe; this protects programmatic parseConfig callers. Tests: - Rust: test_chunk_by_lines_zero_clamped_to_one (0 -> 3 one-line chunks, terminates). - Config: NaN and Infinity fall back to the default 30. --- native/src/parser.rs | 18 ++++++++++++++++++ src/config/schema.ts | 2 +- tests/config.test.ts | 4 ++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/native/src/parser.rs b/native/src/parser.rs index cb2aefd..d926e59 100644 --- a/native/src/parser.rs +++ b/native/src/parser.rs @@ -1180,6 +1180,11 @@ fn chunk_by_lines(content: &str, language: &Language, lines_per_chunk: usize) -> return Vec::new(); } + // Defense in depth: the config layer clamps linesPerChunk to >= 1, but this pub fn is + // also reachable through the napi API with an explicit 0, which would make step_size 0 + // and hang the loop below. Treat anything below 1 as 1 (one line per chunk). + let lines_per_chunk = lines_per_chunk.max(1); + // Cap the overlap so it stays at or below ~25% of the window. At the default // lines_per_chunk (30) the overlap is min(3, 7) = 3, which is bit-identical to // the previous hardcoded behavior. Smaller opt-in windows shrink the overlap so @@ -1606,6 +1611,19 @@ class Greeter: } } + #[test] + fn test_chunk_by_lines_zero_clamped_to_one() { + // A window of 0 must be clamped to 1 rather than hang (step_size would otherwise + // be 0 and the loop would never advance). Behaves like a window of 1. + let content = "a\nb\nc"; + let chunks = chunk_by_lines(content, &Language::Text, 0); + + assert_eq!(chunks.len(), 3, "Zero window clamps to one line per chunk"); + for chunk in &chunks { + assert_eq!(chunk.end_line - chunk.start_line + 1, 1); + } + } + #[test] fn test_jsdoc_extraction() { let content = r#" diff --git a/src/config/schema.ts b/src/config/schema.ts index f76f688..3e5e4b8 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -228,7 +228,7 @@ export function parseConfig(raw: unknown): ParsedCodebaseIndexConfig { maxDepth: typeof rawIndexing.maxDepth === "number" ? (rawIndexing.maxDepth < -1 ? -1 : rawIndexing.maxDepth) : defaultIndexing.maxDepth, maxFilesPerDirectory: typeof rawIndexing.maxFilesPerDirectory === "number" ? Math.max(1, rawIndexing.maxFilesPerDirectory) : defaultIndexing.maxFilesPerDirectory, fallbackToTextOnMaxChunks: typeof rawIndexing.fallbackToTextOnMaxChunks === "boolean" ? rawIndexing.fallbackToTextOnMaxChunks : defaultIndexing.fallbackToTextOnMaxChunks, - linesPerChunk: typeof rawIndexing.linesPerChunk === "number" ? Math.max(1, Math.floor(rawIndexing.linesPerChunk)) : defaultIndexing.linesPerChunk, + linesPerChunk: typeof rawIndexing.linesPerChunk === "number" && Number.isFinite(rawIndexing.linesPerChunk) ? Math.max(1, Math.floor(rawIndexing.linesPerChunk)) : defaultIndexing.linesPerChunk, gitBlame: { enabled: rawIndexing.gitBlame && typeof rawIndexing.gitBlame === "object" && typeof (rawIndexing.gitBlame as Record).enabled === "boolean" ? (rawIndexing.gitBlame as { enabled: boolean }).enabled diff --git a/tests/config.test.ts b/tests/config.test.ts index 279b5d8..c757279 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -272,6 +272,10 @@ describe("config schema", () => { // Values below 1 are clamped to 1, and floats are floored. expect(parseConfig({ indexing: { linesPerChunk: 0 } }).indexing.linesPerChunk).toBe(1); expect(parseConfig({ indexing: { linesPerChunk: 7.9 } }).indexing.linesPerChunk).toBe(7); + // Non-finite numbers (NaN, Infinity) fall back to the default rather than leaking + // through to the native layer, where they would coerce to 0. + expect(parseConfig({ indexing: { linesPerChunk: NaN } }).indexing.linesPerChunk).toBe(30); + expect(parseConfig({ indexing: { linesPerChunk: Infinity } }).indexing.linesPerChunk).toBe(30); }); it("should enforce minimum of 1 for gcIntervalDays", () => { From 565038624cb35cb93e25caea972c4d100d7aca1d Mon Sep 17 00:00:00 2001 From: Dmitri Khokhlov Date: Mon, 17 Aug 2026 00:52:47 -0700 Subject: [PATCH 3/5] refactor(native): dedupe lines_per_chunk default into a single const The napi fallback value 30 was repeated in three unwrap_or(30) call sites (parse_file, parse_file_as_text, parse_files). Extract DEFAULT_LINES_PER_CHUNK so the Rust fallback has one source, and add bidirectional sync comments linking it to the TS default in src/config/defaults.ts. The cross-language pair cannot share a literal; the comments are the sync mechanism. No behavior change (30 -> 30). --- native/src/lib.rs | 12 +++++++++--- src/config/defaults.ts | 2 ++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/native/src/lib.rs b/native/src/lib.rs index db1bcde..df73a7e 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -23,13 +23,19 @@ pub use parser::*; pub use store::*; pub use types::*; +/// Default lines-per-chunk window used when a napi caller omits the argument +/// (undefined -> None). The config layer always supplies a value, so this only +/// covers direct native callers and tests. Must stay in sync with +/// `getDefaultIndexingConfig().linesPerChunk` in src/config/defaults.ts. +const DEFAULT_LINES_PER_CHUNK: u32 = 30; + #[napi] pub fn parse_file( file_path: String, content: String, lines_per_chunk: Option, ) -> Result> { - parser::parse_file_internal(&file_path, &content, lines_per_chunk.unwrap_or(30) as usize) + parser::parse_file_internal(&file_path, &content, lines_per_chunk.unwrap_or(DEFAULT_LINES_PER_CHUNK) as usize) .map_err(|e| Error::from_reason(e.to_string())) } @@ -39,13 +45,13 @@ pub fn parse_file_as_text( content: String, lines_per_chunk: Option, ) -> Result> { - parser::parse_file_as_text_internal(&file_path, &content, lines_per_chunk.unwrap_or(30) as usize) + parser::parse_file_as_text_internal(&file_path, &content, lines_per_chunk.unwrap_or(DEFAULT_LINES_PER_CHUNK) as usize) .map_err(|e| Error::from_reason(e.to_string())) } #[napi] pub fn parse_files(files: Vec, lines_per_chunk: Option) -> Result> { - parser::parse_files_parallel(files, lines_per_chunk.unwrap_or(30) as usize) + parser::parse_files_parallel(files, lines_per_chunk.unwrap_or(DEFAULT_LINES_PER_CHUNK) as usize) .map_err(|e| Error::from_reason(e.to_string())) } diff --git a/src/config/defaults.ts b/src/config/defaults.ts index 96d9e37..cbc5f09 100644 --- a/src/config/defaults.ts +++ b/src/config/defaults.ts @@ -25,6 +25,8 @@ export function getDefaultIndexingConfig(): IndexingConfig { maxDepth: 5, maxFilesPerDirectory: 100, fallbackToTextOnMaxChunks: true, + // Must stay in sync with DEFAULT_LINES_PER_CHUNK in native/src/lib.rs (the napi + // fallback used when a native caller omits the argument). linesPerChunk: 30, gitBlame: { enabled: false }, }; From e05f2d9c30d1742bd117df75c79080d2508e6156 Mon Sep 17 00:00:00 2001 From: Dmitri Khokhlov Date: Mon, 17 Aug 2026 21:21:31 -0700 Subject: [PATCH 4/5] fix(indexing): clamp linesPerChunk to u32::MAX to prevent native wrap parseConfig() accepted any finite linesPerChunk >= 1, but the NAPI parameter is Option. Values above u32::MAX (4294967295) silently wrapped to 0 during the native conversion, and chunk_by_lines treated the 0 as 1, so an oversized configured window over-chunked a file: a 2-line text file with linesPerChunk = 4294967296 produced two 1-line chunks instead of one 2-line chunk. Clamp the public config to the NAPI u32 range (1..=4294967295) so the native layer never receives a wrapped value. Add a regression test for the oversized finite value. --- src/config/schema.ts | 2 +- tests/config.test.ts | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/config/schema.ts b/src/config/schema.ts index 3e5e4b8..1056213 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -228,7 +228,7 @@ export function parseConfig(raw: unknown): ParsedCodebaseIndexConfig { maxDepth: typeof rawIndexing.maxDepth === "number" ? (rawIndexing.maxDepth < -1 ? -1 : rawIndexing.maxDepth) : defaultIndexing.maxDepth, maxFilesPerDirectory: typeof rawIndexing.maxFilesPerDirectory === "number" ? Math.max(1, rawIndexing.maxFilesPerDirectory) : defaultIndexing.maxFilesPerDirectory, fallbackToTextOnMaxChunks: typeof rawIndexing.fallbackToTextOnMaxChunks === "boolean" ? rawIndexing.fallbackToTextOnMaxChunks : defaultIndexing.fallbackToTextOnMaxChunks, - linesPerChunk: typeof rawIndexing.linesPerChunk === "number" && Number.isFinite(rawIndexing.linesPerChunk) ? Math.max(1, Math.floor(rawIndexing.linesPerChunk)) : defaultIndexing.linesPerChunk, + linesPerChunk: typeof rawIndexing.linesPerChunk === "number" && Number.isFinite(rawIndexing.linesPerChunk) ? Math.min(Math.max(1, Math.floor(rawIndexing.linesPerChunk)), 4294967295) : defaultIndexing.linesPerChunk, gitBlame: { enabled: rawIndexing.gitBlame && typeof rawIndexing.gitBlame === "object" && typeof (rawIndexing.gitBlame as Record).enabled === "boolean" ? (rawIndexing.gitBlame as { enabled: boolean }).enabled diff --git a/tests/config.test.ts b/tests/config.test.ts index c757279..42237d9 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -276,6 +276,11 @@ describe("config schema", () => { // through to the native layer, where they would coerce to 0. expect(parseConfig({ indexing: { linesPerChunk: NaN } }).indexing.linesPerChunk).toBe(30); expect(parseConfig({ indexing: { linesPerChunk: Infinity } }).indexing.linesPerChunk).toBe(30); + // Values above u32::MAX wrap to 0 in the native Option and then over-chunk + // (chunk_by_lines treats 0 as 1). Clamp to u32::MAX (4294967295) so the native + // layer never receives a wrapped value. + expect(parseConfig({ indexing: { linesPerChunk: 4_294_967_296 } }).indexing.linesPerChunk).toBe(4294967295); + expect(parseConfig({ indexing: { linesPerChunk: 9_999_999_999 } }).indexing.linesPerChunk).toBe(4294967295); }); it("should enforce minimum of 1 for gcIntervalDays", () => { From 64d582226b182a3f79a0e5a20ab4c75d1c6747fb Mon Sep 17 00:00:00 2001 From: Dmitri Khokhlov Date: Mon, 17 Aug 2026 23:32:42 -0700 Subject: [PATCH 5/5] style(native): rustfmt the lines_per_chunk threading The linesPerChunk PR threaded `lines_per_chunk: Option` through the napi entry points (parse_file, parse_file_as_text, parse_files) and the parser internals, producing single-line call sites that exceed rustfmt's width limit. CI's `cargo fmt --check` flagged six unformatted sites: native/src/lib.rs:35, 45, 53 (napi entry-point forwarding calls) native/src/parser.rs:117, 145 (chunk_by_lines + internal signatures) native/src/parser.rs:1931 (test_parse_cpp_preserves_small_type_symbols) Run `cargo fmt` to wrap them. Behavior unchanged; 128 native tests pass. --- native/src/lib.rs | 23 +++++++++++++++++------ native/src/parser.rs | 15 ++++++++++++--- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/native/src/lib.rs b/native/src/lib.rs index df73a7e..c373de0 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -35,8 +35,12 @@ pub fn parse_file( content: String, lines_per_chunk: Option, ) -> Result> { - parser::parse_file_internal(&file_path, &content, lines_per_chunk.unwrap_or(DEFAULT_LINES_PER_CHUNK) as usize) - .map_err(|e| Error::from_reason(e.to_string())) + parser::parse_file_internal( + &file_path, + &content, + lines_per_chunk.unwrap_or(DEFAULT_LINES_PER_CHUNK) as usize, + ) + .map_err(|e| Error::from_reason(e.to_string())) } #[napi] @@ -45,14 +49,21 @@ pub fn parse_file_as_text( content: String, lines_per_chunk: Option, ) -> Result> { - parser::parse_file_as_text_internal(&file_path, &content, lines_per_chunk.unwrap_or(DEFAULT_LINES_PER_CHUNK) as usize) - .map_err(|e| Error::from_reason(e.to_string())) + parser::parse_file_as_text_internal( + &file_path, + &content, + lines_per_chunk.unwrap_or(DEFAULT_LINES_PER_CHUNK) as usize, + ) + .map_err(|e| Error::from_reason(e.to_string())) } #[napi] pub fn parse_files(files: Vec, lines_per_chunk: Option) -> Result> { - parser::parse_files_parallel(files, lines_per_chunk.unwrap_or(DEFAULT_LINES_PER_CHUNK) as usize) - .map_err(|e| Error::from_reason(e.to_string())) + parser::parse_files_parallel( + files, + lines_per_chunk.unwrap_or(DEFAULT_LINES_PER_CHUNK) as usize, + ) + .map_err(|e| Error::from_reason(e.to_string())) } #[napi] diff --git a/native/src/parser.rs b/native/src/parser.rs index d926e59..d5fccb6 100644 --- a/native/src/parser.rs +++ b/native/src/parser.rs @@ -117,7 +117,10 @@ fn parse_file_with_symbols_internal( let language = Language::from_extension(ext); if language == Language::Text { - return Ok((chunk_by_lines(content, &language, lines_per_chunk), Vec::new())); + return Ok(( + chunk_by_lines(content, &language, lines_per_chunk), + Vec::new(), + )); } let mut parser = Parser::new(); @@ -145,7 +148,12 @@ fn parse_file_with_symbols_internal( Language::Gdscript => tree_sitter_gdscript::LANGUAGE.into(), Language::Matlab => tree_sitter_matlab::LANGUAGE.into(), Language::Apex => tree_sitter_sfapex::apex::LANGUAGE.into(), - _ => return Ok((chunk_by_lines(content, &language, lines_per_chunk), Vec::new())), + _ => { + return Ok(( + chunk_by_lines(content, &language, lines_per_chunk), + Vec::new(), + )) + } }; parser.set_language(&ts_language)?; @@ -1931,7 +1939,8 @@ int main() { #[test] fn test_parse_cpp_preserves_small_type_symbols() { - let chunks = parse_file_internal("small.cpp", "class Tag {};\nstruct Point {};\n", 30).unwrap(); + let chunks = + parse_file_internal("small.cpp", "class Tag {};\nstruct Point {};\n", 30).unwrap(); assert!(chunks.iter().any(|chunk| { chunk.chunk_type == "class_specifier" && chunk.name.as_deref() == Some("Tag")