diff --git a/docs/configuration.md b/docs/configuration.md index 9989bcb..aa5f0c4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -169,6 +169,7 @@ Changing provider, model, dimensions, or embedding strategy can make an existing | `maxDepth` | `5` | Directory traversal depth; `-1` is unlimited | | `maxFilesPerDirectory` | `100` | Per-directory file cap | | `fallbackToTextOnMaxChunks` | `true` | Fall back to line chunks when the semantic cap is reached | +| `linesPerChunk` | `30` | Max lines per chunk for line-based parsing (`.jsonl`, `.txt`, unknown extensions, and the AST fallback). Lower it for finer-grained retrieval on line-delimited files. Only the line-based path is affected; AST-parsed languages are unchanged | | `gitBlame.enabled` | `false` | Store git blame metadata for filtering | Example: diff --git a/native/src/lib.rs b/native/src/lib.rs index c0e8e13..c373de0 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -23,20 +23,47 @@ pub use parser::*; pub use store::*; pub use types::*; +/// Default lines-per-chunk window used when a napi caller omits the argument +/// (undefined -> None). The config layer always supplies a value, so this only +/// covers direct native callers and tests. Must stay in sync with +/// `getDefaultIndexingConfig().linesPerChunk` in src/config/defaults.ts. +const DEFAULT_LINES_PER_CHUNK: u32 = 30; + #[napi] -pub fn parse_file(file_path: String, content: String) -> Result> { - parser::parse_file_internal(&file_path, &content).map_err(|e| Error::from_reason(e.to_string())) +pub fn parse_file( + file_path: String, + content: String, + lines_per_chunk: Option, +) -> Result> { + parser::parse_file_internal( + &file_path, + &content, + lines_per_chunk.unwrap_or(DEFAULT_LINES_PER_CHUNK) as usize, + ) + .map_err(|e| Error::from_reason(e.to_string())) } #[napi] -pub fn parse_file_as_text(file_path: String, content: String) -> Result> { - parser::parse_file_as_text_internal(&file_path, &content) - .map_err(|e| Error::from_reason(e.to_string())) +pub fn parse_file_as_text( + file_path: String, + content: String, + lines_per_chunk: Option, +) -> Result> { + parser::parse_file_as_text_internal( + &file_path, + &content, + lines_per_chunk.unwrap_or(DEFAULT_LINES_PER_CHUNK) as usize, + ) + .map_err(|e| Error::from_reason(e.to_string())) } #[napi] -pub fn parse_files(files: Vec) -> Result> { - parser::parse_files_parallel(files).map_err(|e| Error::from_reason(e.to_string())) +pub fn parse_files(files: Vec, lines_per_chunk: Option) -> Result> { + parser::parse_files_parallel( + files, + lines_per_chunk.unwrap_or(DEFAULT_LINES_PER_CHUNK) as usize, + ) + .map_err(|e| Error::from_reason(e.to_string())) } #[napi] diff --git a/native/src/parser.rs b/native/src/parser.rs index 4b3d06d..d5fccb6 100644 --- a/native/src/parser.rs +++ b/native/src/parser.rs @@ -14,7 +14,11 @@ const MAX_CHUNK_SIZE: usize = 2000; const TARGET_CHUNK_SIZE: usize = 500; const OVERLAP_LINES: usize = 3; -pub fn parse_file_internal(file_path: &str, content: &str) -> Result> { +pub fn parse_file_internal( + file_path: &str, + content: &str, + lines_per_chunk: usize, +) -> Result> { let ext = Path::new(file_path) .extension() .and_then(|e| e.to_str()) @@ -23,7 +27,7 @@ pub fn parse_file_internal(file_path: &str, content: &str) -> Result Result tree_sitter_gdscript::LANGUAGE.into(), Language::Matlab => tree_sitter_matlab::LANGUAGE.into(), Language::Apex => tree_sitter_sfapex::apex::LANGUAGE.into(), - _ => return Ok(chunk_by_lines(content, &language)), + _ => return Ok(chunk_by_lines(content, &language, lines_per_chunk)), }; parser.set_language(&ts_language)?; @@ -61,25 +65,33 @@ pub fn parse_file_internal(file_path: &str, content: &str) -> Result Result> { +pub fn parse_file_as_text_internal( + file_path: &str, + content: &str, + lines_per_chunk: usize, +) -> Result> { let ext = Path::new(file_path) .extension() .and_then(|e| e.to_str()) .unwrap_or(""); let language = Language::from_extension(ext); - Ok(chunk_by_lines(content, &language)) + Ok(chunk_by_lines(content, &language, lines_per_chunk)) } -pub fn parse_files_parallel(files: Vec) -> Result> { +pub fn parse_files_parallel( + files: Vec, + lines_per_chunk: usize, +) -> Result> { let results: Vec = files .par_iter() .filter_map(|file| { let (chunks, symbols) = - parse_file_with_symbols_internal(&file.path, &file.content).ok()?; + parse_file_with_symbols_internal(&file.path, &file.content, lines_per_chunk) + .ok()?; let hash = crate::hasher::xxhash_content(&file.content); Some(ParsedFile { path: file.path.clone(), @@ -96,6 +108,7 @@ pub fn parse_files_parallel(files: Vec) -> Result> { fn parse_file_with_symbols_internal( file_path: &str, content: &str, + lines_per_chunk: usize, ) -> Result<(Vec, Vec)> { let ext = Path::new(file_path) .extension() @@ -104,7 +117,10 @@ fn parse_file_with_symbols_internal( let language = Language::from_extension(ext); if language == Language::Text { - return Ok((chunk_by_lines(content, &language), Vec::new())); + return Ok(( + chunk_by_lines(content, &language, lines_per_chunk), + Vec::new(), + )); } let mut parser = Parser::new(); @@ -132,14 +148,19 @@ fn parse_file_with_symbols_internal( Language::Gdscript => tree_sitter_gdscript::LANGUAGE.into(), Language::Matlab => tree_sitter_matlab::LANGUAGE.into(), Language::Apex => tree_sitter_sfapex::apex::LANGUAGE.into(), - _ => return Ok((chunk_by_lines(content, &language), Vec::new())), + _ => { + return Ok(( + chunk_by_lines(content, &language, lines_per_chunk), + Vec::new(), + )) + } }; parser.set_language(&ts_language)?; let tree = parser .parse(content, None) .ok_or_else(|| anyhow!("Failed to parse file: {}", file_path))?; - let chunks = extract_chunks(&tree, content, &language)?; + let chunks = extract_chunks(&tree, content, &language, lines_per_chunk)?; let symbols = extract_symbols(&tree, content, &language); Ok((chunks, symbols)) } @@ -187,7 +208,12 @@ fn extract_symbol_nodes( } } -fn extract_chunks(tree: &Tree, source: &str, language: &Language) -> Result> { +fn extract_chunks( + tree: &Tree, + source: &str, + language: &Language, + lines_per_chunk: usize, +) -> Result> { let mut chunks = Vec::new(); let root = tree.root_node(); let mut cursor = root.walk(); @@ -195,7 +221,7 @@ fn extract_chunks(tree: &Tree, source: &str, language: &Language) -> Result) { *chunks = merged; } -fn chunk_by_lines(content: &str, language: &Language) -> Vec { +fn chunk_by_lines(content: &str, language: &Language, lines_per_chunk: usize) -> Vec { let lines: Vec<&str> = content.lines().collect(); let total_lines = lines.len(); @@ -1162,12 +1188,17 @@ fn chunk_by_lines(content: &str, language: &Language) -> Vec { return Vec::new(); } - let lines_per_chunk = 30; - let step_size = if lines_per_chunk > OVERLAP_LINES { - lines_per_chunk - OVERLAP_LINES - } else { - lines_per_chunk - }; + // Defense in depth: the config layer clamps linesPerChunk to >= 1, but this pub fn is + // also reachable through the napi API with an explicit 0, which would make step_size 0 + // and hang the loop below. Treat anything below 1 as 1 (one line per chunk). + let lines_per_chunk = lines_per_chunk.max(1); + + // Cap the overlap so it stays at or below ~25% of the window. At the default + // lines_per_chunk (30) the overlap is min(3, 7) = 3, which is bit-identical to + // the previous hardcoded behavior. Smaller opt-in windows shrink the overlap so + // the chunks do not collapse into near-duplicates. + let overlap = OVERLAP_LINES.min(lines_per_chunk / 4); + let step_size = lines_per_chunk.saturating_sub(overlap); let mut chunks = Vec::new(); let mut start = 0; @@ -1389,7 +1420,7 @@ class Greeter { } "#; - let chunks = parse_file_internal("test.ts", content).unwrap(); + let chunks = parse_file_internal("test.ts", content, 30).unwrap(); assert!(!chunks.is_empty()); } @@ -1404,7 +1435,7 @@ class Greeter { const values = [1, 2].map(item => item * 2); "#; - let (_chunks, symbols) = parse_file_with_symbols_internal("arrows.ts", content) + let (_chunks, symbols) = parse_file_with_symbols_internal("arrows.ts", content, 30) .expect("should parse TypeScript arrow functions"); let arrow_names: Vec = symbols @@ -1434,7 +1465,7 @@ class Greeter { } "#; - let (_chunks, symbols) = parse_file_with_symbols_internal("animal.ts", content) + let (_chunks, symbols) = parse_file_with_symbols_internal("animal.ts", content, 30) .expect("should parse exported abstract class"); assert!(symbols.iter().any(|symbol| { @@ -1465,7 +1496,7 @@ class Greeter: return f"Hello, {self.name}!" "#; - let chunks = parse_file_internal("test.py", content).unwrap(); + let chunks = parse_file_internal("test.py", content, 30).unwrap(); assert!(!chunks.is_empty()); } @@ -1479,7 +1510,7 @@ class Greeter: #[test] fn test_parse_php_8_semantic_chunks_have_names() { let content = include_str!("../../tests/fixtures/call-graph/php-8-features.php"); - let chunks = parse_file_internal("php-8-features.php", content).unwrap(); + let chunks = parse_file_internal("php-8-features.php", content, 30).unwrap(); assert!(chunks.iter().any(|chunk| { chunk.chunk_type == "class_declaration" && chunk.name.as_deref() == Some("Job") @@ -1506,7 +1537,7 @@ class Greeter: .collect(); let content = lines.join("\n"); - let chunks = chunk_by_lines(&content, &Language::Text); + let chunks = chunk_by_lines(&content, &Language::Text, 30); assert!(chunks.len() >= 2, "Should have multiple chunks"); @@ -1522,6 +1553,85 @@ class Greeter: } } + #[test] + fn test_chunk_by_lines_default_unchanged() { + // The default window (30) must keep the historical 30-line chunks and 3-line + // overlap (step 27), so existing behavior is bit-identical when the knob is + // left at its default. + let lines: Vec = (0..100) + .map(|i| format!("line {} content here", i)) + .collect(); + let content = lines.join("\n"); + + let chunks = chunk_by_lines(&content, &Language::Text, 30); + + assert!(!chunks.is_empty(), "Should produce chunks"); + let first_span = chunks[0].end_line - chunks[0].start_line + 1; + assert_eq!(first_span, 30, "First chunk should span 30 lines"); + if chunks.len() >= 2 { + let step = chunks[1].start_line - chunks[0].start_line; + assert_eq!(step, 27, "Default step should be 27 (30 - overlap 3)"); + } + } + + #[test] + fn test_chunk_by_lines_custom_size() { + // A smaller opt-in window (5) must shrink both the chunk size and the + // overlap. overlap = min(3, 5/4 = 1) = 1, so step = 5 - 1 = 4. + let lines: Vec = (0..40) + .map(|i| format!("line {} content here", i)) + .collect(); + let content = lines.join("\n"); + + let chunks = chunk_by_lines(&content, &Language::Text, 5); + + assert!(chunks.len() >= 2, "Should produce multiple small chunks"); + for chunk in &chunks { + let span = chunk.end_line - chunk.start_line + 1; + assert!( + span <= 5, + "Each chunk should span at most 5 lines, got {}", + span + ); + } + if chunks.len() >= 2 { + let step = chunks[1].start_line - chunks[0].start_line; + assert_eq!(step, 4, "Step should be 4 (window 5 - overlap 1)"); + // Overlap of 1 line: chunk 1 starts where chunk 0 is still in range. + assert!( + chunks[1].start_line <= chunks[0].end_line, + "Chunks should overlap by 1 line" + ); + } + } + + #[test] + fn test_chunk_by_lines_size_one_no_overlap() { + // A window of 1 line must not overlap itself (overlap = min(3, 0) = 0), + // producing one chunk per non-empty line. + let content = "a\nb\nc"; + let chunks = chunk_by_lines(content, &Language::Text, 1); + + assert_eq!(chunks.len(), 3, "One line per chunk"); + for chunk in &chunks { + let span = chunk.end_line - chunk.start_line + 1; + assert_eq!(span, 1, "Each chunk should span exactly 1 line"); + } + } + + #[test] + fn test_chunk_by_lines_zero_clamped_to_one() { + // A window of 0 must be clamped to 1 rather than hang (step_size would otherwise + // be 0 and the loop would never advance). Behaves like a window of 1. + let content = "a\nb\nc"; + let chunks = chunk_by_lines(content, &Language::Text, 0); + + assert_eq!(chunks.len(), 3, "Zero window clamps to one line per chunk"); + for chunk in &chunks { + assert_eq!(chunk.end_line - chunk.start_line + 1, 1); + } + } + #[test] fn test_jsdoc_extraction() { let content = r#" @@ -1535,7 +1645,7 @@ function validateEmail(email: string): boolean { } "#; - let chunks = parse_file_internal("test.ts", content).unwrap(); + let chunks = parse_file_internal("test.ts", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have at least one chunk"); let chunk = &chunks[0]; @@ -1561,7 +1671,7 @@ fn factorial(n: u64) -> Option { } "#; - let chunks = parse_file_internal("test.rs", content).unwrap(); + let chunks = parse_file_internal("test.rs", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have at least one chunk"); let chunk = &chunks[0]; @@ -1596,7 +1706,7 @@ public enum Operation { } "#; - let chunks = parse_file_internal("Calculator.java", content).unwrap(); + let chunks = parse_file_internal("Calculator.java", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for Java"); let has_class = chunks.iter().any(|c| c.chunk_type == "class_declaration"); @@ -1633,7 +1743,7 @@ public struct Point } "#; - let chunks = parse_file_internal("Person.cs", content).unwrap(); + let chunks = parse_file_internal("Person.cs", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for C#"); } @@ -1661,7 +1771,7 @@ module Utils end "#; - let chunks = parse_file_internal("greeter.rb", content).unwrap(); + let chunks = parse_file_internal("greeter.rb", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for Ruby"); let has_class = chunks.iter().any(|c| c.chunk_type == "class"); @@ -1682,7 +1792,7 @@ module Rack end "#; - let (_, symbols) = parse_file_with_symbols_internal("rack/protection.rb", content) + let (_, symbols) = parse_file_with_symbols_internal("rack/protection.rb", content, 30) .expect("should parse nested Ruby modules"); assert!( symbols @@ -1691,7 +1801,7 @@ end "Expected nested Ruby class symbol named ContentSecurityPolicy" ); - let chunks = parse_file_internal("rack/protection.rb", content) + let chunks = parse_file_internal("rack/protection.rb", content, 30) .expect("should produce semantic chunks for nested Ruby class"); assert!( chunks.iter().any(|chunk| { @@ -1721,7 +1831,7 @@ function add() { greet "World" "#; - let chunks = parse_file_internal("script.sh", content).unwrap(); + let chunks = parse_file_internal("script.sh", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for Bash"); let has_function = chunks.iter().any(|c| c.chunk_type == "function_definition"); @@ -1762,7 +1872,7 @@ void greet(const char* name) { } "#; - let chunks = parse_file_internal("main.c", content).unwrap(); + let chunks = parse_file_internal("main.c", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for C"); let has_function = chunks.iter().any(|c| c.chunk_type == "function_definition"); @@ -1808,7 +1918,7 @@ int main() { } "#; - let chunks = parse_file_internal("main.cpp", content).unwrap(); + let chunks = parse_file_internal("main.cpp", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for C++"); let has_class = chunks.iter().any(|c| c.chunk_type == "class_specifier"); @@ -1829,7 +1939,8 @@ int main() { #[test] fn test_parse_cpp_preserves_small_type_symbols() { - let chunks = parse_file_internal("small.cpp", "class Tag {};\nstruct Point {};\n").unwrap(); + let chunks = + parse_file_internal("small.cpp", "class Tag {};\nstruct Point {};\n", 30).unwrap(); assert!(chunks.iter().any(|chunk| { chunk.chunk_type == "class_specifier" && chunk.name.as_deref() == Some("Tag") @@ -1847,7 +1958,7 @@ inline T scaled_value(T value, constant float& scale) { } "#; - let chunks = parse_file_internal("shader.metal", content).unwrap(); + let chunks = parse_file_internal("shader.metal", content, 30).unwrap(); let function = chunks .iter() .find(|chunk| chunk.name.as_deref() == Some("scaled_value")) @@ -1884,7 +1995,7 @@ lto = true opt-level = 3 "#; - let chunks = parse_file_internal("Cargo.toml", content).unwrap(); + let chunks = parse_file_internal("Cargo.toml", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for TOML"); let has_table = chunks.iter().any(|c| c.chunk_type == "table"); @@ -1922,7 +2033,7 @@ spec: - containerPort: 8080 "#; - let chunks = parse_file_internal("deployment.yaml", content).unwrap(); + let chunks = parse_file_internal("deployment.yaml", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for YAML"); } @@ -1953,7 +2064,7 @@ myFunction(); Please read CONTRIBUTING.md for details. "#; - let chunks = parse_file_internal("README.md", content).unwrap(); + let chunks = parse_file_internal("README.md", content, 30).unwrap(); // Markdown falls back to line-based chunking assert!(!chunks.is_empty(), "Should have chunks for Markdown"); // Should be block type since we use line-based chunking @@ -1979,7 +2090,7 @@ public with sharing class AccountService { } "#; - let chunks = parse_file_internal("AccountService.cls", content).unwrap(); + let chunks = parse_file_internal("AccountService.cls", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for Apex"); let has_class = chunks.iter().any(|c| c.chunk_type == "class_declaration"); @@ -1996,7 +2107,7 @@ trigger AccountTrigger on Account (before insert, before update, after delete) { } "#; - let chunks = parse_file_internal("AccountTrigger.trigger", content).unwrap(); + let chunks = parse_file_internal("AccountTrigger.trigger", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for Apex trigger"); let has_trigger = chunks.iter().any(|c| c.chunk_type == "trigger_declaration"); @@ -2029,7 +2140,7 @@ class InnerThing: pass "#; - let chunks = parse_file_internal("player.gd", content).unwrap(); + let chunks = parse_file_internal("player.gd", content, 30).unwrap(); assert!(!chunks.is_empty(), "Should have chunks for GDScript"); let chunk_types: Vec<&str> = chunks.iter().map(|c| c.chunk_type.as_str()).collect(); @@ -2071,7 +2182,7 @@ public class AccountService { } "#; - let chunks = parse_file_internal("AccountService.cls", content).unwrap(); + let chunks = parse_file_internal("AccountService.cls", content, 30).unwrap(); let class_chunk = chunks.iter().find(|c| c.chunk_type == "class_declaration"); assert!(class_chunk.is_some(), "Should find class_declaration"); assert!( diff --git a/src/config/defaults.ts b/src/config/defaults.ts index c096a4c..cbc5f09 100644 --- a/src/config/defaults.ts +++ b/src/config/defaults.ts @@ -25,6 +25,9 @@ export function getDefaultIndexingConfig(): IndexingConfig { maxDepth: 5, maxFilesPerDirectory: 100, fallbackToTextOnMaxChunks: true, + // Must stay in sync with DEFAULT_LINES_PER_CHUNK in native/src/lib.rs (the napi + // fallback used when a native caller omits the argument). + linesPerChunk: 30, gitBlame: { enabled: false }, }; } diff --git a/src/config/schema.ts b/src/config/schema.ts index e09d52f..1056213 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -65,6 +65,16 @@ export interface IndexingConfig { * instead of skipping the rest of the file. Default: true */ fallbackToTextOnMaxChunks: boolean; + /** + * Max lines per chunk for line-based parsing (.jsonl, .txt, unknown extensions, + * and the AST fallback path). Each chunk is a sliding window of this many lines. + * Default: 30. Lower it for finer-grained retrieval on line-delimited files (for + * example Claude .jsonl session transcripts, where one line is one message). The + * overlap between neighboring chunks is auto-capped at one quarter of this value, + * so smaller windows shrink the overlap too. Only the line-based path is affected; + * AST-parsed languages are unchanged. + */ + linesPerChunk: number; gitBlame: { enabled: boolean; }; @@ -218,6 +228,7 @@ export function parseConfig(raw: unknown): ParsedCodebaseIndexConfig { maxDepth: typeof rawIndexing.maxDepth === "number" ? (rawIndexing.maxDepth < -1 ? -1 : rawIndexing.maxDepth) : defaultIndexing.maxDepth, maxFilesPerDirectory: typeof rawIndexing.maxFilesPerDirectory === "number" ? Math.max(1, rawIndexing.maxFilesPerDirectory) : defaultIndexing.maxFilesPerDirectory, fallbackToTextOnMaxChunks: typeof rawIndexing.fallbackToTextOnMaxChunks === "boolean" ? rawIndexing.fallbackToTextOnMaxChunks : defaultIndexing.fallbackToTextOnMaxChunks, + linesPerChunk: typeof rawIndexing.linesPerChunk === "number" && Number.isFinite(rawIndexing.linesPerChunk) ? Math.min(Math.max(1, Math.floor(rawIndexing.linesPerChunk)), 4294967295) : defaultIndexing.linesPerChunk, gitBlame: { enabled: rawIndexing.gitBlame && typeof rawIndexing.gitBlame === "object" && typeof (rawIndexing.gitBlame as Record).enabled === "boolean" ? (rawIndexing.gitBlame as { enabled: boolean }).enabled diff --git a/src/indexer/index.ts b/src/indexer/index.ts index 1f165be..e6e64ef 100644 --- a/src/indexer/index.ts +++ b/src/indexer/index.ts @@ -4346,7 +4346,7 @@ export class Indexer { const loadedByPath = new Map(loadedFiles.map((file) => [file.path, file])); const descriptorByPath = new Map(descriptorBatch.map((descriptor) => [descriptor.storedPath, descriptor])); const parseStartTime = performance.now(); - const parsedFiles = parseFiles(loadedFiles); + const parsedFiles = parseFiles(loadedFiles, this.config.indexing.linesPerChunk); const parseMs = performance.now() - parseStartTime; this.logger.recordFilesParsed(parsedFiles.length); this.logger.recordParseDuration(parseMs); @@ -4378,7 +4378,7 @@ export class Indexer { this.config.indexing.fallbackToTextOnMaxChunks && chunksToProcess.length > this.config.indexing.maxChunksPerFile ) { - chunksToProcess = parseFileAsText(parsed.path, loadedFile.content); + chunksToProcess = parseFileAsText(parsed.path, loadedFile.content, this.config.indexing.linesPerChunk); } chunksToProcess = selectIndexableChunks( chunksToProcess, diff --git a/src/native/parsing.ts b/src/native/parsing.ts index fdcc4b3..2c39391 100644 --- a/src/native/parsing.ts +++ b/src/native/parsing.ts @@ -1,18 +1,18 @@ import type { CallSiteData, CodeChunk, FileInput, ParsedFile, ParsedSymbol, ChunkType } from "./types.js"; import { native } from "./binding.js"; -export function parseFile(filePath: string, content: string): CodeChunk[] { - const result = native.parseFile(filePath, content); +export function parseFile(filePath: string, content: string, linesPerChunk?: number): CodeChunk[] { + const result = native.parseFile(filePath, content, linesPerChunk); return result.map(mapChunk); } -export function parseFileAsText(filePath: string, content: string): CodeChunk[] { - const result = native.parseFileAsText(filePath, content); +export function parseFileAsText(filePath: string, content: string, linesPerChunk?: number): CodeChunk[] { + const result = native.parseFileAsText(filePath, content, linesPerChunk); return result.map(mapChunk); } -export function parseFiles(files: FileInput[]): ParsedFile[] { - const result = native.parseFiles(files); +export function parseFiles(files: FileInput[], linesPerChunk?: number): ParsedFile[] { + const result = native.parseFiles(files, linesPerChunk); return result.map((f: any) => ({ path: f.path, chunks: f.chunks.map(mapChunk), diff --git a/tests/config.test.ts b/tests/config.test.ts index fb24394..42237d9 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -264,6 +264,25 @@ describe("config schema", () => { expect(parseConfig({ indexing: { maxChunksPerFile: -5 } }).indexing.maxChunksPerFile).toBe(1); }); + it("should default linesPerChunk to 30 and honor explicit overrides", () => { + expect(parseConfig({}).indexing.linesPerChunk).toBe(30); + expect(parseConfig({ indexing: { linesPerChunk: 10 } }).indexing.linesPerChunk).toBe(10); + // Non-number falls back to the default. + expect(parseConfig({ indexing: { linesPerChunk: "small" } }).indexing.linesPerChunk).toBe(30); + // Values below 1 are clamped to 1, and floats are floored. + expect(parseConfig({ indexing: { linesPerChunk: 0 } }).indexing.linesPerChunk).toBe(1); + expect(parseConfig({ indexing: { linesPerChunk: 7.9 } }).indexing.linesPerChunk).toBe(7); + // Non-finite numbers (NaN, Infinity) fall back to the default rather than leaking + // through to the native layer, where they would coerce to 0. + expect(parseConfig({ indexing: { linesPerChunk: NaN } }).indexing.linesPerChunk).toBe(30); + expect(parseConfig({ indexing: { linesPerChunk: Infinity } }).indexing.linesPerChunk).toBe(30); + // Values above u32::MAX wrap to 0 in the native Option and then over-chunk + // (chunk_by_lines treats 0 as 1). Clamp to u32::MAX (4294967295) so the native + // layer never receives a wrapped value. + expect(parseConfig({ indexing: { linesPerChunk: 4_294_967_296 } }).indexing.linesPerChunk).toBe(4294967295); + expect(parseConfig({ indexing: { linesPerChunk: 9_999_999_999 } }).indexing.linesPerChunk).toBe(4294967295); + }); + it("should enforce minimum of 1 for gcIntervalDays", () => { expect(parseConfig({ indexing: { gcIntervalDays: 0 } }).indexing.gcIntervalDays).toBe(1); expect(parseConfig({ indexing: { gcIntervalDays: -1 } }).indexing.gcIntervalDays).toBe(1); diff --git a/tests/native.test.ts b/tests/native.test.ts index 99c3c4e..f428ad1 100644 --- a/tests/native.test.ts +++ b/tests/native.test.ts @@ -133,6 +133,25 @@ const add = (a, b) => a + b; expect(chunks[0]?.chunkType).toBe("block"); }); + it("honors linesPerChunk for line-based (.jsonl) files", () => { + const lines = Array.from({ length: 20 }, (_, i) => `{"i":${i}}`); + const content = lines.join("\n"); + + const defaultChunks = parseFile("session.jsonl", content, 30); + const smallChunks = parseFile("session.jsonl", content, 5); + + // Default window covers all 20 lines in one chunk. + expect(defaultChunks.length).toBe(1); + expect(defaultChunks[0].endLine - defaultChunks[0].startLine + 1).toBe(20); + + // Window of 5 with overlap 1 (min(3, 5/4)) -> step 4 -> chunks at starts 1, 5, 9, 13, 17. + expect(smallChunks.length).toBe(5); + for (const chunk of smallChunks) { + expect(chunk.endLine - chunk.startLine + 1).toBeLessThanOrEqual(5); + } + expect(smallChunks[1].startLine - smallChunks[0].startLine).toBe(4); + }); + it("should parse PHP files", () => { const content = ` item * 2); ]), ); }); + + it("honors linesPerChunk across a batch of line-based files", () => { + const lines = Array.from({ length: 12 }, (_, i) => `line ${i}`); + const content = lines.join("\n"); + const files = [ + { path: "a.txt", content }, + { path: "b.jsonl", content }, + ]; + + const [a, b] = parseFiles(files, 4); + + // Window of 4 with overlap 1 (min(3, 4/4)) -> step 3 -> starts 1, 4, 7, 10. + expect(a.chunks.length).toBe(4); + expect(b.chunks.length).toBe(4); + for (const chunk of a.chunks) { + expect(chunk.endLine - chunk.startLine + 1).toBeLessThanOrEqual(4); + } + }); }); describe("hashContent", () => {