diff --git a/Cargo.lock b/Cargo.lock index 61d82ab6..dfb923a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -495,6 +495,7 @@ version = "0.50.3" dependencies = [ "anyhow", "codegraph-core", + "ignore", "rayon", "regex", "serde", diff --git a/Cargo.toml b/Cargo.toml index 4a6f2b2c..0264354d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ sha2 = "0.10" sha1_smol = "1" rayon = "1.8" globset = "0.4" +ignore = "0.4" # Logging (rust-base-convention) tracing = "0.1" diff --git a/crates/codegraph-cli/src/installer/shared.rs b/crates/codegraph-cli/src/installer/shared.rs index a9c046bb..e6615ef2 100644 --- a/crates/codegraph-cli/src/installer/shared.rs +++ b/crates/codegraph-cli/src/installer/shared.rs @@ -356,33 +356,60 @@ pub fn parse_json_object(text: &str) -> Option> { } } -/// Read an agent config file, distinguishing missing / parsed / unparseable. +/// Read an agent config file on a WRITE path, distinguishing missing / parsed / +/// unparseable. /// /// Tolerates JSONC (comments + trailing commas). A present-but-unparseable file /// is backed up to `.backup` and reported as [`ConfigRead::Unparseable`] /// WITHOUT being modified, so the caller can skip writing instead of clobbering /// the user's config. pub fn read_config_file(path: &Path) -> ConfigRead { + let read = read_config(path); + if matches!(read, ConfigRead::Unparseable) { + let _ = fs::copy(path, path.with_extension("backup")); + } + read +} + +/// Read a JSON/JSONC file into a map, `{}` when missing or unparseable. +/// +/// Read-only and silent: every target's `detect()` reads its agent's config +/// on every run, including agents codegraph was never installed into, and +/// `install --refresh` skips the unconfigured ones on that answer, so a +/// `.backup` left by a read would touch a config codegraph has no business +/// touching (upstream #1870). Callers on the WRITE path must NOT use this (it +/// cannot signal the unparseable case); use [`read_config_file`] and abort on +/// [`ConfigRead::Unparseable`] there. +pub fn read_json_file(path: &Path) -> Map { + match read_config(path) { + ConfigRead::Parsed(map) => map, + ConfigRead::Missing | ConfigRead::Unparseable => Map::new(), + } +} + +fn read_config(path: &Path) -> ConfigRead { let Ok(text) = fs::read_to_string(path) else { return ConfigRead::Missing; }; match parse_json_object(&text) { Some(map) => ConfigRead::Parsed(map), - None => { - let _ = fs::copy(path, path.with_extension("backup")); - ConfigRead::Unparseable - } + None => ConfigRead::Unparseable, } } -/// Read a JSON/JSONC file into a map. Backward-compatible helper that maps -/// [`ConfigRead::Missing`] to `{}`. Callers on the WRITE path must NOT use this -/// (it cannot signal the unparseable case); use [`read_config_file`] and abort -/// on [`ConfigRead::Unparseable`] there. -pub fn read_json_file(path: &Path) -> Map { - match read_config_file(path) { - ConfigRead::Parsed(map) => map, - ConfigRead::Missing | ConfigRead::Unparseable => Map::new(), +/// Copy a config about to be replaced to `.backup` when it does not +/// parse: a read handed the caller `{}` for it, so the write is a replacement, +/// not an edit. No-op when the file is absent or parses. +fn backup_unparseable_config(path: &Path) { + let Ok(bytes) = fs::read(path) else { + return; + }; + if std::str::from_utf8(&bytes) + .ok() + .and_then(parse_json_object) + .is_none() + { + let _ = fs::copy(path, path.with_extension("backup")); } } @@ -406,7 +433,11 @@ pub fn atomic_write_file(path: &Path, content: &str) -> std::io::Result<()> { /// Atomic JSON write with a trailing newline. Ports `writeJsonFile` /// (shared.ts:99) — `JSON.stringify(data, null, 2) + '\n'`. +/// +/// The one place a target replaces a JSON config, so it is also where an +/// unparseable one is preserved (upstream #1870). pub fn write_json_file(path: &Path, data: &Map) -> std::io::Result<()> { + backup_unparseable_config(path); let mut content = to_upstream_json(&Value::Object(data.clone())); content.push('\n'); atomic_write_file(path, &content) @@ -1757,6 +1788,36 @@ name = "important-user-tool" let _ = fs::remove_dir_all(&dir); } + /// Reading a config is read-only (upstream #1870): every target's + /// `detect()` reads its agent's config on every run, and `install + /// --refresh` skips unconfigured agents on that answer, so a read must not + /// leave a `.backup` beside a config codegraph never touches. A backup is + /// made where a config is about to be replaced, or before a write path + /// skips one it cannot parse. + #[test] + fn reading_an_unparseable_config_leaves_no_backup() { + let config = tmp_path("mcp_config.json"); + let backup = config.with_extension("backup"); + fs::write(&config, "{ broken").unwrap(); + + assert!(read_json_file(&config).is_empty()); + assert!(!backup.exists(), "a read is not a write"); + + assert!(matches!(read_config_file(&config), ConfigRead::Unparseable)); + assert_eq!(fs::read_to_string(&backup).unwrap(), "{ broken"); + fs::remove_file(&backup).unwrap(); + + let mut data = Map::new(); + data.insert("k".to_string(), Value::from(1)); + write_json_file(&config, &data).unwrap(); + assert_eq!(fs::read_to_string(&backup).unwrap(), "{ broken"); + + fs::remove_file(&backup).unwrap(); + write_json_file(&config, &data).unwrap(); + assert!(!backup.exists(), "a parseable config needs no backup"); + let _ = fs::remove_dir_all(config.parent().unwrap()); + } + fn tmp_path(name: &str) -> std::path::PathBuf { let dir = std::env::temp_dir().join(format!( "cg-jsonc-{}-{}-{}", diff --git a/crates/codegraph-cli/src/main.rs b/crates/codegraph-cli/src/main.rs index 1ea248a0..65ca7478 100644 --- a/crates/codegraph-cli/src/main.rs +++ b/crates/codegraph-cli/src/main.rs @@ -22,7 +22,7 @@ use codegraph_core::generated_header::detect_generated_file; use codegraph_core::logger::{LoggerConfig, init_logger}; use codegraph_core::node_id::hash_content; use codegraph_core::types::{Edge, ExtractionResult, FileRecord, Language, Node, NodeKind}; -use codegraph_extract::{ExtractOptions, detect_language_with, extract_source_with_observer}; +use codegraph_extract::{ExtractOptions, detect_language_with}; use codegraph_graph::graph::{GodotReach, GraphTraverser, group_definitions}; use codegraph_graph::query::{SearchOptions, search_nodes}; use codegraph_graph::{segment_match, segments}; @@ -209,6 +209,9 @@ fn cli_main() { if let Some(guidance) = index_removal_holder_guidance(&err) { eprint!("{guidance}"); } + if let Some(guidance) = codegraph_store::wsl_shared_index_guidance(err.as_ref()) { + eprint!("{guidance}"); + } std::process::exit(1); } } @@ -1438,6 +1441,22 @@ fn cmd_init( "init", )?; println!("Initialized in {}", project.display()); + // A fresh index on a Windows drive under WSL gets its own directory (#995); + // it is not the documented name, so say where it went and why. + if std::env::var_os("CODEGRAPH_DIR").is_none() + && let Ok(paths) = index_paths(&project) + && paths + .current_root() + .file_name() + .is_some_and(|name| name != codegraph_core::index_paths::DEFAULT_CURRENT_DIR) + { + println!( + "The index is in {}/: this project is on a Windows drive, so WSL keeps its own index \ + rather than share {}/ with CodeGraph on Windows. Set CODEGRAPH_DIR to choose the name yourself.", + codegraph_core::index_paths::WSL_CURRENT_DIR, + codegraph_core::index_paths::DEFAULT_CURRENT_DIR + ); + } print_index_result(&result); installer::run_install_local_targets(project, target) } @@ -1774,6 +1793,22 @@ fn cmd_sync(path: Option, quiet: bool, diagnostics: DiagnosticArgs) -> format_number(outcome.files_removed as i64), format_duration(outcome.duration_ms as i64) ); + // A sync that healed an interrupted index reports the sweep's work, so + // recovering references on unchanged files is not a no-op (#1360). + if outcome.pending_refs_processed > 0 { + let unresolved = if outcome.pending_refs_unresolved > 0 { + format!( + " ({} unresolved)", + format_number(outcome.pending_refs_unresolved as i64) + ) + } else { + String::new() + }; + println!( + "Resolved {} pending references{unresolved}", + format_number(outcome.pending_refs_resolved as i64) + ); + } for warning in &sync_warnings { eprintln!("warning: {warning}"); } @@ -1784,6 +1819,9 @@ fn cmd_sync(path: Option, quiet: bool, diagnostics: DiagnosticArgs) -> "filesReindexed": outcome.files_reindexed, "filesSkipped": outcome.files_skipped_unchanged, "filesRemoved": outcome.files_removed, + "pendingRefsProcessed": outcome.pending_refs_processed, + "pendingRefsResolved": outcome.pending_refs_resolved, + "pendingRefsUnresolved": outcome.pending_refs_unresolved, "warnings": sync_warnings, })); Ok(()) @@ -5301,6 +5339,7 @@ fn index_project_inner( let mut files_indexed = 0; let mut files_skipped = 0; let mut files_errored = 0; + let mut files_not_source = 0_u64; let mut warnings = Vec::new(); // Stream the graph to the store in capped batches instead of holding the whole @@ -5350,7 +5389,8 @@ fn index_project_inner( diagnostic_run.phase_start("parse_write"); let parse_started = std::time::Instant::now(); - type ParsePayload = (String, FileRecord, ExtractionResult); + // `None` for an MPEG transport stream named `.ts`: video, not source (#1910). + type ParsePayload = Option<(String, FileRecord, ExtractionResult)>; let schedule_tracker = tracker.clone(); let parse_tracker = tracker.clone(); let buffer_tracker = tracker.clone(); @@ -5363,55 +5403,34 @@ fn index_project_inner( let relative = &files[index]; let full = project.join(relative); - parse_tracker.stage(index, "metadata"); - let metadata = match fs::metadata(&full) - .with_context(|| format!("reading metadata for {}", full.display())) - { - Ok(metadata) => metadata, - Err(error) => { - parse_tracker.failed(index, &error); - return Err(error); - } - }; - parse_tracker.stage(index, "read"); - let source = match fs::read_to_string(&full) - .with_context(|| format!("reading source file {}", full.display())) - { - Ok(source) => source, - Err(error) => { - parse_tracker.failed(index, &error); - return Err(error); - } + let (metadata, source) = + match codegraph_extract::read_source_file(&full, relative, options.max_file_size) + .with_context(|| format!("reading source file {}", full.display())) + { + Ok(read) => read, + Err(error) => { + parse_tracker.failed(index, &error); + return Err(error); + } + }; + // A file over the limit was never read: its size stamp stands in for + // its content, in the hash and in the generated-file check. A video + // clip named `.ts` is not source at all, so it is not recorded. + let Some(hash_input) = source.hash_input() else { + return Ok(None); }; parse_tracker.stage(index, "prepare"); let language = detect_language_with(relative, &options.extensions); parse_tracker.file_info(index, metadata.len(), language); - let result = if metadata.len() > options.max_file_size { - ExtractionResult { - nodes: Vec::new(), - edges: Vec::new(), - unresolved_references: Vec::new(), - errors: vec![format!( - "File exceeds max size ({} > {}): {relative}", - metadata.len(), - options.max_file_size - )], - duration_ms: 0, - } - } else { - extract_source_with_observer( - relative, - &source, - None, - &options.extensions, - |stage| parse_tracker.extraction_stage(index, stage), - ) - }; + let result = + codegraph_extract::engine::extraction_of(relative, &source, &options, |stage| { + parse_tracker.extraction_stage(index, stage) + }); let file = FileRecord { path: relative.clone(), - content_hash: hash_content(&source), + content_hash: hash_content(&hash_input), language, size: metadata.len() as i64, modified_at: modified_millis(&metadata), @@ -5422,13 +5441,18 @@ fn index_project_inner( .filter(|node| node.file_path == *relative) .count() as i64, errors: result.errors.clone(), - generated: detect_generated_file(relative, &source), + generated: detect_generated_file(relative, &hash_input), }; parse_tracker.parsed(index, &result); - Ok((relative.clone(), file, result)) + Ok(Some((relative.clone(), file, result))) }, |buffered| buffer_tracker.buffered(buffered), - |index, (_relative, file, mut result)| { + |index, payload| { + let Some((_relative, file, mut result)) = payload else { + files_not_source += 1; + persist_tracker.persisted(index); + return Ok(()); + }; let fatal_errors = file .errors .iter() @@ -5480,7 +5504,7 @@ fn index_project_inner( }), ); - let scan_files = bar.position(); + let scan_files = bar.position() - files_not_source; finish_phase( &bar, &format!("Indexed {} files", format_number(scan_files as i64)), @@ -5536,13 +5560,13 @@ fn index_project_inner( diagnostic_run.phase_start("framework_extract"); let framework_started = std::time::Instant::now(); let pb = phase_spinner("Detecting frameworks", quiet); - let mut resolver = ReferenceResolver::new(project.to_string_lossy()); + let mut resolver = + ReferenceResolver::new(project.to_string_lossy()).with_max_file_size(options.max_file_size); // Detect frameworks then run their per-file extract (route/component/handler // nodes + refs) BEFORE resolution, mirroring the upstream tree-sitter.ts:4796-4819 // framework-extraction pass feeding the resolution pipeline. { - let context = - codegraph_resolve::StoreResolutionContext::new(&store, project.to_string_lossy()); + let context = resolver.store_context(&store); resolver.initialize(&context); } if resolver.has_framework_resolvers() { diff --git a/crates/codegraph-cli/src/structural_gate.rs b/crates/codegraph-cli/src/structural_gate.rs index e884c3f5..da9cd4e7 100644 --- a/crates/codegraph-cli/src/structural_gate.rs +++ b/crates/codegraph-cli/src/structural_gate.rs @@ -34,8 +34,10 @@ use regex::Regex; /// Structural keywords matched as EXACT words (boundary on both sides): short /// or ambiguous tokens where prefix matching would false-positive ("flow" in /// "flower", "path" in "pathological"). Grouped by language; a term appears once -/// even when several languages share it ("como" is Portuguese for how AND -/// unaccented-typed Spanish "cómo"). Verbatim from `directory.ts:255-310`. +/// even when several languages share it. Ambiguous everyday words like PT/ES +/// "como" and DE "wie" are excluded: the hook instead requires another strong +/// keyword, a verified code token, or indexed prose segments (upstream #1654). +/// Verbatim from `directory.ts:255-310`. const STRUCTURAL_WORDS: &[&str] = &[ // English — the pre-#1126 list minus what moved to STRUCTURAL_STEMS: the // bare-stem entries never matched their own derived forms (`\barchitect\b` @@ -55,7 +57,7 @@ const STRUCTURAL_WORDS: &[&str] = &[ "flux", "chemins?", "casse", - // Spanish (cómo/como=how, dónde/donde=where, flujo=flow, ruta/camino=path, + // Spanish (cómo=how, dónde/donde=where, flujo=flow, ruta/camino=path, // rompe=breaks, llaman / quién llama = call(s) — bare "llama" is excluded: // it's also the animal/model name in English prompts) "cómo", @@ -68,16 +70,13 @@ const STRUCTURAL_WORDS: &[&str] = &[ "llaman", "quién llama", "quien llama", - // Portuguese (como=how — also covers unaccented Spanish; onde=where, - // fluxo=flow, caminho=path) - "como", + // Portuguese (onde=where, fluxo=flow, caminho=path) "onde", "fluxos?", "caminhos?", - // German (wie=how, wo/woher/wohin=where, Pfad=path, Fluss/Ablauf=flow, + // German (wo/woher/wohin=where, Pfad=path, Fluss/Ablauf=flow, // bricht/kaputt=breaks, ruft=calls, hängt=depends — "hängt … von X ab" // splits the separable verb "abhängen", so the "abhäng" stem can't catch it) - "wie", "wo", "woher", "wohin", diff --git a/crates/codegraph-cli/tests/prompt_hook.rs b/crates/codegraph-cli/tests/prompt_hook.rs index c3367fb6..5943bd0e 100644 --- a/crates/codegraph-cli/tests/prompt_hook.rs +++ b/crates/codegraph-cli/tests/prompt_hook.rs @@ -292,6 +292,47 @@ fn gate_no_false_fire_on_ordinary_words() { } } +/// PT/ES "como" and DE "wie" are everyday words; alone they are not structural +/// evidence. Another structural keyword still fires (upstream #1654). +#[test] +fn everyday_como_and_wie_need_corroboration() { + let (_dir, project) = index_mini("como-wie"); + let hook = |q: &str| { + let (out, err, ok) = cli(&[ + "prompt-hook", + "--path", + project.to_str().unwrap(), + "--query", + q, + ]); + assert!(ok, "failed for {q:?}: stdout={out} stderr={err}"); + out + }; + for q in [ + "eu como pizza toda sexta", + "faz como a gente combinou ontem", + "commita isso como fix, nao como feat", + "hazlo como ayer", + "mach es wie gestern", + ] { + let out = hook(q); + assert!( + out.trim().is_empty(), + "everyday {q:?} must stay silent, got:\n{out}" + ); + } + for q in [ + "como funciona a máquina de estados?", + "wie funktioniert die Zustandsmaschine?", + ] { + let out = hook(q); + assert!( + out.contains(" Self { + let path = std::env::temp_dir().join(format!( + "codegraph-cli-source-reads-{tag}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + fs::create_dir_all(&path).unwrap(); + Self(path) + } + + fn write(&self, relative: &str, bytes: impl AsRef<[u8]>) { + let path = self.0.join(relative); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, bytes).unwrap(); + } +} + +impl Drop for TestDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn run(cwd: &Path, args: &[&str]) -> (String, String, bool) { + let output = Command::new(env!("CARGO_BIN_EXE_codegraph")) + .args(args) + .current_dir(cwd) + .env("CODEGRAPH_NO_DAEMON", "1") + .env("CODEGRAPH_NO_WATCH", "1") + .output() + .expect("run codegraph"); + ( + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + output.status.success(), + ) +} + +/// 64 transport-stream packets: the 0x47 sync byte, then a deterministic +/// pseudo-random payload standing in for compressed video. +fn mpeg_ts_clip() -> Vec { + let mut state = 0x2545_f491_u32; + let mut payload = move || { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state.to_le_bytes()[0] + }; + (0..64) + .flat_map(|_| { + std::iter::once(0x47) + .chain((0..187).map(|_| payload())) + .collect::>() + }) + .collect() +} + +const ONE_MIB: usize = 1024 * 1024; + +fn file_paths(store: &Store) -> Vec { + store + .all_files() + .unwrap() + .into_iter() + .map(|file| file.path) + .collect() +} + +#[test] +fn index_and_sync_treat_video_latin1_and_oversize_files_like_upstream() { + let dir = TestDir::new("index"); + dir.write("src/app.ts", "export function app() { return 1; }\n"); + dir.write("testdata/clip.ts", mpeg_ts_clip()); + dir.write( + "legacy/latin1.c", + b"/* caf\xe9 */\nint legacy_entry(void) { return 0; }\n", + ); + let oversize = format!("export const big = 1;\n{}", " ".repeat(ONE_MIB)); + dir.write("big/bundle.ts", &oversize); + let path = dir.0.to_str().unwrap(); + + let (stdout, stderr, ok) = run(&dir.0, &["init", path]); + assert!(ok, "init failed: {stdout} {stderr}"); + // The oversize file is recorded as skipped; the clip is not counted at all. + assert!(stdout.contains("Indexed 2 files"), "stdout={stdout}"); + assert!(stdout.contains("Skipped 1 files"), "stdout={stdout}"); + + let db = dir.0.join(".codegraph").join("codegraph.db"); + { + let store = Store::open(&db).unwrap(); + assert_eq!( + file_paths(&store), + vec!["big/bundle.ts", "legacy/latin1.c", "src/app.ts"] + ); + let bundle = store.file_by_path("big/bundle.ts").unwrap().unwrap(); + assert_eq!( + bundle.content_hash, + hash_content(&format!("codegraph:oversize:{}", oversize.len())) + ); + assert!( + store + .nodes_by_name("legacy_entry") + .unwrap() + .iter() + .any(|node| node.file_path == "legacy/latin1.c") + ); + } + + let (stdout, stderr, ok) = run(&dir.0, &["status", path, "--json"]); + assert!(ok, "status failed: {stdout} {stderr}"); + let status: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + assert_eq!( + status["pendingChanges"]["addedPaths"], + serde_json::json!([]), + "an untracked clip is not pending source: {stdout}" + ); + + // A same-size rewrite of an oversize file is not a change; a tracked `.ts` + // that became a clip is removed like a deletion. + let rewritten = format!("export const BIG = 2;\n{}", " ".repeat(ONE_MIB)); + assert_eq!(rewritten.len(), oversize.len()); + dir.write("big/bundle.ts", &rewritten); + dir.write("src/app.ts", mpeg_ts_clip()); + let (stdout, stderr, ok) = run(&dir.0, &["sync", path]); + assert!(ok, "sync failed: {stdout} {stderr}"); + { + let store = Store::open(&db).unwrap(); + assert_eq!(file_paths(&store), vec!["big/bundle.ts", "legacy/latin1.c"]); + } + let (stdout, stderr, ok) = run(&dir.0, &["status", path, "--json"]); + assert!(ok, "status failed: {stdout} {stderr}"); + let status: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + for kind in ["addedPaths", "modifiedPaths", "removedPaths"] { + assert_eq!( + status["pendingChanges"][kind], + serde_json::json!([]), + "{kind}: {stdout}" + ); + } + + // Crossing the limit is a change in either direction. + dir.write("big/bundle.ts", "export const small = 3;\n"); + let (stdout, stderr, ok) = run(&dir.0, &["sync", path]); + assert!(ok, "sync failed: {stdout} {stderr}"); + let store = Store::open(&db).unwrap(); + assert!( + store + .nodes_by_name("small") + .unwrap() + .iter() + .any(|node| node.file_path == "big/bundle.ts") + ); +} diff --git a/crates/codegraph-cli/tests/sync_recovery_report.rs b/crates/codegraph-cli/tests/sync_recovery_report.rs new file mode 100644 index 00000000..3f9d317e --- /dev/null +++ b/crates/codegraph-cli/tests/sync_recovery_report.rs @@ -0,0 +1,116 @@ +//! A sync that heals an interrupted index says so (upstream v1.6.1 #1360): the +//! orphan sweep's work is reported instead of reading as a no-op. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use codegraph_core::types::{EdgeKind, Language, UnresolvedRef}; +use codegraph_store::Store; + +struct TestDir(PathBuf); + +impl Drop for TestDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn run(cwd: &Path, args: &[&str]) -> (String, String, bool) { + let output = Command::new(env!("CARGO_BIN_EXE_codegraph")) + .args(args) + .current_dir(cwd) + .env("CODEGRAPH_NO_DAEMON", "1") + .env("CODEGRAPH_NO_WATCH", "1") + .output() + .expect("run codegraph"); + ( + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + output.status.success(), + ) +} + +#[test] +fn a_healing_sync_reports_the_pending_references_it_resolved() { + let dir = TestDir(std::env::temp_dir().join(format!( + "codegraph-cli-sync-recovery-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + ))); + fs::create_dir_all(dir.0.join("src")).unwrap(); + fs::write( + dir.0.join("src/math.ts"), + "export function add(a: number, b: number): number { return a + b; }\n", + ) + .unwrap(); + fs::write( + dir.0.join("src/app.ts"), + "import { add } from './math';\nexport function run(): number { return add(1, 2); }\n", + ) + .unwrap(); + let path = dir.0.to_str().unwrap(); + let (stdout, stderr, ok) = run(&dir.0, &["init", path]); + assert!(ok, "init failed: {stdout} {stderr}"); + + let (stdout, stderr, ok) = run(&dir.0, &["sync", path]); + assert!(ok, "sync failed: {stdout} {stderr}"); + assert!( + !stdout.contains("pending references"), + "a healthy sync sweeps nothing: {stdout}" + ); + + // Simulate an interrupted resolution pass: the resolved call edge is gone, + // its reference is parked again, and the incomplete-resolution marker is set. + { + let mut store = Store::open(&dir.0.join(".codegraph").join("codegraph.db")).unwrap(); + let caller = store + .nodes_by_file_path("src/app.ts") + .unwrap() + .into_iter() + .find(|node| node.name == "run") + .expect("run node"); + store.delete_resolved_edges_from_file("src/app.ts").unwrap(); + store + .insert_unresolved_refs(&[ + UnresolvedRef { + id: None, + from_node_id: caller.id.clone(), + reference_name: "add".to_string(), + reference_kind: EdgeKind::Calls, + line: 2, + col: 0, + candidates: None, + file_path: "src/app.ts".to_string(), + language: Language::TypeScript, + is_function_ref: false, + reference_subkind: None, + }, + UnresolvedRef { + id: None, + from_node_id: caller.id, + reference_name: "missingEverywhere".to_string(), + reference_kind: EdgeKind::Calls, + line: 2, + col: 0, + candidates: None, + file_path: "src/app.ts".to_string(), + language: Language::TypeScript, + is_function_ref: false, + reference_subkind: None, + }, + ]) + .unwrap(); + store.set_resolution_incomplete().unwrap(); + } + + let (stdout, stderr, ok) = run(&dir.0, &["sync", path]); + assert!(ok, "sync failed: {stdout} {stderr}"); + assert!( + stdout.contains("Resolved 1 pending references (1 unresolved)"), + "stdout={stdout}" + ); +} diff --git a/crates/codegraph-core/src/config.rs b/crates/codegraph-core/src/config.rs index f90eb566..13eff207 100644 --- a/crates/codegraph-core/src/config.rs +++ b/crates/codegraph-core/src/config.rs @@ -92,11 +92,33 @@ pub struct IndexingConfig { pub deprioritize: Vec, } +/// The largest source file extraction parses unless a project raises +/// `indexing.max_file_size`; resolution reads no file extraction would skip. +// upstream extraction/index.ts:101, src/file-limits.ts (`MAX_SOURCE_FILE_SIZE_BYTES`) +// Skip files larger than this (bytes). Generated bundles, minified JS, and +// vendored blobs blow the WASM heap. 1 MB covers essentially all hand-written source. +pub const DEFAULT_MAX_FILE_SIZE: u64 = 1024 * 1024; + fn default_max_file_size() -> u64 { - // upstream extraction/index.ts:101 - // Skip files larger than this (bytes). Generated bundles, minified JS, and - // vendored blobs blow the WASM heap. 1 MB covers essentially all hand-written source. - 1024 * 1024 + DEFAULT_MAX_FILE_SIZE +} + +/// Whether `relative_dir` — a project-relative, `/`-separated directory path — +/// is a `build` directory that names a JVM package under a conventional source +/// root (`**/src//{java,kotlin,scala}/**/build`) rather than build +/// output. The default `build` prune exempts exactly that directory, not its +/// subtree: the other defaults still apply inside it (upstream #1642). +pub fn is_jvm_source_build_dir(relative_dir: &str) -> bool { + let segments = relative_dir.split('/').collect::>(); + let Some(last) = segments.len().checked_sub(1) else { + return false; + }; + segments[last] == "build" + && (0..last).any(|index| { + segments[index] == "src" + && index + 2 < last + && matches!(segments[index + 2], "java" | "kotlin" | "scala") + }) } fn default_ignore_dirs() -> Vec { @@ -377,6 +399,29 @@ mod tests { } } + #[test] + fn jvm_source_build_dirs_are_package_segments_not_output() { + for kept in [ + "src/main/java/build", + "src/main/java/com/acme/build", + "module/src/test/kotlin/build", + "src/androidTest/scala/x/build", + ] { + assert!(is_jvm_source_build_dir(kept), "{kept}"); + } + for pruned in [ + "build", + "app/build", + "src/build", + "src/main/build", + "src/main/java", + "src/main/resources/build", + "lib/src/main/java/build/inner", + ] { + assert!(!is_jvm_source_build_dir(pruned), "{pruned}"); + } + } + #[test] fn test_default_config_parses() { let toml_str = r#" diff --git a/crates/codegraph-core/src/index_paths.rs b/crates/codegraph-core/src/index_paths.rs index daa143fb..8dd63b43 100644 --- a/crates/codegraph-core/src/index_paths.rs +++ b/crates/codegraph-core/src/index_paths.rs @@ -34,6 +34,10 @@ use thiserror::Error; /// from [`IndexPaths::resolve`]. pub const DEFAULT_CURRENT_DIR: &str = ".codegraph"; +/// The index-root name WSL gives a fresh project on a Windows drive, so it +/// never shares one index with CodeGraph on Windows (upstream #995). +pub const WSL_CURRENT_DIR: &str = ".codegraph-wsl"; + /// Version byte of the binary payload hashed into the project identity. Bump /// only with a deliberate, reviewed identity-format change. const IDENTITY_PAYLOAD_VERSION: u8 = 1; @@ -117,7 +121,10 @@ impl IndexPaths { let project_identity = physical_identity(&project)?; - let directory_name = configured_directory_name(codegraph_dir)?; + let directory_name = match codegraph_dir { + None => default_directory_name(&project, crate::wsl::is_wsl_windows_drive(&project)), + Some(_) => configured_directory_name(codegraph_dir)?, + }; let candidate = lexical_normalize(&project.join(directory_name)); reject_project_alias(&project, &candidate, directory_name)?; let current_root = physical_normalize(&project, &candidate)?; @@ -259,6 +266,32 @@ fn lexical_normalize(path: &Path) -> PathBuf { out } +/// The index-root name for `project` with no `CODEGRAPH_DIR` set. Off a WSL +/// Windows drive it is always [`DEFAULT_CURRENT_DIR`]. On one, Windows-native +/// CodeGraph opens `.codegraph` in the same tree and SQLite's locking does not +/// hold across the 9p/DrvFs bridge, so sharing one index fails with "disk I/O +/// error" (upstream #995): +/// 1. `.codegraph-wsl/` exists: it, even once Windows builds a `.codegraph`; +/// 2. `.codegraph/codegraph.db` exists: `.codegraph`, so an index built before +/// this default is kept rather than silently rebuilt elsewhere; +/// 3. neither: `.codegraph-wsl`, so a fresh WSL index never shares. +fn default_directory_name(project: &Path, wsl_windows_drive: bool) -> &'static str { + if !wsl_windows_drive { + return DEFAULT_CURRENT_DIR; + } + if project.join(WSL_CURRENT_DIR).is_dir() { + return WSL_CURRENT_DIR; + } + if project + .join(DEFAULT_CURRENT_DIR) + .join("codegraph.db") + .exists() + { + return DEFAULT_CURRENT_DIR; + } + WSL_CURRENT_DIR +} + fn configured_directory_name(codegraph_dir: Option<&str>) -> Result<&str, IndexPathsError> { let Some(raw_value) = codegraph_dir else { return Ok(DEFAULT_CURRENT_DIR); @@ -692,6 +725,30 @@ mod tests { let _ = std::fs::remove_dir_all(&cache); } + #[test] + fn wsl_windows_drive_projects_default_to_their_own_index_root() { + let fresh = temp_dir("wsl-fresh"); + assert_eq!(default_directory_name(&fresh, false), DEFAULT_CURRENT_DIR); + assert_eq!(default_directory_name(&fresh, true), WSL_CURRENT_DIR); + + let legacy = temp_dir("wsl-legacy"); + std::fs::create_dir_all(legacy.join(DEFAULT_CURRENT_DIR)).unwrap(); + assert_eq!( + default_directory_name(&legacy, true), + WSL_CURRENT_DIR, + "an index root without a database is not an index to keep" + ); + std::fs::write(legacy.join(DEFAULT_CURRENT_DIR).join("codegraph.db"), b"").unwrap(); + assert_eq!(default_directory_name(&legacy, true), DEFAULT_CURRENT_DIR); + + std::fs::create_dir_all(legacy.join(WSL_CURRENT_DIR)).unwrap(); + assert_eq!( + default_directory_name(&legacy, true), + WSL_CURRENT_DIR, + "WSL keeps its own index once it has one" + ); + } + #[test] fn configured_directory_name_stays_local_to_each_project() { let project_a = temp_dir("collide-a"); diff --git a/crates/codegraph-core/src/lib.rs b/crates/codegraph-core/src/lib.rs index cef3be8b..bfcb7dda 100644 --- a/crates/codegraph-core/src/lib.rs +++ b/crates/codegraph-core/src/lib.rs @@ -8,8 +8,10 @@ pub mod generated_header; pub mod index_paths; pub mod logger; pub mod node_id; +pub mod source_file; pub mod traits; pub mod types; +pub mod wsl; pub use errors::{CodeGraphError, Result}; pub use index_paths::{IndexPaths, IndexPathsError}; diff --git a/crates/codegraph-core/src/source_file.rs b/crates/codegraph-core/src/source_file.rs new file mode 100644 index 00000000..87c15b49 --- /dev/null +++ b/crates/codegraph-core/src/source_file.rs @@ -0,0 +1,278 @@ +//! Reading a project source file by path, the way every indexing pass does +//! (upstream v1.6.1 `file-limits.ts` and `grammars.ts`, #1910): never more than +//! the size limit plus one byte, with a size stamp standing in for a file over +//! it, and an MPEG transport stream named `.ts` recognised as video rather than +//! TypeScript before anything parses it. + +use std::borrow::Cow; +use std::fs; +use std::io::{self, Read}; +use std::path::Path; + +/// MPEG transport stream: fixed 188-byte packets, each opening with 0x47. +const MPEG_TS_PACKET_SIZE: usize = 188; +const MPEG_TS_SYNC_BYTE: u8 = 0x47; +/// Consecutive packets whose sync byte must line up before a file counts as +/// video — 3 KB of head. A shorter stream is cheap to parse anyway. +const MPEG_TS_MIN_PACKETS: usize = 16; +/// How much of a file's head [`is_mpeg_transport_stream`] looks at. +const MPEG_TS_SNIFF_BYTES: usize = MPEG_TS_PACKET_SIZE * MPEG_TS_MIN_PACKETS; + +/// A source file as an indexing pass sees it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SourceText { + /// The file's text, within the limit. Bytes that are not UTF-8 become + /// U+FFFD, as upstream's `toString('utf8')` decodes them, so a Latin-1 or + /// binary file is indexed like any other instead of failing the pass. + Text(String), + /// A file over the limit, never read: its size in bytes stands in for it. + Oversize(u64), + /// An MPEG transport stream named `.ts`: video, not source. + MpegTransportStream, +} + +impl SourceText { + /// What the index hashes for this file: its text within the limit, its + /// size stamp over it, and nothing for a file that is not source. + pub fn hash_input(&self) -> Option> { + match self { + Self::Text(text) => Some(Cow::Borrowed(text)), + Self::Oversize(size) => Some(Cow::Owned(oversize_stamp(*size))), + Self::MpegTransportStream => None, + } + } +} + +/// What stands in for the content of a file over the size limit. Such a file +/// is never parsed, so the stamp is a function of its size alone: a same-size +/// rewrite is not a change (nothing is indexed from it), while crossing the +/// limit in either direction is. +pub fn oversize_stamp(size: u64) -> String { + format!("codegraph:oversize:{size}") +} + +/// Read the source file at `path`, named `relative` in the project, without +/// ever holding more than `max_bytes` plus one byte. The size is checked by +/// `stat` before the file is opened, again on the open handle, and once more +/// after a read that itself stops one byte past the limit, so a file growing +/// meanwhile is still stamped rather than decoded. Returns the metadata the +/// file's record is built from. A path that is not a regular file is an error. +pub fn read_source_file( + path: &Path, + relative: &str, + max_bytes: u64, +) -> io::Result<(fs::Metadata, SourceText)> { + let initial = regular_file_metadata(fs::metadata(path)?)?; + if initial.len() > max_bytes { + let size = initial.len(); + return Ok((initial, SourceText::Oversize(size))); + } + let mut file = fs::File::open(path)?; + let opened = regular_file_metadata(file.metadata()?)?; + if opened.len() > max_bytes { + let size = opened.len(); + return Ok((opened, SourceText::Oversize(size))); + } + let mut bytes = Vec::with_capacity(usize::try_from(opened.len()).unwrap_or(0)); + (&mut file) + .take(max_bytes.saturating_add(1)) + .read_to_end(&mut bytes)?; + let read = file.metadata()?; + let read_len = bytes.len() as u64; + if read_len > max_bytes || read.len() > max_bytes { + let size = read.len().max(read_len); + return Ok((read, SourceText::Oversize(size))); + } + if has_mpeg_ts_extension(relative) && is_mpeg_transport_stream(&bytes) { + return Ok((read, SourceText::MpegTransportStream)); + } + let text = String::from_utf8(bytes) + .unwrap_or_else(|error| String::from_utf8_lossy(error.as_bytes()).into_owned()); + Ok((read, SourceText::Text(text))) +} + +/// Whether [`read_source_file`] would see source at `path`: everything except +/// an MPEG transport stream named `.ts` within the limit (a file over it is +/// still recorded, by its size stamp). Reads at most the head the check needs, +/// and only for a `.ts` name, so a pending-change scan stays cheap. +pub fn is_source_file(path: &Path, relative: &str, max_bytes: u64) -> io::Result { + if !has_mpeg_ts_extension(relative) { + return Ok(true); + } + let metadata = regular_file_metadata(fs::metadata(path)?)?; + if metadata.len() > max_bytes { + return Ok(true); + } + let mut head = Vec::with_capacity(MPEG_TS_SNIFF_BYTES); + fs::File::open(path)? + .take(MPEG_TS_SNIFF_BYTES as u64) + .read_to_end(&mut head)?; + Ok(!is_mpeg_transport_stream(&head)) +} + +fn regular_file_metadata(metadata: fs::Metadata) -> io::Result { + if metadata.is_file() { + Ok(metadata) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidInput, + "source path is not a regular file", + )) + } +} + +/// Whether `relative` carries the one extension MPEG-TS shares with a language. +fn has_mpeg_ts_extension(relative: &str) -> bool { + let bytes = relative.as_bytes(); + bytes.len() > 3 && bytes[bytes.len() - 3..].eq_ignore_ascii_case(b".ts") +} + +/// Whether a file starting with `head` is an MPEG transport stream, the other +/// thing a `.ts` file can be: tree-sitter takes tens of seconds on a clip for +/// zero symbols. Both conditions are required: +/// 1. the sync byte 0x47 opens each of the first [`MPEG_TS_MIN_PACKETS`] +/// 188-byte packets, as it opens every packet of a stream; +/// 2. the head is binary: at least 1/64 of it is control bytes other than +/// the whitespace ones, as any compressed payload carries. +/// +/// 0x47 is `G`, so source could satisfy (1) by putting one at every stride; +/// no source carries the dozens of control bytes (2) asks for. +fn is_mpeg_transport_stream(head: &[u8]) -> bool { + let head = &head[..head.len().min(MPEG_TS_SNIFF_BYTES)]; + let last_sync = MPEG_TS_PACKET_SIZE * (MPEG_TS_MIN_PACKETS - 1); + if head.len() <= last_sync { + return false; + } + if (0..=last_sync) + .step_by(MPEG_TS_PACKET_SIZE) + .any(|offset| head[offset] != MPEG_TS_SYNC_BYTE) + { + return false; + } + let control = head + .iter() + .filter(|&&byte| byte < 0x20 && !(0x09..=0x0d).contains(&byte)) + .count(); + control * 64 >= head.len() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_dir(tag: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!( + "codegraph-source-file-{tag}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + fs::create_dir_all(&dir).expect("create temp dir"); + dir + } + + /// `packets` transport-stream packets with a deterministic binary payload. + fn clip(packets: usize) -> Vec { + let mut state = 0x9e37_79b9_u32; + let mut payload = move || { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state.to_le_bytes()[0] + }; + (0..packets) + .flat_map(|_| { + std::iter::once(MPEG_TS_SYNC_BYTE) + .chain((0..187).map(|_| payload())) + .collect::>() + }) + .collect() + } + + #[test] + fn a_transport_stream_needs_sixteen_aligned_packets_and_a_binary_head() { + assert!(is_mpeg_transport_stream(&clip(16))); + assert!(is_mpeg_transport_stream(&clip(64))); + assert!(!is_mpeg_transport_stream(&clip(15)), "too short to judge"); + let mut misaligned = clip(64); + misaligned[188 * 7] = b'x'; + assert!(!is_mpeg_transport_stream(&misaligned)); + + // TypeScript with a `G` at every stride and one raw NUL is still text. + let mut source = Vec::new(); + for _ in 0..16 { + let mut line = b"G".to_vec(); + line.extend(std::iter::repeat_n(b'a', 186)); + line.push(b'\n'); + source.extend(line); + } + source[100] = 0; + assert!(!is_mpeg_transport_stream(&source)); + } + + #[test] + fn the_extension_check_is_the_last_three_characters_case_blind() { + assert!(has_mpeg_ts_extension("clips/a.ts")); + assert!(has_mpeg_ts_extension("CLIP.TS")); + assert!(has_mpeg_ts_extension("types.d.ts")); + assert!(!has_mpeg_ts_extension(".ts")); + assert!(!has_mpeg_ts_extension("a.tsx")); + assert!(!has_mpeg_ts_extension("a.mts")); + } + + #[test] + fn reads_text_lossily_stamps_oversize_and_recognises_clips() { + let dir = temp_dir("read"); + fs::write(dir.join("app.ts"), "export const a = 1;\n").unwrap(); + fs::write(dir.join("latin1.c"), b"/* caf\xe9 */\n").unwrap(); + fs::write(dir.join("clip.ts"), clip(64)).unwrap(); + fs::write(dir.join("clip.bin.c"), clip(64)).unwrap(); + fs::write(dir.join("big.ts"), "a".repeat(65)).unwrap(); + fs::write(dir.join("edge.ts"), "a".repeat(64)).unwrap(); + fs::create_dir(dir.join("folder.ts")).unwrap(); + + let read = |name: &str| read_source_file(&dir.join(name), name, 64).map(|(_, text)| text); + assert_eq!( + read("app.ts").unwrap(), + SourceText::Text("export const a = 1;\n".to_string()) + ); + assert_eq!( + read("latin1.c").unwrap(), + SourceText::Text("/* caf\u{fffd} */\n".to_string()) + ); + assert_eq!(read("big.ts").unwrap(), SourceText::Oversize(65)); + assert_eq!(read("edge.ts").unwrap(), SourceText::Text("a".repeat(64))); + assert!(read("folder.ts").is_err()); + assert!(read("missing.ts").is_err()); + + let read_large = + |name: &str| read_source_file(&dir.join(name), name, 1024 * 1024).map(|(_, text)| text); + assert_eq!( + read_large("clip.ts").unwrap(), + SourceText::MpegTransportStream + ); + // Only a `.ts` name can be a transport stream; other binaries decode. + assert!(matches!( + read_large("clip.bin.c").unwrap(), + SourceText::Text(_) + )); + + assert_eq!( + SourceText::Oversize(65).hash_input().as_deref(), + Some("codegraph:oversize:65") + ); + assert_eq!(SourceText::MpegTransportStream.hash_input(), None); + + let is_source = |name: &str, max: u64| is_source_file(&dir.join(name), name, max).unwrap(); + assert!(!is_source("clip.ts", 1024 * 1024)); + assert!( + is_source("clip.ts", 64), + "an oversize clip is recorded by its stamp" + ); + assert!(is_source("clip.bin.c", 1024 * 1024)); + assert!(is_source("app.ts", 1024 * 1024)); + fs::remove_dir_all(&dir).ok(); + } +} diff --git a/crates/codegraph-core/src/wsl.rs b/crates/codegraph-core/src/wsl.rs new file mode 100644 index 00000000..5fa182fa --- /dev/null +++ b/crates/codegraph-core/src/wsl.rs @@ -0,0 +1,55 @@ +//! WSL detection, shared by the index-root default (upstream #995) and the +//! watcher's `/mnt/` opt-out. + +use std::path::Path; + +/// Whether this process runs under WSL: Linux, and `WSL_DISTRO_NAME` / +/// `WSL_INTEROP` set or `/proc/version` naming Microsoft or WSL. +pub fn is_wsl() -> bool { + if !cfg!(target_os = "linux") { + return false; + } + if std::env::var_os("WSL_DISTRO_NAME").is_some() || std::env::var_os("WSL_INTEROP").is_some() { + return true; + } + std::fs::read_to_string("/proc/version") + .map(|version| { + let version = version.to_ascii_lowercase(); + version.contains("microsoft") || version.contains("wsl") + }) + .unwrap_or(false) +} + +/// Whether `path` is a Windows drive as WSL mounts it (`/mnt/c`, +/// `/mnt/d/project`). A pure path check. +pub fn is_windows_drive_mount(path: &Path) -> bool { + let normalized = path.to_string_lossy().replace('\\', "/"); + let mut parts = normalized.split('/'); + matches!( + (parts.next(), parts.next(), parts.next()), + (Some(""), Some("mnt"), Some(drive)) + if drive.len() == 1 && drive.as_bytes()[0].is_ascii_alphabetic() + ) +} + +/// Whether `path` is on a Windows drive under WSL, where Windows-native +/// CodeGraph opens the same tree. Off a `/mnt/` path this never reads +/// the environment or `/proc/version`. +pub fn is_wsl_windows_drive(path: &Path) -> bool { + is_windows_drive_mount(path) && is_wsl() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn windows_drive_mounts_are_single_letter_mnt_paths() { + assert!(is_windows_drive_mount(Path::new("/mnt/c"))); + assert!(is_windows_drive_mount(Path::new("/mnt/D/project"))); + assert!(!is_windows_drive_mount(Path::new("/mnt/abc/project"))); + assert!(!is_windows_drive_mount(Path::new("/home/user/project"))); + assert!(!is_windows_drive_mount(Path::new("/mnt"))); + assert!(!is_wsl_windows_drive(Path::new("/home/user/project"))); + } +} diff --git a/crates/codegraph-daemon/src/lock.rs b/crates/codegraph-daemon/src/lock.rs index f9740283..23f82f8d 100644 --- a/crates/codegraph-daemon/src/lock.rs +++ b/crates/codegraph-daemon/src/lock.rs @@ -14,6 +14,13 @@ use crate::process::is_process_alive; const EMPTY_RETRY_DELAY: Duration = Duration::from_millis(20); +/// Waits between retries of a pid-file replace on Windows, doubling to a cap: +/// eight attempts wait about 1.6s in all, well inside a launcher's connect +/// window (upstream #2053, #1773). Windows refuses to replace a file another +/// process holds open without sharing deletion — an antivirus scan, an indexer +/// — for a moment, and for longer on a loaded machine. +const LOCK_REPLACE_RETRY_DELAYS_MS: [u64; 7] = [25, 50, 100, 200, 400, 400, 400]; + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct DaemonLockInfo { @@ -85,9 +92,17 @@ pub fn try_acquire_daemon_lock(project_root: &Path) -> Result { .create_new(true) .open(&pid_path) { - Ok(_placeholder) => { - fs::rename(&tmp, &pid_path) - .with_context(|| format!("publishing daemon lock {}", pid_path.display()))?; + Ok(placeholder) => { + drop(placeholder); + replace_owned_lock( + &pid_path, + &tmp, + "", + is_transient_sharing_violation, + |from, to| fs::rename(from, to), + thread::sleep, + ) + .with_context(|| format!("publishing daemon lock {}", pid_path.display()))?; true } Err(err) if err.kind() == ErrorKind::AlreadyExists => { @@ -118,9 +133,61 @@ pub fn rewrite_lock_socket_path(pid_path: &Path, socket_path: &Path) -> Result<( let mut info = decode_lock_info(&raw) .with_context(|| format!("decoding daemon lock {}", pid_path.display()))?; info.socket_path = socket_path.to_path_buf(); - fs::write(pid_path, encode_lock_info(&info)?) - .with_context(|| format!("rewriting daemon lock {}", pid_path.display()))?; - Ok(()) + // A complete temp record renamed over the lock, so a reader never sees a + // truncated one, and only while the lock still holds the record read above. + let tmp = pid_path.with_extension(format!("pid.{}.bound", process::id())); + fs::write(&tmp, encode_lock_info(&info)?) + .with_context(|| format!("writing temp daemon lock {}", tmp.display()))?; + replace_owned_lock( + pid_path, + &tmp, + &raw, + is_transient_sharing_violation, + |from, to| fs::rename(from, to), + thread::sleep, + ) + .with_context(|| format!("rewriting daemon lock {}", pid_path.display())) +} + +/// Replace `pid_path` with the fully written `tmp` while `pid_path` still holds +/// `expected`. A transient failure (`is_transient`) is retried per +/// [`LOCK_REPLACE_RETRY_DELAYS_MS`], re-checking ownership before every +/// attempt, so a retry never replaces a record whose owner changed meanwhile; +/// any other failure aborts at once. `tmp` is removed unless it was renamed. +fn replace_owned_lock( + pid_path: &Path, + tmp: &Path, + expected: &str, + is_transient: impl Fn(&std::io::Error) -> bool, + mut rename: impl FnMut(&Path, &Path) -> std::io::Result<()>, + mut wait: impl FnMut(Duration), +) -> Result<()> { + let outcome = (|| { + let mut delays = LOCK_REPLACE_RETRY_DELAYS_MS.iter(); + loop { + let current = fs::read_to_string(pid_path).unwrap_or_default(); + if current != expected { + anyhow::bail!("lost daemon lock ownership of {}", pid_path.display()); + } + match rename(tmp, pid_path) { + Ok(()) => return Ok(()), + Err(error) if is_transient(&error) => match delays.next() { + Some(delay) => wait(Duration::from_millis(*delay)), + None => return Err(error.into()), + }, + Err(error) => return Err(error.into()), + } + } + })(); + let _ = fs::remove_file(tmp); + outcome +} + +/// A Windows sharing failure another process's open handle causes: +/// `ERROR_ACCESS_DENIED`, `ERROR_SHARING_VIOLATION` or `ERROR_LOCK_VIOLATION`. +/// Never transient elsewhere. +fn is_transient_sharing_violation(error: &std::io::Error) -> bool { + cfg!(windows) && matches!(error.raw_os_error(), Some(5 | 32 | 33)) } pub fn clear_stale_daemon_lock(pid_path: &Path, expected_dead_pid: Option) -> bool { @@ -412,6 +479,123 @@ fn now_millis() -> u128 { mod tests { use super::*; + fn sharing_violation() -> std::io::Error { + std::io::Error::from_raw_os_error(32) + } + + /// The replace a lock publish ends with retries a transient sharing + /// violation, re-checking ownership first, and never touches a record + /// whose owner changed (upstream #2053, #1773). + #[test] + fn lock_replace_retries_transient_failures_while_the_record_is_ours() { + let base = temp_base("replace-retry"); + let pid_path = base.join("daemon.pid"); + let tmp = base.join("daemon.pid.tmp"); + fs::create_dir_all(&base).unwrap(); + fs::write(&pid_path, "ours").unwrap(); + fs::write(&tmp, "next").unwrap(); + + let mut failures = 3; + let mut waits = Vec::new(); + replace_owned_lock( + &pid_path, + &tmp, + "ours", + |_| true, + |from, to| { + if failures > 0 { + failures -= 1; + return Err(sharing_violation()); + } + fs::rename(from, to) + }, + |delay| waits.push(delay.as_millis()), + ) + .expect("transient failures are retried"); + assert_eq!(fs::read_to_string(&pid_path).unwrap(), "next"); + assert!(!tmp.exists()); + assert_eq!(waits, vec![25, 50, 100]); + + // A failure that never clears gives up after the whole budget. + fs::write(&tmp, "later").unwrap(); + let mut attempts = 0; + let mut waits = Vec::new(); + let result = replace_owned_lock( + &pid_path, + &tmp, + "next", + |_| true, + |_, _| { + attempts += 1; + Err(sharing_violation()) + }, + |delay| waits.push(delay.as_millis()), + ); + assert!(result.is_err()); + assert_eq!(attempts, 8); + assert_eq!(waits, vec![25, 50, 100, 200, 400, 400, 400]); + assert!(!tmp.exists(), "the temp record is cleaned up"); + assert_eq!(fs::read_to_string(&pid_path).unwrap(), "next"); + let _ = fs::remove_dir_all(&base); + } + + #[test] + fn lock_replace_stops_on_lost_ownership_and_on_non_transient_errors() { + let base = temp_base("replace-stop"); + let pid_path = base.join("daemon.pid"); + let tmp = base.join("daemon.pid.tmp"); + fs::create_dir_all(&base).unwrap(); + fs::write(&pid_path, "ours").unwrap(); + fs::write(&tmp, "next").unwrap(); + + let mut attempts = 0; + let result = replace_owned_lock( + &pid_path, + &tmp, + "ours", + |_| true, + |_, to| { + attempts += 1; + // Another daemon takes the lock while this one waits. + fs::write(to, "theirs").unwrap(); + Err(sharing_violation()) + }, + |_| {}, + ); + assert!(result.is_err()); + assert_eq!(attempts, 1, "a changed owner stops the retry"); + assert_eq!(fs::read_to_string(&pid_path).unwrap(), "theirs"); + + fs::write(&tmp, "next").unwrap(); + let mut attempts = 0; + let result = replace_owned_lock( + &pid_path, + &tmp, + "theirs", + |_| false, + |_, _| { + attempts += 1; + Err(std::io::Error::other("disk full")) + }, + |_| {}, + ); + assert!(result.is_err()); + assert_eq!(attempts, 1, "a non-transient error is not retried"); + assert!(!tmp.exists()); + let _ = fs::remove_dir_all(&base); + } + + #[test] + fn only_windows_sharing_errors_are_transient() { + assert_eq!( + is_transient_sharing_violation(&sharing_violation()), + cfg!(windows) + ); + assert!(!is_transient_sharing_violation(&std::io::Error::other( + "disk full" + ))); + } + #[test] fn legacy_plain_pid_decodes() { let info = decode_lock_info("1234\n").expect("pid decodes"); diff --git a/crates/codegraph-extract/Cargo.toml b/crates/codegraph-extract/Cargo.toml index 1f0eb0c7..bfe9a875 100644 --- a/crates/codegraph-extract/Cargo.toml +++ b/crates/codegraph-extract/Cargo.toml @@ -21,6 +21,7 @@ rayon = { workspace = true } tree-sitter = { workspace = true } streaming-iterator = { workspace = true } codegraph-core = { path = "../codegraph-core" } +ignore = { workspace = true } serde_json = { workspace = true } tracing = { workspace = true } diff --git a/crates/codegraph-extract/src/embedded/liquid.rs b/crates/codegraph-extract/src/embedded/liquid.rs index 05957051..1609c28a 100644 --- a/crates/codegraph-extract/src/embedded/liquid.rs +++ b/crates/codegraph-extract/src/embedded/liquid.rs @@ -1,4 +1,5 @@ use std::cell::RefCell; +use std::sync::OnceLock; use codegraph_core::node_id::{NodeIdAllocator, utf16_column}; use codegraph_core::types::{EdgeKind, ExtractionResult, Language, Node, NodeKind}; @@ -9,6 +10,51 @@ use crate::embedded::shared::{ line_start_for, line_starts, unresolved_ref, }; +/// The tags the extractor reads, each as its name pattern and the argument +/// pattern that follows it. +#[derive(Debug, Clone, Copy)] +enum LiquidTag { + /// `render 'name'` / `include 'name'`. + Snippet, + /// `section 'name'`. + Section, + /// `assign name =`. + Assign, +} + +impl LiquidTag { + /// The braced spelling (matched against one whole `{% … %}` tag) and the + /// bare spelling (matched against one line of a `{% liquid %}` body). + fn patterns(self) -> &'static (Regex, Regex) { + static PATTERNS: OnceLock<[(Regex, Regex); 3]> = OnceLock::new(); + let patterns = PATTERNS.get_or_init(|| { + [ + ("render|include", r#"['"]([^'"]+)['"]"#), + ("section", r#"['"]([^'"]+)['"]"#), + ("assign", r"([0-9A-Za-z_]+)\s*="), + ] + .map(|(name, argument)| { + ( + Regex::new(&format!(r"^\{{%-?\s*({name})\s+{argument}")).unwrap(), + Regex::new(&format!(r"^[ \t]*({name})[ \t]+{argument}")).unwrap(), + ) + }) + }); + &patterns[self as usize] + } +} + +/// One occurrence of a tag in either spelling. +struct TagOccurrence<'s> { + /// From the tag's start through its argument: `{% render 'card'` for a + /// braced tag, `render 'card'` for a bare line of a `{% liquid %}` block. + text: &'s str, + /// The tag name, then the argument's captures. + groups: Vec<&'s str>, + /// Byte offset of `text` in the source. + offset: usize, +} + pub struct LiquidExtractor<'a> { file_path: &'a str, source: &'a str, @@ -40,6 +86,95 @@ impl<'a> LiquidExtractor<'a> { node } + /// Every occurrence of `tag`, in both spellings Liquid allows (upstream + /// #1906). Inside a `{% liquid %}` tag each body line is a tag without + /// braces of its own: + /// + /// ```liquid + /// {% liquid + /// assign heading = section.settings.title + /// render 'card', title: heading + /// %} + /// ``` + /// + /// Whole braced tags are consumed one at a time, so a tag-like string or an + /// inline `{% # … %}` comment never starts a second match inside a tag, and + /// `{% comment %}` / `{% raw %}` regions are skipped in both spellings. + /// Bare tags are anchored at a body line's start, which keeps prose, + /// filters and `#` comment lines out. Sorted by offset. + fn tag_occurrences(&self, tag: LiquidTag) -> Vec> { + static TAGS: OnceLock = OnceLock::new(); + static FIRST_WORD: OnceLock = OnceLock::new(); + let tags = + TAGS.get_or_init(|| Regex::new(r"\{%-?\s*([0-9A-Za-z_]+|#)((?s:.*?))-?%\}").unwrap()); + let first_word = FIRST_WORD.get_or_init(|| Regex::new(r"^[ \t]*([0-9A-Za-z_]+)").unwrap()); + let (braced, bare) = tag.patterns(); + let groups = |captures: ®ex::Captures<'a>| { + captures + .iter() + .skip(1) + .map(|group| group.map_or("", |m| m.as_str())) + .collect::>() + }; + + let mut found = Vec::new(); + let mut blocks = Vec::new(); + let mut suppressed: Option<&str> = None; + for captures in tags.captures_iter(self.source) { + let whole = captures.get(0).unwrap(); + let name = captures.get(1).unwrap(); + if let Some(region) = suppressed { + if name.as_str().strip_prefix("end") == Some(region) { + suppressed = None; + } + continue; + } + match name.as_str() { + "comment" | "raw" => suppressed = Some(name.as_str()), + "liquid" => blocks.push((name.end(), captures.get(2).map_or("", |m| m.as_str()))), + _ => { + if let Some(matched) = braced.captures(whole.as_str()) { + found.push(TagOccurrence { + text: matched.get(0).unwrap().as_str(), + groups: groups(&matched), + offset: whole.start(), + }); + } + } + } + } + + for (body_start, body) in blocks { + let mut offset = body_start; + let mut suppressed: Option<&str> = None; + for line in body.split('\n') { + let name = first_word + .captures(line) + .and_then(|captures| captures.get(1)) + .map(|m| m.as_str()); + if let Some(region) = suppressed { + if name.and_then(|name| name.strip_prefix("end")) == Some(region) { + suppressed = None; + } + } else if let Some(region @ ("comment" | "raw")) = name { + suppressed = Some(region); + } else if let Some(matched) = bare.captures(line) { + let whole = matched.get(0).unwrap().as_str(); + let text = whole.trim_start(); + found.push(TagOccurrence { + text, + groups: groups(&matched), + offset: offset + (whole.len() - text.len()), + }); + } + offset += line.len() + 1; + } + } + + found.sort_by_key(|occurrence| occurrence.offset); + found + } + pub fn extract(self) -> ExtractionResult { let start = std::time::Instant::now(); let mut result = empty_result(0); @@ -85,14 +220,11 @@ impl<'a> LiquidExtractor<'a> { } fn extract_snippets(&self, result: &mut ExtractionResult, file_id: &str) { - let regex = Regex::new(r#"\{%[-]?\s*(render|include)\s+['"]([^'"]+)['"]"#).unwrap(); - for cap in regex.captures_iter(self.source) { - let full = cap.get(0).unwrap(); - let tag_type = cap.get(1).unwrap().as_str(); - let name = cap.get(2).unwrap().as_str(); - let line = line_number_for_offset(&self.line_starts, full.start()); - let col = full.start() as i64 - line_start_for(&self.line_starts, line) as i64; - self.push_import_node(result, file_id, name, full.as_str(), full.start()); + for occurrence in self.tag_occurrences(LiquidTag::Snippet) { + let (tag_type, name) = (occurrence.groups[0], occurrence.groups[1]); + let line = line_number_for_offset(&self.line_starts, occurrence.offset); + let col = occurrence.offset as i64 - line_start_for(&self.line_starts, line) as i64; + self.push_import_node(result, file_id, name, occurrence.text, occurrence.offset); let node = self.identify( default_node( self.file_path, @@ -103,9 +235,9 @@ impl<'a> LiquidExtractor<'a> { line, line, col, - col + full.as_str().len() as i64, + col + occurrence.text.len() as i64, ), - full.start(), + occurrence.offset, ); let node_id = node.id.clone(); result.nodes.push(node); @@ -123,13 +255,11 @@ impl<'a> LiquidExtractor<'a> { } fn extract_sections(&self, result: &mut ExtractionResult, file_id: &str) { - let regex = Regex::new(r#"\{%[-]?\s*section\s+['"]([^'"]+)['"]"#).unwrap(); - for cap in regex.captures_iter(self.source) { - let full = cap.get(0).unwrap(); - let name = cap.get(1).unwrap().as_str(); - let line = line_number_for_offset(&self.line_starts, full.start()); - let col = full.start() as i64 - line_start_for(&self.line_starts, line) as i64; - self.push_import_node(result, file_id, name, full.as_str(), full.start()); + for occurrence in self.tag_occurrences(LiquidTag::Section) { + let name = occurrence.groups[1]; + let line = line_number_for_offset(&self.line_starts, occurrence.offset); + let col = occurrence.offset as i64 - line_start_for(&self.line_starts, line) as i64; + self.push_import_node(result, file_id, name, occurrence.text, occurrence.offset); let node = self.identify( default_node( self.file_path, @@ -140,9 +270,9 @@ impl<'a> LiquidExtractor<'a> { line, line, col, - col + full.as_str().len() as i64, + col + occurrence.text.len() as i64, ), - full.start(), + occurrence.offset, ); let node_id = node.id.clone(); result.nodes.push(node); @@ -190,12 +320,10 @@ impl<'a> LiquidExtractor<'a> { } fn extract_assignments(&self, result: &mut ExtractionResult, file_id: &str) { - let regex = Regex::new(r"\{%[-]?\s*assign\s+(\w+)\s*=").unwrap(); - for cap in regex.captures_iter(self.source) { - let full = cap.get(0).unwrap(); - let name = cap.get(1).unwrap().as_str(); - let line = line_number_for_offset(&self.line_starts, full.start()); - let col = full.start() as i64 - line_start_for(&self.line_starts, line) as i64; + for occurrence in self.tag_occurrences(LiquidTag::Assign) { + let name = occurrence.groups[1]; + let line = line_number_for_offset(&self.line_starts, occurrence.offset); + let col = occurrence.offset as i64 - line_start_for(&self.line_starts, line) as i64; let node = self.identify( default_node( self.file_path, @@ -206,9 +334,9 @@ impl<'a> LiquidExtractor<'a> { line, line, col, - col + full.as_str().len() as i64, + col + occurrence.text.len() as i64, ), - full.start(), + occurrence.offset, ); let node_id = node.id.clone(); result.nodes.push(node); diff --git a/crates/codegraph-extract/src/embedded/mybatis.rs b/crates/codegraph-extract/src/embedded/mybatis.rs index 89117597..999dfc23 100644 --- a/crates/codegraph-extract/src/embedded/mybatis.rs +++ b/crates/codegraph-extract/src/embedded/mybatis.rs @@ -1,3 +1,7 @@ +use std::cell::RefCell; +use std::sync::OnceLock; + +use codegraph_core::node_id::{NodeIdAllocator, utf16_column}; use codegraph_core::types::{EdgeKind, ExtractionResult, Language, NodeKind}; use regex::Regex; @@ -6,114 +10,181 @@ use crate::embedded::shared::{ unresolved_ref, }; +/// The two mapper dialects (upstream `mybatis-extractor.ts`, #1182). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Dialect { + /// MyBatis 3 ``. + MyBatis, + /// iBatis 2 ``, whose namespace is optional and which adds the + /// generic `` and `` verbs. + IBatis, +} + pub struct MyBatisExtractor<'a> { file_path: &'a str, - source: &'a str, + /// The source as written: the file node and identity columns read it. + original: &'a str, + /// The source with every XML comment blanked byte for byte, so the scans + /// never see a commented-out statement or include while every byte offset + /// and line still maps to [`Self::original`]. + source: String, line_starts: Vec, + node_ids: RefCell, } impl<'a> MyBatisExtractor<'a> { pub fn new(file_path: &'a str, source: &'a str) -> Self { + let stripped = strip_xml_comments(source); Self { file_path, - source, - line_starts: line_starts(source), + original: source, + line_starts: line_starts(&stripped), + source: stripped, + node_ids: RefCell::new(NodeIdAllocator::default()), } } pub fn extract(self) -> ExtractionResult { let start = std::time::Instant::now(); let mut result = empty_result(0); - let file_node = file_like_node(self.file_path, self.source, Language::Xml); + let file_node = file_like_node(self.file_path, self.original, Language::Xml); let file_id = file_node.id.clone(); result.nodes.push(file_node); - if let Some((namespace, body_start, body_end)) = self.find_mapper_root() { - self.extract_mapper(&mut result, &file_id, &namespace, body_start, body_end); + if let Some((dialect, namespace, body_start, body_end)) = self.find_mapper_root() { + self.extract_mapper( + &mut result, + &file_id, + dialect, + &namespace, + body_start, + body_end, + ); } result.duration_ms += start.elapsed().as_millis() as i64; result } - /// Locates the mapper root element and its namespace. + /// Locates the mapper root element, its dialect and its namespace. /// - /// Both dialect forms are accepted: the MyBatis `` - /// root and the older iBatis 2 `` root (upstream + /// A MyBatis `` needs a namespace. Failing that, an iBatis 2 + /// `` root is accepted with or without one: a namespace-less map + /// qualifies each statement through its `Map.statement` id (upstream /// #1182). `\b` keeps `` — the iBatis *config* file, which - /// holds no statements — from being mistaken for a statement map. The - /// closing tag is derived from whichever root matched, so the body window - /// ends at the real root close instead of a hard-coded ``. - fn find_mapper_root(&self) -> Option<(String, usize, usize)> { - let open_re = Regex::new(r#"<(mapper|sqlMap)\b([^>]*)>"#).unwrap(); - let ns_re = Regex::new(r#"\bnamespace\s*=\s*"([^"]+)""#).unwrap(); - let open = open_re.captures(self.source)?; - let matched = open.get(0)?; - let root_tag = open.get(1)?.as_str(); - let attrs = open.get(2).map_or("", |m| m.as_str()); - let namespace = ns_re - .captures(attrs) - .and_then(|cap| cap.get(1).map(|m| m.as_str().to_string()))?; - let body_start = matched.end(); - let close_tag = format!(""); - let body_end = self.source[body_start..] - .find(&close_tag) - .map_or(self.source.len(), |idx| body_start + idx); - Some((namespace, body_start, body_end)) + /// holds no statements — from being mistaken for a statement map. The body + /// window ends at the matching root close, or the end of the file. + fn find_mapper_root(&self) -> Option<(Dialect, String, usize, usize)> { + static MAPPER: OnceLock = OnceLock::new(); + static SQL_MAP: OnceLock = OnceLock::new(); + let body = |open: regex::Captures<'_>, close_tag: &str| { + let body_start = open.get(0).map_or(0, |m| m.end()); + let body_end = self.source[body_start..] + .find(close_tag) + .map_or(self.source.len(), |idx| body_start + idx); + (body_start, body_end) + }; + + let mapper = MAPPER.get_or_init(|| Regex::new(r"]*)>").unwrap()); + if let Some(open) = mapper.captures(&self.source) { + let attrs = open.get(1).map_or("", |m| m.as_str()); + if let Some(namespace) = attribute(attrs, Attribute::Namespace) { + let namespace = namespace.to_string(); + let (body_start, body_end) = body(open, ""); + return Some((Dialect::MyBatis, namespace, body_start, body_end)); + } + } + let sql_map = SQL_MAP.get_or_init(|| Regex::new(r"]*)>").unwrap()); + // KEEP-RUST: a self-closing `` is how a + // `` lists its maps, not a map; it holds no statements, + // so it is never the root (upstream would read the config's tail as one). + let open = sql_map.captures_iter(&self.source).find(|open| { + !open + .get(1) + .is_some_and(|attrs| attrs.as_str().trim_end().ends_with('/')) + })?; + let attrs = open.get(1).map_or("", |m| m.as_str()); + let namespace = attribute(attrs, Attribute::Namespace) + .unwrap_or_default() + .to_string(); + let (body_start, body_end) = body(open, ""); + Some((Dialect::IBatis, namespace, body_start, body_end)) } fn extract_mapper( &self, result: &mut ExtractionResult, file_id: &str, + dialect: Dialect, namespace: &str, body_start: usize, body_end: usize, ) { - let open_re = Regex::new(r#"<(select|insert|update|delete|sql)\b([^>]*)>"#).unwrap(); - let id_re = Regex::new(r#"\bid\s*=\s*"([^"]+)""#).unwrap(); + static MYBATIS_OPEN: OnceLock = OnceLock::new(); + static IBATIS_OPEN: OnceLock = OnceLock::new(); + static CLOSE: OnceLock = OnceLock::new(); + let open_re = match dialect { + Dialect::MyBatis => MYBATIS_OPEN.get_or_init(|| { + Regex::new(r"<(select|insert|update|delete|sql)\b([^>]*)>").unwrap() + }), + Dialect::IBatis => IBATIS_OPEN.get_or_init(|| { + Regex::new(r"<(select|insert|update|delete|sql|statement|procedure)\b([^>]*)>") + .unwrap() + }), + }; + // A close tag may carry whitespace before its `>` (``, legal + // XML); missing it let the statement run on to the next close and + // swallow the statement in between (upstream #1222). + let close_re = CLOSE.get_or_init(|| { + Regex::new(r"").unwrap() + }); let body = &self.source[body_start..body_end]; let mut search_start = 0; - while let Some(open) = open_re.find(&body[search_start..]) { + while let Some(captures) = open_re.captures(&body[search_start..]) { + let open = captures.get(0).unwrap(); let open_abs_body = search_start + open.start(); - let full_open = &body[search_start + open.start()..search_start + open.end()]; - let Some(captures) = open_re.captures(full_open) else { - search_start += open.end(); - continue; - }; let elem_type = captures.get(1).unwrap().as_str(); let attrs = captures.get(2).map_or("", |m| m.as_str()); - let close_tag = format!(""); let content_start_body = search_start + open.end(); - let Some(close_rel) = body[content_start_body..].find(&close_tag) else { + let Some(close) = close_re + .captures_iter(&body[content_start_body..]) + .find(|close| close.get(1).is_some_and(|verb| verb.as_str() == elem_type)) + .and_then(|close| close.get(0)) + else { search_start += open.end(); continue; }; - let close_abs_body = content_start_body + close_rel; - let full_end_body = close_abs_body + close_tag.len(); + let close_abs_body = content_start_body + close.start(); + let full_end_body = content_start_body + close.end(); search_start = full_end_body; - let Some(id) = id_re - .captures(attrs) - .and_then(|cap| cap.get(1).map(|m| m.as_str().to_string())) - else { + let Some(id) = attribute(attrs, Attribute::Id) else { continue; }; let absolute_index = body_start + open_abs_body; let start_line = line_number_for_offset(&self.line_starts, absolute_index); let end_line = line_number_for_offset(&self.line_starts, body_start + full_end_body); - let qualified = format!("{namespace}::{id}"); + let (qualified, name) = qualify_statement(namespace, id); let elem_body = &body[content_start_body..close_abs_body]; let mut node = default_node( self.file_path, Language::Xml, NodeKind::Method, - id, - qualified.clone(), + name, + qualified, start_line, end_line, 0, 0, ); + // A vendor-split pair (`databaseId`) written on one line shares a + // name and a line; the allocator keeps the second (#1349). + node.id = self.node_ids.borrow_mut().generate( + self.file_path, + node.kind, + &node.name, + start_line.max(1) as u32, + utf16_column(self.original, absolute_index), + ); node.signature = Some(build_signature(elem_type, attrs)); node.docstring = Some(preview_sql(elem_body)); let node_id = node.id.clone(); @@ -137,10 +208,15 @@ impl<'a> MyBatisExtractor<'a> { elem_body: &str, elem_body_abs: usize, ) { - let include_re = Regex::new(r#"]*\brefid\s*=\s*"([^"]+)""#).unwrap(); + static INCLUDE: OnceLock = OnceLock::new(); + let include_re = INCLUDE.get_or_init(|| { + Regex::new(r#"]*\brefid\s*=\s*(?:"([^"']+)"|'([^"']+)')"#).unwrap() + }); for cap in include_re.captures_iter(elem_body) { let full = cap.get(0).unwrap(); - let refid = cap.get(1).unwrap().as_str(); + let Some(refid) = cap.get(1).or_else(|| cap.get(2)).map(|m| m.as_str()) else { + continue; + }; let ref_qualified = qualify_refid(namespace, refid); let line = line_number_for_offset(&self.line_starts, elem_body_abs + full.start()); result.unresolved_references.push(unresolved_ref( @@ -156,6 +232,88 @@ impl<'a> MyBatisExtractor<'a> { } } +/// Blanks every `` comment byte for byte (newlines kept), so a +/// commented-out statement or include is never matched while offsets and +/// lines are unchanged. `` is skipped intact: a `", index + 4).map_or(bytes.len(), |end| end + 3); + for byte in &mut out[index..stop] { + if *byte != b'\n' { + *byte = b' '; + } + } + index = stop; + continue; + } + index += 1; + } + String::from_utf8(out).unwrap_or_else(|_| source.to_string()) +} + +fn find_bytes(haystack: &[u8], needle: &[u8], from: usize) -> Option { + haystack + .get(from..)? + .windows(needle.len()) + .position(|window| window == needle) + .map(|position| from + position) +} + +#[derive(Debug, Clone, Copy)] +enum Attribute { + Namespace, + Id, + ResultType, + ParameterType, + DatabaseId, +} + +/// The value of attribute `which` in `attrs`, in either quote style. The +/// identifier-shaped values read here are Java names, statement ids or type +/// aliases and never contain a quote, so the value excludes both. +fn attribute(attrs: &str, which: Attribute) -> Option<&str> { + static PATTERNS: OnceLock<[Regex; 5]> = OnceLock::new(); + let patterns = PATTERNS.get_or_init(|| { + [ + "namespace", + "id", + "resultType", + "parameterType", + "databaseId", + ] + .map(|name| Regex::new(&format!(r#"\b{name}\s*=\s*(?:"([^"']+)"|'([^"']+)')"#)).unwrap()) + }); + let captures = patterns[which as usize].captures(attrs)?; + captures + .get(1) + .or_else(|| captures.get(2)) + .map(|m| m.as_str()) +} + +/// The `{namespace}::{id}` qualified name the MyBatis synthesizer +/// suffix-matches against a Java `{Class}::{method}`, and the display name. A +/// namespace-less iBatis map carries the qualifier in the id itself +/// (`Account.getById` → `Account::getById`, named `getById`). +fn qualify_statement(namespace: &str, id: &str) -> (String, String) { + if !namespace.is_empty() { + return (format!("{namespace}::{id}"), id.to_string()); + } + match id.rsplit_once('.') { + Some((owner, name)) => (format!("{owner}::{name}"), name.to_string()), + None => (id.to_string(), id.to_string()), + } +} + /// Builds the `{namespace}::{id}` reference name a `` fragment node /// carries, from an `` value (upstream #1209). /// @@ -165,12 +323,14 @@ impl<'a> MyBatisExtractor<'a> { /// exactly the `qualified_name` the fragment node stores /// (`com.example.UserMapper` + `::` + `baseColumns`). Rewriting every dot to /// `::` would produce a name no node carries, and dropping the namespace would -/// let a same-`id` fragment in another namespace answer the include. +/// let a same-`id` fragment in another namespace answer the include. In a +/// namespace-less iBatis map a bare `refid` names its fragment as is. fn qualify_refid(namespace: &str, refid: &str) -> String { match refid.rsplit_once('.') { Some((owner, fragment)) if !owner.is_empty() && !fragment.is_empty() => { format!("{owner}::{fragment}") } + _ if namespace.is_empty() => refid.to_string(), _ => format!("{namespace}::{refid}"), } } @@ -179,14 +339,17 @@ fn build_signature(elem_type: &str, attrs: &str) -> String { if elem_type == "sql" { return "".to_string(); } - let result_re = Regex::new(r#"\bresultType\s*=\s*"([^"]+)""#).unwrap(); - let param_re = Regex::new(r#"\bparameterType\s*=\s*"([^"]+)""#).unwrap(); let mut parts = vec![elem_type.to_ascii_uppercase()]; - if let Some(param) = param_re.captures(attrs).and_then(|cap| cap.get(1)) { - parts.push(format!("param={}", param.as_str())); + if let Some(param) = attribute(attrs, Attribute::ParameterType) { + parts.push(format!("param={param}")); + } + if let Some(result) = attribute(attrs, Attribute::ResultType) { + parts.push(format!("result={result}")); } - if let Some(result) = result_re.captures(attrs).and_then(|cap| cap.get(1)) { - parts.push(format!("result={}", result.as_str())); + // A vendor-split statement carries `databaseId`; surface it so the two + // otherwise identical `{namespace}::{id}` nodes are distinguishable. + if let Some(database) = attribute(attrs, Attribute::DatabaseId) { + parts.push(format!("databaseId={database}")); } parts.join(" ") } diff --git a/crates/codegraph-extract/src/engine.rs b/crates/codegraph-extract/src/engine.rs index 7577f918..a40b808b 100644 --- a/crates/codegraph-extract/src/engine.rs +++ b/crates/codegraph-extract/src/engine.rs @@ -6,7 +6,6 @@ use anyhow::{Context, Result}; use codegraph_core::config::{Config, IndexingConfig}; -use codegraph_core::node_id::hash_content; use codegraph_core::types::{ExtractionResult, Language}; use rayon::prelude::*; use regex::Regex; @@ -20,6 +19,7 @@ use tree_sitter::Parser; use crate::ext_config::ExtensionOverrides; use crate::lang::{cpp_code_mask, spec_for_language}; use crate::walker::TreeSitterWalker; +use codegraph_core::source_file::{SourceText, read_source_file}; /// Stable diagnostic fragment used when a supported grammar reports parse /// errors and extraction collapses to only the synthetic file node. @@ -409,25 +409,33 @@ pub fn extract_file_with_options_observer( let root = root.as_ref(); let relative_path = normalize_path(relative_path.as_ref()); let full_path = root.join(&relative_path); - let metadata = fs::metadata(&full_path) - .with_context(|| format!("stat source file {}", full_path.display()))?; - if metadata.len() > options.max_file_size { - return Ok(size_skip_result( - &relative_path, - metadata.len(), - options.max_file_size, - )); - } - let source = fs::read_to_string(&full_path) + let (_, source) = read_source_file(&full_path, &relative_path, options.max_file_size) .with_context(|| format!("read source file {}", full_path.display()))?; - let _content_hash = hash_content(&source); - Ok(extract_source_with_observer( - &relative_path, - &source, - None, - &options.extensions, - observer, - )) + Ok(extraction_of(&relative_path, &source, options, observer)) +} + +/// The extraction result for one file as [`read_source_file`] saw it: a parse +/// of its text, the size-skip result over the limit, and nothing at all for an +/// MPEG transport stream, which is not source. +pub fn extraction_of( + relative_path: &str, + source: &SourceText, + options: &ExtractOptions, + observer: impl FnMut(ExtractionStage), +) -> ExtractionResult { + match source { + SourceText::Text(text) => { + extract_source_with_observer(relative_path, text, None, &options.extensions, observer) + } + SourceText::Oversize(size) => size_skip_result(relative_path, *size, options.max_file_size), + SourceText::MpegTransportStream => ExtractionResult { + nodes: Vec::new(), + edges: Vec::new(), + unresolved_references: Vec::new(), + errors: Vec::new(), + duration_ms: 0, + }, + } } pub fn extract_project( @@ -438,24 +446,9 @@ pub fn extract_project( let files = scan_project(root, options)?; let parse = |relative: &String| -> Result { let full = root.join(relative); - let metadata = - fs::metadata(&full).with_context(|| format!("stat source file {}", full.display()))?; - if metadata.len() > options.max_file_size { - return Ok(size_skip_result( - relative, - metadata.len(), - options.max_file_size, - )); - } - let source = fs::read_to_string(&full) + let (_, source) = read_source_file(&full, relative, options.max_file_size) .with_context(|| format!("read source file {}", full.display()))?; - let _content_hash = hash_content(&source); - Ok(extract_source_with( - relative, - &source, - None, - &options.extensions, - )) + Ok(extraction_of(relative, &source, options, |_| {})) }; let mut results = if options.parallel { @@ -487,10 +480,10 @@ pub fn scan_project_with_stats(root: &Path, options: &ExtractOptions) -> Result< .iter() .map(String::as_str) .collect::>(); - let gitignore = read_root_gitignore(root); + let gitignore = RootGitignore::load(root); // Evaluated in order (default paths → config exclude → .gitignore), so a // later `!pattern` negation re-includes a path an earlier set excluded. - let pattern_sets: Vec<&[String]> = vec![&options.ignore_paths, &options.exclude, &gitignore]; + let pattern_sets: Vec<&[String]> = vec![&options.ignore_paths, &options.exclude]; let include = IncludeSet::new(&options.include, &options.exclude); // The EXACT resolved reserved index-root PATH for THIS project, resolved // once. scan_dir prunes a directory iff its FULL path equals this root, @@ -506,6 +499,7 @@ pub fn scan_project_with_stats(root: &Path, options: &ExtractOptions) -> Result< &ignored_dirs, &reserved_roots, &pattern_sets, + (&gitignore, GitignoreAncestry::default()), &include, &options.extensions, &mut files, @@ -525,6 +519,7 @@ fn scan_dir( ignored_dirs: &HashSet<&str>, reserved_roots: &std::collections::BTreeSet, pattern_sets: &[&[String]], + (gitignore, above): (&RootGitignore, GitignoreAncestry), include: &IncludeSet, overrides: &ExtensionOverrides, files: &mut Vec, @@ -542,12 +537,18 @@ fn scan_dir( // while a same-basename user directory elsewhere is not. let is_reserved_root_here = (dir == root && name == ".git") || reserved_roots.contains(&path); - if is_reserved_root_here || ignored_dirs.contains(name.as_ref()) { + let relative = normalize_path(path.strip_prefix(root).unwrap_or(&path)); + // `build` is also a legal JVM package segment: keep it under a + // conventional source root while still pruning build output (#1642). + let jvm_package = name == "build" + && codegraph_core::config::is_jvm_source_build_dir(&relative) + && entry.file_type().is_ok_and(|kind| kind.is_dir()); + if is_reserved_root_here || (ignored_dirs.contains(name.as_ref()) && !jvm_package) { continue; } - let relative = normalize_path(path.strip_prefix(root).unwrap_or(&path)); - let ignored = is_path_ignored(&relative, pattern_sets); let file_type = entry.file_type()?; + let own = gitignore.matched(&relative, file_type.is_dir()); + let ignored = is_path_ignored(&relative, pattern_sets, above.decide(own)); if file_type.is_dir() { // A model-ignored dir is normally pruned before descent, so a FILE // include under a gitignored ancestor would never be reached. @@ -562,6 +563,7 @@ fn scan_dir( ignored_dirs, reserved_roots, pattern_sets, + (gitignore, above.child(own)), include, overrides, files, @@ -610,7 +612,9 @@ fn merge_results(results: &mut [ExtractionResult]) -> Result { Ok(merged) } -fn size_skip_result(file_path: &str, size: u64, max: u64) -> ExtractionResult { +/// The result recorded for a file over the size limit: no symbols, and the +/// error every indexing path reports for it. +pub fn size_skip_result(file_path: &str, size: u64, max: u64) -> ExtractionResult { ExtractionResult { nodes: Vec::new(), edges: Vec::new(), @@ -622,22 +626,96 @@ fn size_skip_result(file_path: &str, size: u64, max: u64) -> ExtractionResult { } } -fn read_root_gitignore(root: &Path) -> Vec { - fs::read_to_string(root.join(".gitignore")) - .unwrap_or_default() - .lines() - .map(str::trim) - .filter(|line| !line.is_empty() && !line.starts_with('#')) - .map(ToOwned::to_owned) - .collect() +/// The project-root `.gitignore`, read with git's own rules through the +/// `ignore` crate, as upstream reads it with the `ignore` package: a leading or +/// inner `/` anchors a rule to the root, a slash-less rule applies at any +/// depth, `*` and `**` glob, a trailing `/` matches directories only, and `!` +/// re-includes. Shared with `codegraph-watch`, so the watcher's `.gitignore` +/// verdict is the scan's by construction. An unreadable, non-UTF-8 or NUL-laden +/// file is treated as absent and an unparseable line is skipped, never fatal. +#[derive(Debug, Clone, Default)] +pub struct RootGitignore { + matcher: Option, +} + +impl RootGitignore { + pub fn load(root: &Path) -> Self { + let Ok(text) = fs::read_to_string(root.join(".gitignore")) else { + return Self::default(); + }; + if text.contains('\0') { + return Self::default(); + } + // Rooted at `.` so a root-relative candidate is never prefix-stripped. + let mut builder = ignore::gitignore::GitignoreBuilder::new("."); + for line in text.lines() { + // An invalid glob drops only its own line; the rest still apply. + let _ = builder.add_line(None, line); + } + let matcher = builder.build().ok().filter(|matcher| !matcher.is_empty()); + Self { matcher } + } + + /// The root `.gitignore` verdict for `relative` (root-relative, + /// `/`-separated): `Some(true)` ignored, `Some(false)` re-included, `None` + /// undecided. Walks the directories above it; the scan instead carries + /// that state down its walk in a [`GitignoreAncestry`]. + pub fn verdict(&self, relative: &str, is_dir: bool) -> Option { + self.matcher.as_ref()?; + let above = relative + .match_indices('/') + .fold(GitignoreAncestry::default(), |above, (end, _)| { + above.child(self.matched(&relative[..end], true)) + }); + above.decide(self.matched(relative, is_dir)) + } + + /// The last rule matching `relative` itself, ignoring its ancestors. + fn matched(&self, relative: &str, is_dir: bool) -> Option { + match self.matcher.as_ref()?.matched(relative, is_dir) { + ignore::Match::None => None, + ignore::Match::Ignore(_) => Some(true), + ignore::Match::Whitelist(_) => Some(false), + } + } +} + +/// What the root `.gitignore` decided for the directories above a path. +#[derive(Debug, Clone, Copy, Default)] +struct GitignoreAncestry { + ignored: bool, + reincluded: bool, } -/// Evaluate ordered `.gitignore`-style pattern sets with last-match-wins -/// negation: a `!pattern` line un-ignores a path an earlier pattern excluded. -/// Sets are scanned in order, patterns within a set in order, and the final -/// matching pattern decides — so a later `!res/values/` re-includes what a -/// default `res/values*` excluded. -fn is_path_ignored(relative: &str, pattern_sets: &[&[String]]) -> bool { +impl GitignoreAncestry { + /// The ancestry of an entry below a directory whose own verdict was `own`. + fn child(self, own: Option) -> Self { + Self { + ignored: self.ignored || own == Some(true), + reincluded: self.reincluded || own == Some(false), + } + } + + /// An ignored ancestor hides the path, since git never re-includes below an + /// excluded directory; otherwise the path's own last matching rule decides, + /// and failing that a re-included ancestor re-includes it, so `!res/values/` + /// still overrides a default path set that matched the files inside. + fn decide(self, own: Option) -> Option { + if self.ignored { + Some(true) + } else { + own.or(self.reincluded.then_some(false)) + } + } +} + +/// Evaluate the ordered `.gitignore`-style config pattern sets with last-match +/// wins negation, then let the root `.gitignore` verdict, when it has one, +/// decide last: a `!pattern` line un-ignores a path an earlier pattern +/// excluded. Sets are scanned in order, patterns within a set in order, and the +/// final matching pattern decides — so a later `!res/values/` re-includes what +/// a default `res/values*` excluded. +fn is_path_ignored(relative: &str, pattern_sets: &[&[String]], gitignore: Option) -> bool { let mut ignored = false; for set in pattern_sets { for pattern in set.iter() { @@ -650,7 +728,7 @@ fn is_path_ignored(relative: &str, pattern_sets: &[&[String]]) -> bool { } } } - ignored + gitignore.unwrap_or(ignored) } fn pattern_matches(relative: &str, pattern: &str) -> bool { @@ -926,6 +1004,50 @@ mod tests { fs::remove_dir_all(&project).ok(); } + /// `build` is a legal JVM package segment: a Java/Kotlin/Scala package + /// directory named `build` under a conventional source root stays indexed, + /// while module build output and the other defaults inside it stay pruned + /// (upstream #1642). + #[test] + fn scan_keeps_jvm_packages_named_build_but_prunes_build_output() { + let project = unique_project("jvm_build_package"); + touch( + &project, + "src/main/java/com/acme/build/Builder.java", + "class Builder {}", + ); + touch( + &project, + "app/src/test/kotlin/build/BuildTest.kt", + "class BuildTest", + ); + touch(&project, "src/main/scala/build/Tool.scala", "object Tool"); + touch(&project, "build/generated/Gen.java", "class Gen {}"); + touch(&project, "app/build/classes/Out.java", "class Out {}"); + touch( + &project, + "src/main/resources/build/Res.java", + "class Res {}", + ); + touch( + &project, + "src/main/java/build/node_modules/dep.js", + "export {}", + ); + + let files = scan_project(&project, &ExtractOptions::default()).expect("scan project"); + assert_eq!( + files, + vec![ + "app/src/test/kotlin/build/BuildTest.kt".to_string(), + "src/main/java/com/acme/build/Builder.java".to_string(), + "src/main/scala/build/Tool.scala".to_string(), + ] + ); + + fs::remove_dir_all(&project).ok(); + } + /// A team authoring first-party code under `addons/` can re-include it by /// overriding `indexing.ignore_dirs` (the same override surface a custom /// `.codegraph/config.toml` populates), proving the default is opt-out. @@ -1357,14 +1479,20 @@ mod tests { fn is_path_ignored_negation_is_last_match_wins() { let defaults = vec!["res/values*".to_string()]; let user = vec!["!res/values/".to_string()]; - assert!(is_path_ignored("res/values/strings.xml", &[&defaults])); + assert!(is_path_ignored( + "res/values/strings.xml", + &[&defaults], + None + )); assert!(!is_path_ignored( "res/values/strings.xml", - &[&defaults, &user] + &[&defaults, &user], + None )); assert!(is_path_ignored( "res/values-es/strings.xml", - &[&defaults, &user] + &[&defaults, &user], + None )); } @@ -1383,6 +1511,71 @@ mod tests { fs::remove_dir_all(&project).ok(); } + /// The root `.gitignore` is read with git's own rules, as upstream's `ignore` + /// matcher reads it: a leading or inner `/` anchors a rule to the root, a + /// slash-less rule applies at any depth, `*` and `**` glob, and nothing below + /// an ignored directory can be re-included. + #[test] + fn scan_project_reads_the_root_gitignore_with_git_rules() { + let project = unique_project("gitignore_git_rules"); + touch( + &project, + ".gitignore", + "/artifacts/\n*.gen.ts\ndocs/**/draft/\nscratch/\nlogs/\n!logs/keep.ts\n", + ); + for file in [ + "src/app.ts", + "artifacts/a.ts", + "src/artifacts/b.ts", + "src/types.gen.ts", + "docs/guide/draft/c.ts", + "docs/guide/final/d.ts", + "src/scratch/e.ts", + "logs/keep.ts", + ] { + touch(&project, file, "export const x = 1;\n"); + } + let files = scan_project(&project, &ExtractOptions::default()).expect("scan"); + assert_eq!( + files, + vec!["docs/guide/final/d.ts", "src/app.ts", "src/artifacts/b.ts"] + ); + fs::remove_dir_all(&project).ok(); + } + + /// An explicit root `.gitignore` still decides a JVM package named `build` + /// (upstream #1642): a bare `build/` hides it as git does, while Android's + /// anchored `/build` and Spring Initializr's `build/` plus + /// `!**/src/main/**/build/` keep it. Module build output stays pruned. + #[test] + fn root_gitignore_still_decides_jvm_packages_named_build() { + for (gitignore, kept) in [ + ("build/\n", false), + ("/build\n", true), + ( + "build/\n!**/src/main/**/build/\n!**/src/test/**/build/\n", + true, + ), + ] { + let project = unique_project("jvm_build_gitignore"); + touch(&project, ".gitignore", gitignore); + touch(&project, "src/main/java/App.java", "class App {}"); + touch( + &project, + "src/main/java/com/acme/build/Hidden.java", + "class Hidden {}", + ); + touch(&project, "app/build/generated/Out.java", "class Out {}"); + let files = scan_project(&project, &ExtractOptions::default()).expect("scan"); + let mut expected = vec!["src/main/java/App.java".to_string()]; + if kept { + expected.push("src/main/java/com/acme/build/Hidden.java".to_string()); + } + assert_eq!(files, expected, "{gitignore:?}"); + fs::remove_dir_all(&project).ok(); + } + } + #[test] fn scan_project_reports_visible_unsupported_extensions_in_the_same_walk() { let project = unique_project("unsupported-extensions"); diff --git a/crates/codegraph-extract/src/lib.rs b/crates/codegraph-extract/src/lib.rs index aa9d2b83..4472355d 100644 --- a/crates/codegraph-extract/src/lib.rs +++ b/crates/codegraph-extract/src/lib.rs @@ -16,10 +16,13 @@ pub mod lang; pub mod spec; pub mod walker; +pub use codegraph_core::source_file::{ + SourceText, is_source_file, oversize_stamp, read_source_file, +}; pub use engine::{ - ExtractOptions, ExtractionStage, PARSE_COLLAPSE_WARNING, detect_language, detect_language_with, - extract_file, extract_file_with_options, extract_file_with_options_observer, extract_project, - extract_source, extract_source_with, extract_source_with_observer, - include_exclude_pattern_matches, is_extraction_warning, + ExtractOptions, ExtractionStage, PARSE_COLLAPSE_WARNING, RootGitignore, detect_language, + detect_language_with, extract_file, extract_file_with_options, + extract_file_with_options_observer, extract_project, extract_source, extract_source_with, + extract_source_with_observer, include_exclude_pattern_matches, is_extraction_warning, }; pub use ext_config::ExtensionOverrides; diff --git a/crates/codegraph-extract/tests/liquid_blocks.rs b/crates/codegraph-extract/tests/liquid_blocks.rs new file mode 100644 index 00000000..9088b57f --- /dev/null +++ b/crates/codegraph-extract/tests/liquid_blocks.rs @@ -0,0 +1,257 @@ +//! Tags inside a `{% liquid %}` block (upstream v1.6.1 #1906): each body line is +//! a tag without braces of its own. Ported from upstream `extraction.test.ts`. + +use codegraph_core::types::{ExtractionResult, Language, NodeKind}; +use codegraph_extract::extract_source; + +fn extract(file: &str, lines: &[&str]) -> ExtractionResult { + extract_with(file, lines, "\n") +} + +fn extract_with(file: &str, lines: &[&str], newline: &str) -> ExtractionResult { + let result = extract_source(file, &lines.join(newline), Some(Language::Liquid)); + assert!(result.errors.is_empty(), "{file}: {:?}", result.errors); + result +} + +fn names_of(result: &ExtractionResult, kind: NodeKind) -> Vec<&str> { + result + .nodes + .iter() + .filter(|node| node.kind == kind) + .map(|node| node.name.as_str()) + .collect() +} + +fn reference_names(result: &ExtractionResult) -> Vec<&str> { + result + .unresolved_references + .iter() + .map(|reference| reference.reference_name.as_str()) + .collect() +} + +#[test] +fn render_assign_and_section_inside_a_liquid_block_are_extracted() { + let result = extract( + "sections/featured.liquid", + &[ + "{% liquid", + " assign heading = section.settings.title", + " render 'card', title: heading", + "%}", + ], + ); + assert!(names_of(&result, NodeKind::Import).contains(&"card")); + assert!(names_of(&result, NodeKind::Variable).contains(&"heading")); + + let result = extract( + "layout/theme.liquid", + &["{% liquid", " section 'header'", "%}"], + ); + assert!(names_of(&result, NodeKind::Import).contains(&"header")); +} + +#[test] +fn a_tag_inside_a_liquid_block_reports_its_real_line() { + let result = extract( + "sections/featured.liquid", + &[ + "
", + "{% liquid", + " assign x = 1", + " render 'card'", + "%}", + ], + ); + let card = result + .nodes + .iter() + .find(|node| node.kind == NodeKind::Import && node.name == "card") + .expect("card import"); + assert_eq!(card.start_line, 4); +} + +#[test] +fn both_spellings_are_counted_once_each() { + let result = extract( + "sections/featured.liquid", + &["{% render 'card' %}", "{% liquid", " render 'card'", "%}"], + ); + let lines = result + .nodes + .iter() + .filter(|node| node.kind == NodeKind::Import && node.name == "card") + .map(|node| node.start_line) + .collect::>(); + assert_eq!(lines, vec![1, 3]); +} + +#[test] +fn a_bare_render_outside_a_liquid_block_is_prose() { + let result = extract( + "sections/featured.liquid", + &[ + "

We render the card below.

", + "{{ product | render_as: 'card' }}", + ], + ); + assert!(!names_of(&result, NodeKind::Import).contains(&"card")); +} + +#[test] +fn whitespace_control_on_the_liquid_tag_is_accepted() { + let result = extract( + "sections/featured.liquid", + &["{%- liquid", " render 'card'", "-%}"], + ); + assert!(names_of(&result, NodeKind::Import).contains(&"card")); +} + +#[test] +fn tag_like_strings_and_inline_comments_are_not_scanned_twice() { + let result = extract( + "sections/featured.liquid", + &[ + "{% assign example = \"{% render 'ghost'\" %}", + "{% # {% render 'ghost' %}", + "{% liquid", + " assign example = \"{% include 'ghost'\"", + " # section 'ghost'", + " echo 'render ghost'", + "%}", + "{% liquid render 'live' %}", + "{% liquid include 'after' %}", + ], + ); + let imports = result + .nodes + .iter() + .filter(|node| node.kind == NodeKind::Import) + .map(|node| (node.name.as_str(), node.start_line, node.start_column)) + .collect::>(); + assert_eq!(imports, vec![("live", 8, 10), ("after", 9, 10)]); + assert_eq!( + reference_names(&result), + vec!["snippets/live.liquid", "snippets/after.liquid"] + ); +} + +#[test] +fn liquid_block_positions_survive_both_line_endings() { + for newline in ["\n", "\r\n"] { + let result = extract_with( + "sections/featured.liquid", + &[ + "
", + "{%- liquid", + " assign heading = 'x'", + "\tinclude 'legacy'", + " render 'card'", + " section 'footer'", + "-%}", + " {% render 'after' %}", + ], + newline, + ); + let mut nodes = result + .nodes + .iter() + .filter(|node| matches!(node.kind, NodeKind::Variable | NodeKind::Import)) + .map(|node| { + ( + node.name.as_str(), + node.start_line, + node.start_column, + node.end_column, + ) + }) + .collect::>(); + nodes.sort(); + assert_eq!( + nodes, + vec![ + ("after", 8, 2, 19), + ("card", 5, 2, 15), + ("footer", 6, 2, 18), + ("heading", 3, 2, 18), + ("legacy", 4, 1, 17), + ], + "{newline:?}" + ); + let mut references = result + .unresolved_references + .iter() + .map(|reference| { + ( + reference.reference_name.as_str(), + reference.line, + reference.col, + ) + }) + .collect::>(); + references.sort(); + assert_eq!( + references, + vec![ + ("sections/footer.liquid", 6, 2), + ("snippets/after.liquid", 8, 2), + ("snippets/card.liquid", 5, 2), + ("snippets/legacy.liquid", 4, 1), + ], + "{newline:?}" + ); + } +} + +#[test] +fn comment_and_raw_regions_are_ignored_in_both_spellings() { + for tag in ["comment", "raw"] { + let open = format!("{{%- {tag} -%}}"); + let close = format!("{{%- end{tag} -%}}"); + let bare_open = format!(" {tag}"); + let bare_close = format!(" end{tag}"); + let result = extract( + "sections/featured.liquid", + &[ + &open, + "{% render 'ghost' %}", + "{% include 'ghost' %}", + "{% section 'ghost' %}", + "{% assign ghost = 1 %}", + "{% liquid", + " render 'ghost'", + "%}", + &close, + "{% liquid", + &bare_open, + " render 'ghost'", + " include 'ghost'", + " section 'ghost'", + " assign ghost = 1", + &bare_close, + " # render 'ghost'", + " render 'live'", + "%}", + "{% include 'after' %}", + ], + ); + let names = result + .nodes + .iter() + .filter(|node| node.kind != NodeKind::File) + .map(|node| node.name.as_str()) + .collect::>(); + assert_eq!(names, vec!["live", "live", "after", "after"], "{tag}"); + let references = result + .unresolved_references + .iter() + .map(|reference| (reference.reference_name.as_str(), reference.line)) + .collect::>(); + assert_eq!( + references, + vec![("snippets/live.liquid", 18), ("snippets/after.liquid", 20)], + "{tag}" + ); + } +} diff --git a/crates/codegraph-extract/tests/mybatis_mappers.rs b/crates/codegraph-extract/tests/mybatis_mappers.rs new file mode 100644 index 00000000..50dc2aae --- /dev/null +++ b/crates/codegraph-extract/tests/mybatis_mappers.rs @@ -0,0 +1,166 @@ +//! MyBatis and iBatis mapper forms upstream v1.6.1 reads (#1182, #1222): +//! close tags with whitespace, commented-out statements, either quote style, +//! namespace-less `` ids, the iBatis-only verbs, and vendor-split +//! statements written on one line. + +use codegraph_core::types::{EdgeKind, ExtractionResult, Language, Node, NodeKind}; +use codegraph_extract::extract_source; + +fn extract(file: &str, source: &str) -> ExtractionResult { + let result = extract_source(file, source, Some(Language::Xml)); + assert!(result.errors.is_empty(), "{file}: {:?}", result.errors); + result +} + +fn statements(result: &ExtractionResult) -> Vec<&Node> { + result + .nodes + .iter() + .filter(|node| node.kind == NodeKind::Method) + .collect() +} + +fn qualified_names(result: &ExtractionResult) -> Vec<&str> { + statements(result) + .iter() + .map(|node| node.qualified_name.as_str()) + .collect() +} + +fn reference_names(result: &ExtractionResult) -> Vec<&str> { + result + .unresolved_references + .iter() + .filter(|reference| reference.reference_kind == EdgeKind::References) + .map(|reference| reference.reference_name.as_str()) + .collect() +} + +#[test] +fn a_close_tag_may_carry_whitespace_before_its_bracket() { + let result = extract( + "mapper.xml", + "\n\ + \x20 \n\ + \x20 \n\ + \n", + ); + assert_eq!( + qualified_names(&result), + vec!["com.example.M::first", "com.example.M::second"] + ); + assert_eq!(statements(&result)[0].end_line, 2); +} + +#[test] +fn commented_out_statements_and_includes_are_ignored_but_cdata_is_not_a_comment() { + let result = extract( + "mapper.xml", + "\n\ + \x20 \n\ + \x20 \n\ + \x20 \n\ + \x20 \n\ + \n", + ); + assert_eq!( + qualified_names(&result), + vec![ + "com.example.M::live", + "com.example.M::cols", + "com.example.M::after" + ] + ); + assert_eq!(reference_names(&result), vec!["com.example.M::cols"]); + let include = &result.unresolved_references[0]; + assert_eq!(include.line, 4, "comment blanking keeps original lines"); +} + +#[test] +fn ibatis_sqlmaps_without_a_namespace_qualify_by_their_ids() { + let result = extract( + "Account.xml", + "\n\ + \x20 \n\ + \x20 CALL raw()\n\ + \x20 {call p()}\n\ + \x20 id, name\n\ + \x20 x\n\ + \x20 \n\ + \n", + ); + assert_eq!( + qualified_names(&result), + vec![ + "Account::getById", + "Account::raw", + "Account::proc", + "Account::cols", + "bare", + "local" + ] + ); + let names = statements(&result) + .iter() + .map(|node| node.name.as_str()) + .collect::>(); + assert_eq!( + names, + vec!["getById", "raw", "proc", "cols", "bare", "local"] + ); + assert_eq!(reference_names(&result), vec!["Account::cols", "bare"]); +} + +#[test] +fn either_quote_style_is_read_and_ibatis_verbs_stay_ibatis_only() { + let ibatis = extract( + "Legacy.xml", + "\n CALL run()\n\n", + ); + assert_eq!(qualified_names(&ibatis), vec!["Legacy::run"]); + + let mybatis = extract( + "QueryMapper.xml", + "\n\ + \x20 \n\ + \x20 CALL x()\n\ + \x20 \n\ + \n", + ); + assert_eq!(qualified_names(&mybatis), vec!["com.example.Q::q"]); + assert_eq!( + statements(&mybatis)[0].signature.as_deref(), + Some("SELECT param=int result=Row databaseId=oracle") + ); +} + +#[test] +fn vendor_split_statements_on_one_line_keep_both_nodes() { + let result = extract( + "mapper.xml", + "\n", + ); + let nodes = statements(&result); + assert_eq!(nodes.len(), 2, "{nodes:#?}"); + assert_ne!(nodes[0].id, nodes[1].id); + assert_eq!( + nodes + .iter() + .map(|node| node.signature.as_deref()) + .collect::>(), + vec![ + Some("SELECT databaseId=oracle"), + Some("SELECT databaseId=mysql") + ] + ); + for node in nodes { + assert!( + result + .edges + .iter() + .any(|edge| edge.kind == EdgeKind::Contains && edge.target == node.id) + ); + } +} diff --git a/crates/codegraph-mcp/Cargo.toml b/crates/codegraph-mcp/Cargo.toml index dd487e4d..be8548ea 100644 --- a/crates/codegraph-mcp/Cargo.toml +++ b/crates/codegraph-mcp/Cargo.toml @@ -54,7 +54,7 @@ tokio-util = { version = "0.7" } # dependency of an integration test binary, so a Cargo feature (auto-enabled for # this crate's own dev/test builds via the dev-dependency self-reference below) # is the only reliable gate. NEVER enabled in the shipped binary. -test-hooks = ["codegraph-store/test-hooks"] +test-hooks = ["codegraph-store/test-hooks", "codegraph-watch/test-hooks"] [dev-dependencies] # Self-reference with `test-hooks` so the seams above are compiled into the lib diff --git a/crates/codegraph-mcp/src/dynamic_boundaries.rs b/crates/codegraph-mcp/src/dynamic_boundaries.rs index ed5647cf..38af9216 100644 --- a/crates/codegraph-mcp/src/dynamic_boundaries.rs +++ b/crates/codegraph-mcp/src/dynamic_boundaries.rs @@ -577,6 +577,54 @@ struct FormSpec { re: Regex, key_from: Option Option>, key_window: Option, + /// Final say on a match, for forms whose argument decides it. + accept: Option, +} + +/// Given the stripped and original bytes and the offset just past a match, +/// `false` drops the match. +type AcceptMatch = fn(&[u8], &[u8], usize) -> bool; + +/// Whether the call argument starting at `start` is anything but ONE complete +/// string literal (upstream #1967): `import('./a')` or a backtick string with +/// no substitution is an ordinary import, while a template with a `${}` +/// substitution, a concatenation or a bare expression picks the module at +/// runtime. String contents are blank in `stripped` (quotes kept), so the +/// substitution check reads `original` at the same offsets. +fn is_runtime_import_argument(stripped: &[u8], original: &[u8], start: usize) -> bool { + let skip_space = |mut at: usize| { + while stripped.get(at).is_some_and(u8::is_ascii_whitespace) { + at += 1; + } + at + }; + let open = skip_space(start); + let quote = match stripped.get(open) { + Some(b')') => return false, // `import()` — nothing to resolve + Some("e @ (b'"' | b'\'' | b'`')) => quote, + _ => return true, + }; + let Some(close) = stripped[open + 1..] + .iter() + .position(|&byte| byte == quote) + .map(|offset| open + 1 + offset) + else { + return false; + }; + if quote == b'`' { + let mut at = open + 1; + while at < close { + match original.get(at) { + // An escape pair: `\${` is literal text, but `\\${` interpolates. + Some(b'\\') => at += 2, + Some(b'$') if original.get(at + 1) == Some(&b'{') => return true, + _ => at += 1, + } + } + } + // `)` ends the call and `,` starts import()'s options argument; anything + // else (`+`, `.concat(`, …) builds the specifier at runtime. + !matches!(stripped.get(skip_space(close + 1)), Some(b')' | b',')) } fn key_computed_call(orig: &str) -> Option { @@ -638,15 +686,16 @@ fn forms() -> &'static [FormSpec] { re: Regex::new(r"[\w$)\]]\s*\[([^\[\]\n]{1,80})\]\s*\(").expect("computed-call"), key_from: Some(key_computed_call), key_window: None, + accept: None, }, FormSpec { form: "dynamic-import", label: "dynamic import", langs: Some(JS_FAMILY), - re: Regex::new(r#"\b(?:import|require)\s*\(\s*(?:[^\s'"`)])"#) - .expect("dynamic-import-js"), + re: Regex::new(r"\b(?:import|require)\s*\(").expect("dynamic-import-js"), key_from: None, key_window: None, + accept: Some(is_runtime_import_argument), }, FormSpec { form: "dynamic-import", @@ -656,6 +705,7 @@ fn forms() -> &'static [FormSpec] { .expect("dynamic-import-py"), key_from: None, key_window: None, + accept: None, }, FormSpec { form: "ruby-send", @@ -665,6 +715,7 @@ fn forms() -> &'static [FormSpec] { .expect("ruby-send"), key_from: Some(key_ruby_send), key_window: None, + accept: None, }, FormSpec { form: "php-dynamic", @@ -676,6 +727,7 @@ fn forms() -> &'static [FormSpec] { .expect("php-dynamic"), key_from: Some(key_single_literal), key_window: Some(80), + accept: None, }, FormSpec { form: "reflection", @@ -687,6 +739,7 @@ fn forms() -> &'static [FormSpec] { .expect("reflection"), key_from: Some(key_single_literal), key_window: Some(80), + accept: None, }, FormSpec { form: "proxy-reflect", @@ -696,6 +749,7 @@ fn forms() -> &'static [FormSpec] { .expect("proxy-reflect"), key_from: None, key_window: None, + accept: None, }, FormSpec { form: "typed-bus", @@ -707,6 +761,7 @@ fn forms() -> &'static [FormSpec] { .expect("typed-bus"), key_from: Some(key_typed_bus), key_window: None, + accept: None, }, FormSpec { form: "var-key-dispatch", @@ -718,6 +773,7 @@ fn forms() -> &'static [FormSpec] { .expect("var-key-dispatch"), key_from: None, key_window: None, + accept: None, }, FormSpec { form: "selector", @@ -727,6 +783,7 @@ fn forms() -> &'static [FormSpec] { .expect("selector"), key_from: Some(key_selector), key_window: None, + accept: None, }, ] }) @@ -1008,6 +1065,11 @@ pub fn scan_dynamic_dispatch( continue; } for m in spec.re.find_iter(&stripped_str) { + if let Some(accept) = spec.accept + && !accept(&stripped, original, m.end()) + { + continue; + } let m_index = m.start(); let mut slice_end = m.end(); if let Some(window) = spec.key_window { @@ -1057,6 +1119,39 @@ pub fn scan_dynamic_dispatch( mod tests { use super::*; + /// A JS dynamic import is a runtime boundary unless its argument is exactly + /// one complete string literal (upstream #1967). + #[test] + fn js_dynamic_import_flags_templates_concatenations_and_expressions() { + let flagged = |body: &str| { + scan_dynamic_dispatch(body, "typescript", 1) + .iter() + .any(|m| m.form == "dynamic-import") + }; + for runtime in [ + "const m = await import(`./locales/${lang}.js`);", + "const m = await import('./x/' + y);", + "const m = await import(\"./a\".concat(b));", + "const m = await import(name);", + "const m = require('./plugins/' + id);", + "const m = await import(`\\\\${dir}/a.js`);", + ] { + assert!(flagged(runtime), "{runtime}"); + } + for literal in [ + "const m = await import('./a');", + "const m = await import(\"./a\");", + "const m = await import(`./a.js`);", + "const m = await import('./data.json', { with: { type: 'json' } });", + "const m = await import(`\\${literal}.js`);", + "const m = await import( './spaced' );", + "const m = require('./a');", + "import();", + ] { + assert!(!flagged(literal), "{literal}"); + } + } + fn one(body: &str, lang: &str, start: i64) -> BoundaryMatch { let mut m = scan_dynamic_dispatch(body, lang, start); assert_eq!(m.len(), 1, "expected exactly one match for {body:?}"); diff --git a/crates/codegraph-mcp/src/engine.rs b/crates/codegraph-mcp/src/engine.rs index 5988b96f..4cd2af18 100644 --- a/crates/codegraph-mcp/src/engine.rs +++ b/crates/codegraph-mcp/src/engine.rs @@ -7,7 +7,7 @@ //! file:line for its API call + output template. use std::cell::RefCell; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -126,6 +126,10 @@ pub struct CodeGraphEngine { source_probes: RefCell>, /// Request-local set of files whose current bytes actually reached the result. source_citations: RefCell>, + /// Request-local set of indexed files a graph answer (search, callers, + /// callees, impact) names as locations. Checked for drift before the answer + /// is served while auto-sync is not healthy (upstream #1959). + answer_files: RefCell>, } impl CodeGraphEngine { @@ -152,6 +156,7 @@ impl CodeGraphEngine { deprioritize, source_probes: RefCell::new(HashMap::new()), source_citations: RefCell::new(HashSet::new()), + answer_files: RefCell::new(BTreeSet::new()), }) } @@ -173,9 +178,10 @@ impl CodeGraphEngine { /// Freshness is fail-closed: it is proven only after successfully loading the /// file record and matching either size + millisecond mtime (the sync fast /// path, with no hash) or, after a stat mismatch, the indexed sha256 content - /// hash. A missing/unreadable record, read failure, or unhashable oversized - /// stat mismatch remains possibly drifted. The full probe is memoized for the - /// current handler. + /// hash — of the file's text, or of its size stamp for a file over the limit, + /// which is compared without being read, exactly as indexing hashed it. A + /// missing/unreadable record or a read failure remains possibly drifted. The + /// full probe is memoized for the current handler. fn project_source(&self, file_path: &str) -> SourceProbe { if let Some(cached) = self.source_probes.borrow().get(file_path).cloned() { return cached; @@ -193,34 +199,31 @@ impl CodeGraphEngine { && metadata_modified_millis(&metadata) .is_some_and(|mtime| file.modified_at == mtime) }); - if metadata.len() > self.config.indexing.max_file_size { - SourceProbe { - content: None, - freshness: if stat_matches { + match codegraph_core::source_file::read_source_file( + &abs, + file_path, + self.config.indexing.max_file_size, + ) { + Ok((_, source)) => { + let freshness = if stat_matches + || source.hash_input().is_some_and(|input| { + stored + .as_ref() + .is_some_and(|file| file.content_hash == hash_content(&input)) + }) { SourceFreshness::ProvenFresh } else { SourceFreshness::PossiblyDrifted - }, - } - } else { - match fs::read_to_string(&abs) { - Ok(content) => { - let freshness = if stat_matches - || stored - .as_ref() - .is_some_and(|file| file.content_hash == hash_content(&content)) - { - SourceFreshness::ProvenFresh - } else { - SourceFreshness::PossiblyDrifted - }; - SourceProbe { - content: Some(Arc::::from(content)), - freshness, + }; + let content = match source { + codegraph_core::source_file::SourceText::Text(text) => { + Some(Arc::::from(text)) } - } - Err(_) => SourceProbe::default(), + _ => None, + }; + SourceProbe { content, freshness } } + Err(_) => SourceProbe::default(), } } _ => SourceProbe::default(), @@ -338,6 +341,8 @@ impl CodeGraphEngine { pub fn execute(&self, tool_name: &str, args: &Value) -> ToolResult { self.source_probes.borrow_mut().clear(); self.source_citations.borrow_mut().clear(); + self.answer_files.borrow_mut().clear(); + let health = codegraph_watch::watch_health(&self.project_root); let result = match tool_name { "codegraph_search" => self.handle_search(args), "codegraph_callers" => self.handle_callers(args), @@ -352,11 +357,51 @@ impl CodeGraphEngine { other => return ToolResult::error(format!("Unknown tool: {other}")), }; match result { - Ok(tr) => self.with_staleness_banner(tr), + Ok(tr) => { + let tr = match &health { + Some(health) if *health != codegraph_watch::WatchHealth::Healthy => { + self.refuse_drifted_answer(tr) + } + _ => tr, + }; + with_health_banner(self.with_staleness_banner(tr), health.as_ref()) + } Err(e) => ToolResult::error(format!("Tool execution failed: {e}")), } } + fn mark_answer_files<'a>(&self, nodes: impl IntoIterator) { + self.answer_files + .borrow_mut() + .extend(nodes.into_iter().map(|node| node.file_path.clone())); + } + + /// While auto-sync is paused or stopped the graph is frozen: a graph answer + /// that names a file changed (or gone) since its last sync would present + /// stale edges as current, so the drifted files are named instead of the + /// answer (upstream #1959). `codegraph_node` and explore keep their own + /// per-file drift gates, which serve or omit current bytes. + fn refuse_drifted_answer(&self, result: ToolResult) -> ToolResult { + if result.is_error == Some(true) { + return result; + } + let files = std::mem::take(&mut *self.answer_files.borrow_mut()); + let stale = files + .into_iter() + .filter(|file| self.project_source(file).is_possibly_drifted()) + .collect::>(); + if stale.is_empty() { + return result; + } + let mut lines = vec![ + "⚠️ CodeGraph cannot answer from this index:".to_string(), + "These files changed or became unavailable after their last sync:".to_string(), + ]; + lines.extend(stale.iter().map(|file| format!("- {file}"))); + lines.push("Retry after a successful codegraph sync, or narrow the query.".to_string()); + ToolResult::text(lines.join("\n")) + } + /// `codegraph_check` — additive cycle-detection tool (not in the upstream pin). /// Renders `find_circular_dependencies` /// (`upstream graph/queries.ts:225-263`) as file chains. @@ -421,6 +466,7 @@ impl CodeGraphEngine { ))); } let nodes: Vec = results.into_iter().map(|r| r.node).collect(); + self.mark_answer_files(&nodes); Ok(ToolResult::text(truncate_output(&format_search_results( &nodes, )))) @@ -481,9 +527,11 @@ impl CodeGraphEngine { } } } + self.mark_answer_files(&related); Ok(related) }; let selected = grouped.groups.iter().flatten().cloned().collect::>(); + self.mark_answer_files(&selected); let godot = match dir { CallDir::Callers => self.godot_honesty(&selected)?, CallDir::Callees => GodotHonesty::default(), @@ -636,9 +684,11 @@ impl CodeGraphEngine { } } } + self.mark_answer_files(nodes.values()); Ok((order, nodes)) }; let selected = grouped.groups.iter().flatten().cloned().collect::>(); + self.mark_answer_files(&selected); let godot = self.godot_honesty(&selected)?; if grouped.groups.len() == 1 { @@ -2011,6 +2061,18 @@ impl CodeGraphEngine { )); lines.push("**Backend:** node:sqlite (Node built-in) — full WAL + FTS5".to_string()); lines.push("**Journal mode:** wal (concurrent reads safe)".to_string()); + match codegraph_watch::watch_health(&self.project_root) { + Some(codegraph_watch::WatchHealth::Healthy) => { + lines.push("**Auto-sync:** watching".to_string()); + } + Some(codegraph_watch::WatchHealth::Recovering { reason }) => lines.push(format!( + "**Auto-sync:** RECOVERING — a full catch-up has not committed yet ({reason})" + )), + Some(codegraph_watch::WatchHealth::Disabled { reason }) => { + lines.push(format!("**Auto-sync:** DISABLED ({reason})")); + } + None => {} + } let by_kind = self.store.node_counts_by_kind()?; if !by_kind.is_empty() { @@ -3278,6 +3340,37 @@ impl Cluster { // === Free-function renderers (1:1 with upstream helpers) ==================== /// `formatSearchResults` (`tools.ts:3324-3338`). +/// Prepend the whole-index notice a paused or stopped watcher owes every +/// answer (upstream #876, #1959): the per-file drift banner cannot cover edits +/// the index never heard about while auto-sync was off. +fn with_health_banner( + mut result: ToolResult, + health: Option<&codegraph_watch::WatchHealth>, +) -> ToolResult { + if result.is_error == Some(true) { + return result; + } + let banner = match health { + Some(codegraph_watch::WatchHealth::Recovering { .. }) => { + "⚠️ CodeGraph auto-sync is RECOVERING — another process held the index lock, so \ + syncing paused; changes are still being collected, but the full index catch-up \ + has not completed. Read files directly to confirm current content before relying \ + on these results." + .to_string() + } + Some(codegraph_watch::WatchHealth::Disabled { reason }) => format!( + "⚠️ CodeGraph auto-sync is DISABLED — live file watching stopped, so the index is \ + frozen and any file edited since then is stale here. Read files directly to confirm \ + current content before relying on it.\n Reason: {reason}" + ), + Some(codegraph_watch::WatchHealth::Healthy) | None => return result, + }; + if let Some(first) = result.content.first_mut() { + first.text = format!("{banner}\n\n{}", first.text); + } + result +} + fn format_search_results(nodes: &[Node]) -> String { let mut lines = vec![ format!("## Search Results ({} found)", nodes.len()), @@ -4305,6 +4398,7 @@ mod tests { deprioritize: Arc::new(DeprioritizeMatcher::default()), source_probes: RefCell::new(HashMap::new()), source_citations: RefCell::new(HashSet::new()), + answer_files: RefCell::new(BTreeSet::new()), } } @@ -5461,6 +5555,112 @@ mod tests { ); } + /// While auto-sync is paused or stopped the graph is frozen, so a graph + /// answer that names a drifted file is refused, the file is named instead, + /// and every answer carries the whole-index notice (upstream #876, #1959). + #[test] + fn graph_answers_naming_a_drifted_file_are_refused_while_auto_sync_is_off() { + let mut engine = test_engine(); + put_indexed_source(&engine, "src/a.rs", "fn alpha() {}\n", Language::Rust, 1); + put_indexed_source(&engine, "src/b.rs", "fn beta() {}\n", Language::Rust, 1); + put_nodes( + &mut engine, + &[ + node_lang( + "alpha", + "alpha", + "src/a.rs", + 1, + 1, + NodeKind::Function, + Language::Rust, + ), + node_lang( + "beta", + "beta", + "src/b.rs", + 1, + 1, + NodeKind::Function, + Language::Rust, + ), + ], + ); + write_src(&engine, "src/a.rs", "fn alpha() { changed(); }\n"); + let search = |engine: &CodeGraphEngine, query: &str| { + text_of(&engine.execute("codegraph_search", &serde_json::json!({ "query": query }))) + }; + + // No watcher in this process: per-file gates only, answers as usual. + assert!(search(&engine, "alpha").contains("alpha")); + + let guard = codegraph_watch::register_watch_health_for_tests( + &engine.project_root, + codegraph_watch::WatchHealth::Recovering { + reason: "lock held".to_string(), + }, + ); + let refused = search(&engine, "alpha"); + assert!( + refused.starts_with("⚠️ CodeGraph auto-sync is RECOVERING"), + "{refused}" + ); + assert!( + refused.contains("cannot answer from this index"), + "{refused}" + ); + assert!(refused.contains("- src/a.rs"), "{refused}"); + let fresh = search(&engine, "beta"); + assert!( + fresh.starts_with("⚠️ CodeGraph auto-sync is RECOVERING"), + "{fresh}" + ); + assert!(!fresh.contains("cannot answer"), "{fresh}"); + assert!(fresh.contains("beta"), "{fresh}"); + drop(guard); + + let _guard = codegraph_watch::register_watch_health_for_tests( + &engine.project_root, + codegraph_watch::WatchHealth::Disabled { + reason: "inotify exhausted".to_string(), + }, + ); + let disabled = search(&engine, "alpha"); + assert!( + disabled.starts_with("⚠️ CodeGraph auto-sync is DISABLED"), + "{disabled}" + ); + assert!(disabled.contains("Reason: inotify exhausted"), "{disabled}"); + assert!(disabled.contains("- src/a.rs"), "{disabled}"); + } + + /// Indexing records a file over the limit by its size stamp, so the probe + /// compares it by that stamp, unread: a same-size rewrite is no drift + /// (upstream #1910), as it is no change to sync. + #[test] + fn project_source_compares_an_oversized_file_by_its_size_stamp() { + let mut engine = test_engine(); + let file = "src/oversized.rs"; + put_indexed_source(&engine, file, "fn oversized() {}\n", Language::Rust, 1); + let metadata = std::fs::metadata(engine.project_root.join(file)).unwrap(); + let mut stored = engine.store.file_by_path(file).unwrap().unwrap(); + stored.modified_at += 1; + stored.content_hash = + hash_content(&codegraph_core::source_file::oversize_stamp(metadata.len())); + put_file(&engine, &stored); + Arc::get_mut(&mut engine.config) + .unwrap() + .indexing + .max_file_size = metadata.len() - 1; + + let probe = engine.project_source(file); + assert_eq!(probe.freshness, SourceFreshness::ProvenFresh); + assert!( + probe.content.is_none(), + "oversized source bytes stay unread" + ); + } + #[test] fn ext_node_staleness_banner_is_only_emitted_for_drifted_files() { let mut engine = test_engine(); diff --git a/crates/codegraph-mcp/src/rmcp_handler.rs b/crates/codegraph-mcp/src/rmcp_handler.rs index c01387d3..c70eceae 100644 --- a/crates/codegraph-mcp/src/rmcp_handler.rs +++ b/crates/codegraph-mcp/src/rmcp_handler.rs @@ -414,8 +414,10 @@ fn execute_owned(project_path: &Path, tool_name: &str, args: &Value) -> ToolResu Ok(engine) => engine, Err(e) => { return ToolResult::error(format!( - "Failed to open project at {}: {e}", - project_path.display() + "Failed to open project at {}: {e}{}", + project_path.display(), + codegraph_store::wsl_shared_index_guidance(e.as_ref()) + .map_or(String::new(), |guidance| format!("\n{guidance}")) )); } }; @@ -998,22 +1000,18 @@ mod handler_tests { TempDir { path } } - /// Write a placeholder (non-SQLite) db file so `db_exists_for(p)` is true — - /// resolution treats the dir as indexed, but a real engine open fails. + /// Write a placeholder (non-SQLite) db file under a published namespace so + /// `db_exists_for(p)` is true — resolution treats the dir as indexed, but a + /// real engine open fails. fn placeholder_indexed(tag: &str) -> TempDir { let dir = unique_dir(tag); - let db = db_path_for(&dir.path).expect("default project resolves"); - std::fs::create_dir_all(db.parent().unwrap()).unwrap(); - std::fs::write(&db, b"not a real sqlite db").unwrap(); + crate::roots::write_index_fixture(&dir.path, b"not a real sqlite db"); dir } fn placeholder_indexed_child(workspace: &TempDir, name: &str) -> PathBuf { let child = workspace.path.join(name); - std::fs::create_dir_all(&child).unwrap(); - let db = db_path_for(&child).expect("child project resolves"); - std::fs::create_dir_all(db.parent().unwrap()).unwrap(); - std::fs::write(&db, b"not a real sqlite db").unwrap(); + crate::roots::write_index_fixture(&child, b"not a real sqlite db"); child } diff --git a/crates/codegraph-mcp/src/roots.rs b/crates/codegraph-mcp/src/roots.rs index 32d2c634..c5e19e4a 100644 --- a/crates/codegraph-mcp/src/roots.rs +++ b/crates/codegraph-mcp/src/roots.rs @@ -391,8 +391,16 @@ fn classify_resolve( ) -> RootStatus { use codegraph_core::IndexPathsError; match resolved { + // An index is a published namespace, not merely a file: a stray, empty or + // schema-less `codegraph.db` with no state record (an interrupted copy, a + // stray touch) must not capture the upward walk for every project below + // it (upstream #1895). The CLI's ancestor walk uses the same authority, + // which reads only the state slots and never opens SQLite. Ok(paths) => { - if paths.current_db().is_file() { + if paths.current_db().is_file() + && codegraph_store::Store::extraction_status(&paths) + != codegraph_store::ExtractionStatus::Missing + { RootStatus::Indexed } else { RootStatus::Absent @@ -696,6 +704,34 @@ fn percent_decode(input: &str) -> String { String::from_utf8_lossy(&out).into_owned() } +/// Test fixture: an index in the eyes of root discovery under `paths` — the +/// namespace created and published as Current, which is what makes a directory +/// an index (#1895), around a database file holding `db_bytes`. +#[cfg(test)] +pub(crate) fn write_index_fixture_at(paths: &codegraph_core::IndexPaths, db_bytes: &[u8]) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let lease = codegraph_store::IndexLease::create_exclusive(paths, deadline, || false) + .expect("fixture namespace lease"); + codegraph_store::publish_index_state(paths, &lease, codegraph_store::StatePhase::Building) + .expect("publish fixture building state"); + std::fs::create_dir_all(paths.current_db().parent().unwrap()).unwrap(); + std::fs::write(paths.current_db(), db_bytes).unwrap(); + codegraph_store::publish_index_state(paths, &lease, codegraph_store::StatePhase::Current) + .expect("publish fixture current state"); +} + +/// [`write_index_fixture_at`] for `project` under the current `CODEGRAPH_DIR`. +#[cfg(test)] +pub(crate) fn write_index_fixture(project: &Path, db_bytes: &[u8]) { + std::fs::create_dir_all(project).unwrap(); + let paths = codegraph_core::IndexPaths::resolve( + project, + std::env::var("CODEGRAPH_DIR").ok().as_deref(), + ) + .expect("fixture project resolves"); + write_index_fixture_at(&paths, db_bytes); +} + #[cfg(test)] mod tests { use super::*; @@ -723,12 +759,7 @@ mod tests { let seq = SEQ.fetch_add(1, Ordering::Relaxed); let path = std::env::temp_dir().join(format!("cg-mcp-roots-{tag}-{}-{seq}", std::process::id())); - // The project dir must exist before `db_path_for` (which resolves the - // physical identity) can succeed. - std::fs::create_dir_all(&path).unwrap(); - let db = db_path_for(&path).expect("default project resolves"); - std::fs::create_dir_all(db.parent().unwrap()).unwrap(); - std::fs::write(&db, b"placeholder").unwrap(); + write_index_fixture(&path, b"placeholder"); TempProject { path } } @@ -1140,12 +1171,30 @@ mod tests { )); std::fs::create_dir_all(&dir).unwrap(); let paths = codegraph_core::IndexPaths::resolve(&dir, None).expect("resolve default"); + write_index_fixture_at(&paths, b"placeholder"); + assert!(matches!( + classify_resolve(codegraph_core::IndexPaths::resolve(&dir, None)), + RootStatus::Indexed + )); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A database file with no published namespace is not an index (#1895). + #[test] + fn classify_resolve_unpublished_database_file_is_absent() { + let dir = std::env::temp_dir().join(format!( + "cg-roots-classify-unpublished-{}-{}", + std::process::id(), + SEQ.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&dir).unwrap(); + let paths = codegraph_core::IndexPaths::resolve(&dir, None).expect("resolve default"); let db = paths.current_db(); std::fs::create_dir_all(db.parent().unwrap()).unwrap(); std::fs::write(&db, b"placeholder").unwrap(); assert!(matches!( classify_resolve(codegraph_core::IndexPaths::resolve(&dir, None)), - RootStatus::Indexed + RootStatus::Absent )); let _ = std::fs::remove_dir_all(&dir); } @@ -1298,10 +1347,37 @@ mod tests { } fn mark_indexed(path: &Path) { - std::fs::create_dir_all(path).unwrap(); - let db = db_path_for(path).expect("indexed fixture path resolves"); - std::fs::create_dir_all(db.parent().unwrap()).unwrap(); - std::fs::write(db, b"placeholder").unwrap(); + write_index_fixture(path, b"placeholder"); + } + + /// A stray `codegraph.db` with no published namespace — an interrupted + /// copy, a stray touch — at a workspace root must not capture discovery: + /// the sub-project scan still finds the real index below it (#1895). + #[test] + fn a_stray_database_file_does_not_capture_root_discovery() { + let workspace = unindexed_dir("stray-db"); + std::fs::create_dir(workspace.path().join(".git")).unwrap(); + let stray = db_path_for(workspace.path()).expect("workspace resolves"); + std::fs::create_dir_all(stray.parent().unwrap()).unwrap(); + std::fs::write(&stray, b"").unwrap(); + let child = workspace.path().join("service-a"); + mark_indexed(&child); + + assert!(!db_exists_for(workspace.path())); + let resolved = resolve_server_root(workspace.path(), true); + assert_eq!(resolved.root.as_deref(), Some(child.as_path())); + assert!(resolved.via_subproject_scan); + + let nested = child.join("src"); + std::fs::create_dir_all(&nested).unwrap(); + assert_eq!( + resolve_server_root(&nested, false).root.as_deref(), + Some(child.as_path()) + ); + assert!(matches!( + resolve_project_arg(workspace.path().to_str(), None, None), + ProjectArg::NotIndexed + )); } #[test] diff --git a/crates/codegraph-mcp/src/server.rs b/crates/codegraph-mcp/src/server.rs index 87a86269..33d9446c 100644 --- a/crates/codegraph-mcp/src/server.rs +++ b/crates/codegraph-mcp/src/server.rs @@ -579,8 +579,10 @@ impl McpServer { Err(e) => { return Dispatch::Reply( serde_json::to_value(ToolResult::error(format!( - "Failed to open project at {}: {e}", - project_path.display() + "Failed to open project at {}: {e}{}", + project_path.display(), + codegraph_store::wsl_shared_index_guidance(e.as_ref()) + .map_or(String::new(), |guidance| format!("\n{guidance}")) ))) .expect("ToolResult serializes"), ); @@ -748,12 +750,7 @@ mod tests { let seq = SEQ.fetch_add(1, Ordering::Relaxed); let path = std::env::temp_dir().join(format!("cg-mcp-init-{tag}-{}-{seq}", std::process::id())); - // The project dir must exist before `db_path_for` (which resolves the - // physical identity) can succeed. - std::fs::create_dir_all(&path).unwrap(); - let db = db_path_for(&path).expect("default project resolves"); - std::fs::create_dir_all(db.parent().unwrap()).unwrap(); - std::fs::write(&db, b"placeholder").unwrap(); + crate::roots::write_index_fixture(&path, b"placeholder"); TempProject { path } } @@ -769,10 +766,7 @@ mod tests { fn indexed_child(workspace: &TempProject, name: &str) -> PathBuf { let child = workspace.path().join(name); - std::fs::create_dir_all(&child).unwrap(); - let db = db_path_for(&child).expect("child project resolves"); - std::fs::create_dir_all(db.parent().unwrap()).unwrap(); - std::fs::write(db, b"placeholder").unwrap(); + crate::roots::write_index_fixture(&child, b"placeholder"); child } diff --git a/crates/codegraph-resolve/src/context.rs b/crates/codegraph-resolve/src/context.rs index 912226e2..8aa550d6 100644 --- a/crates/codegraph-resolve/src/context.rs +++ b/crates/codegraph-resolve/src/context.rs @@ -32,6 +32,7 @@ pub(crate) const DEFAULT_CACHE_LIMIT: usize = 5_000; pub struct StoreResolutionContext<'a> { store: &'a Store, project_root: String, + max_file_size: u64, caches: RefCell, } @@ -50,8 +51,15 @@ struct Caches { workspace_packages: Option>, all_files: Option>>, files_by_basename: Option>>>, + /// Existence answers for the pass: import resolution probes every extension + /// of a specifier from every importing file, and neither the borrowed store + /// nor the tree changes within a pass (upstream `fileExistsMemo`). + file_exists: HashMap, } +/// Bound on memoized existence answers; the memo restarts past it, as upstream. +pub(crate) const FILE_EXISTS_MEMO_LIMIT: usize = 200_000; + impl Caches { fn new() -> Self { let limit = DEFAULT_CACHE_LIMIT; @@ -72,6 +80,7 @@ impl Caches { workspace_packages: None, all_files: None, files_by_basename: None, + file_exists: HashMap::new(), } } } @@ -82,10 +91,19 @@ impl<'a> StoreResolutionContext<'a> { Self { store, project_root: project_root.into(), + max_file_size: codegraph_core::config::DEFAULT_MAX_FILE_SIZE, caches: RefCell::new(Caches::new()), } } + /// Read files up to the addressed project's extraction limit + /// (`indexing.max_file_size`) instead of the default one. + #[must_use] + pub fn with_max_file_size(mut self, max_file_size: u64) -> Self { + self.max_file_size = max_file_size; + self + } + /// Drop all caches (`clearCaches`, `index.ts:313-324`). Called between /// resolution passes so the conformance pass sees freshly persisted edges. pub fn clear_caches(&self) { @@ -100,6 +118,7 @@ impl<'a> StoreResolutionContext<'a> { c.qualified_name_cache.clear(); c.all_files = None; c.files_by_basename = None; + c.file_exists.clear(); } fn cached_all_files(&self) -> Arc> { @@ -230,24 +249,28 @@ impl ResolutionContext for StoreResolutionContext<'_> { } fn file_exists(&self, file_path: &str) -> bool { + if let Some(&exists) = self.caches.borrow().file_exists.get(file_path) { + return exists; + } // Known-file fast path then filesystem fallback (index.ts:358-374). // The store is the index of known files. - if self.store.file_by_path(file_path).ok().flatten().is_some() { - return true; - } let normalized = file_path.replace('\\', "/"); - if normalized != file_path - && self - .store - .file_by_path(&normalized) - .ok() - .flatten() - .is_some() - { - return true; + let exists = self.store.file_by_path(file_path).ok().flatten().is_some() + || (normalized != file_path + && self + .store + .file_by_path(&normalized) + .ok() + .flatten() + .is_some()) + || pathutil::lexical_path_within_root(&self.project_root, file_path) + .is_some_and(|full_path| full_path.exists()); + let mut c = self.caches.borrow_mut(); + if c.file_exists.len() >= FILE_EXISTS_MEMO_LIMIT { + c.file_exists.clear(); } - pathutil::lexical_path_within_root(&self.project_root, file_path) - .is_some_and(|full_path| full_path.exists()) + c.file_exists.insert(file_path.to_string(), exists); + exists } fn read_file(&self, file_path: &str) -> Option { @@ -261,7 +284,8 @@ impl ResolutionContext for StoreResolutionContext<'_> { return c.file_cache.get(&file_path.to_string()).flatten(); } let full_path = Path::new(&self.project_root).join(file_path); - let content: Option> = std::fs::read_to_string(&full_path).ok().map(Arc::from); + let content: Option> = + read_source_file(&full_path, file_path, self.max_file_size).map(Arc::from); c.file_cache.set(file_path.to_string(), content.clone()); content } @@ -512,6 +536,22 @@ fn load_go_module(project_root: &str) -> Option { load_go_module_in(project_root, "") } +/// Read `path` (`relative` in the project) for resolution only if extraction +/// would index it: a regular file no larger than `max_bytes` that is not a video +/// clip named `.ts` (upstream #1553, #1910). The shared bounded reader stats it +/// first, so a dependency archive an import resolves to is never decoded and a +/// FIFO is never opened, and it decodes the text exactly as extraction does. +pub(crate) fn read_source_file(path: &Path, relative: &str, max_bytes: u64) -> Option { + match codegraph_core::source_file::read_source_file(path, relative, max_bytes) + .ok()? + .1 + { + codegraph_core::source_file::SourceText::Text(text) => Some(text), + codegraph_core::source_file::SourceText::Oversize(_) + | codegraph_core::source_file::SourceText::MpegTransportStream => None, + } +} + /// Read the `module` line of `//go.mod`. fn load_go_module_in(project_root: &str, dir: &str) -> Option { let relative = if dir.is_empty() { diff --git a/crates/codegraph-resolve/src/resolver.rs b/crates/codegraph-resolve/src/resolver.rs index 051759ce..57eed402 100644 --- a/crates/codegraph-resolve/src/resolver.rs +++ b/crates/codegraph-resolve/src/resolver.rs @@ -1284,6 +1284,9 @@ fn target_is_visible_across_files( /// `FrameworkResolver`s). pub struct ReferenceResolver { project_root: String, + /// Largest file resolution reads, the project's extraction limit + /// (`indexing.max_file_size`), handed to every context the resolver builds. + max_file_size: u64, /// Detected `FrameworkResolver` implementations (`frameworks` field, /// `index.ts:264`). Empty until [`Self::initialize`] detects react/vue/nestjs. framework_resolver_extensions: Vec>, @@ -1318,6 +1321,7 @@ impl ReferenceResolver { pub fn new(project_root: impl Into) -> Self { Self { project_root: project_root.into(), + max_file_size: codegraph_core::config::DEFAULT_MAX_FILE_SIZE, framework_resolver_extensions: Vec::new(), known_names: None, c_family_visible_macros: BTreeMap::new(), @@ -1327,6 +1331,25 @@ impl ReferenceResolver { } } + /// Read files up to the addressed project's extraction limit + /// (`indexing.max_file_size`) instead of the default one, so resolution + /// never reads a file extraction would skip and never skips one it indexed. + #[must_use] + pub fn with_max_file_size(mut self, max_file_size: u64) -> Self { + self.max_file_size = max_file_size; + self + } + + /// A store-backed resolution context for this resolver's project and file + /// size limit. + pub fn store_context<'a>( + &self, + store: &'a Store, + ) -> crate::context::StoreResolutionContext<'a> { + crate::context::StoreResolutionContext::new(store, &self.project_root) + .with_max_file_size(self.max_file_size) + } + /// Detect the project's frameworks and populate the resolver's /// `FrameworkResolver` list (`initialize`, `index.ts:263-266`). Must run /// before `resolve_all` so Strategy-1 dispatch sees the detected resolvers. @@ -1346,7 +1369,7 @@ impl ReferenceResolver { pub fn run_post_extract(&self, store: &mut Store) -> anyhow::Result { let mut all_updates: Vec = Vec::new(); { - let context = crate::context::StoreResolutionContext::new(store, &self.project_root); + let context = self.store_context(store); for resolver in &self.framework_resolver_extensions { if let Some(nodes) = resolver.post_extract(&context) { all_updates.extend(nodes); @@ -1414,8 +1437,11 @@ impl ReferenceResolver { context, extensions, |relative| { - std::fs::read_to_string(std::path::Path::new(&self.project_root).join(relative)) - .ok() + crate::context::read_source_file( + &std::path::Path::new(&self.project_root).join(relative), + relative, + self.max_file_size, + ) }, ) } @@ -2139,7 +2165,7 @@ impl ReferenceResolver { } let resolved: Vec = { - let context = crate::context::StoreResolutionContext::new(store, &self.project_root); + let context = self.store_context(store); deferred .iter() .filter_map(|reference| { @@ -2231,7 +2257,7 @@ impl ReferenceResolver { pub fn resolve_and_persist(&mut self, store: &mut Store) -> anyhow::Result { let unresolved_refs = store.all_unresolved_refs()?; let result = { - let context = crate::context::StoreResolutionContext::new(store, &self.project_root); + let context = self.store_context(store); self.resolve_all(&unresolved_refs, &context) }; @@ -2365,7 +2391,7 @@ impl ReferenceResolver { store.set_resolution_incomplete()?; { - let context = crate::context::StoreResolutionContext::new(store, &self.project_root); + let context = self.store_context(store); self.warm_caches(&context); } @@ -2428,18 +2454,17 @@ impl ReferenceResolver { // context. Prior batches' inheritance edges are already committed // (persisted at the bottom of this loop), so `get_supertypes` // reads them straight from the store — no snapshot, no adjacency. - let context = - crate::context::StoreResolutionContext::new(store, &self.project_root); + let context = self.store_context(store); self.resolve_all(&batch, &context) } else { let base = match &node_snapshot { Some(snapshot) => snapshot, None => { base_adjacency = (*build_edge_adjacency(store)?).clone(); - node_snapshot.insert(SnapshotResolutionContext::from_store( - store, - &self.project_root, - )?) + node_snapshot.insert( + SnapshotResolutionContext::from_store(store, &self.project_root)? + .with_max_file_size(self.max_file_size), + ) } }; // Install the adjacency from chunks 0..N-1 BEFORE resolving chunk N @@ -2614,7 +2639,7 @@ impl ReferenceResolver { let scoped = union_persisted_refs([by_files, by_sites, by_names])?; let result = { - let context = crate::context::StoreResolutionContext::new(store, &self.project_root); + let context = self.store_context(store); self.resolve_all(&scoped, &context) }; diff --git a/crates/codegraph-resolve/src/snapshot_context.rs b/crates/codegraph-resolve/src/snapshot_context.rs index 892143c8..53ab2e08 100644 --- a/crates/codegraph-resolve/src/snapshot_context.rs +++ b/crates/codegraph-resolve/src/snapshot_context.rs @@ -174,6 +174,10 @@ struct SnapshotCaches { source_facts: BoundedFileMemo>>, import_mappings: BoundedFileMemo>>, re_exports: BoundedFileMemo>>, + /// Filesystem existence probes for paths the snapshot does not know: the + /// same candidates are probed from every importing file, and the tree does + /// not change within a pass (upstream `fileExistsMemo`). + fs_exists: BoundedFileMemo, #[cfg(test)] counters: SnapshotCacheCounters, } @@ -188,6 +192,7 @@ impl SnapshotCaches { source_facts: BoundedFileMemo::new(content_limit), import_mappings: BoundedFileMemo::new(DEFAULT_CACHE_LIMIT), re_exports: BoundedFileMemo::new(DEFAULT_CACHE_LIMIT), + fs_exists: BoundedFileMemo::new(crate::context::FILE_EXISTS_MEMO_LIMIT), #[cfg(test)] counters: SnapshotCacheCounters::default(), } @@ -230,6 +235,8 @@ struct NodeSnapshot { pub struct SnapshotResolutionContext { snapshot: Arc, caches: Arc, + /// Largest file read for resolution, the project's extraction limit. + max_file_size: u64, /// Per-chunk `implements`/`extends` adjacency for [`Self::get_supertypes`]. /// Empty until the resolver installs a chunk's map (T4); an empty map yields /// the same result the store context gives before any such edge exists. @@ -367,6 +374,7 @@ impl SnapshotResolutionContext { go_modules, }), caches: Arc::new(SnapshotCaches::new()), + max_file_size: codegraph_core::config::DEFAULT_MAX_FILE_SIZE, edges: Arc::new(HashMap::new()), }) } @@ -379,7 +387,8 @@ impl SnapshotResolutionContext { .file_loads .fetch_add(1, Ordering::Relaxed); let full_path = Path::new(&self.snapshot.project_root).join(file_path); - std::fs::read_to_string(full_path).ok().map(Arc::from) + crate::context::read_source_file(&full_path, file_path, self.max_file_size) + .map(Arc::from) }) } @@ -397,10 +406,20 @@ impl SnapshotResolutionContext { Self { snapshot: Arc::clone(&self.snapshot), caches: Arc::clone(&self.caches), + max_file_size: self.max_file_size, edges, } } + /// Read files up to the addressed project's extraction limit + /// (`indexing.max_file_size`) instead of the default one. Set it before the + /// first read: per-chunk clones share one file cache. + #[must_use] + pub fn with_max_file_size(mut self, max_file_size: u64) -> Self { + self.max_file_size = max_file_size; + self + } + #[cfg(test)] fn cache_counts(&self) -> (usize, usize, usize) { ( @@ -485,8 +504,10 @@ impl ResolutionContext for SnapshotResolutionContext { if normalized != file_path && self.snapshot.known_file_paths.contains(&normalized) { return true; } - pathutil::lexical_path_within_root(&self.snapshot.project_root, file_path) - .is_some_and(|full_path| full_path.exists()) + self.caches.fs_exists.get_or_init(file_path, || { + pathutil::lexical_path_within_root(&self.snapshot.project_root, file_path) + .is_some_and(|full_path| full_path.exists()) + }) } fn read_file(&self, file_path: &str) -> Option { diff --git a/crates/codegraph-resolve/tests/resolution_file_reads.rs b/crates/codegraph-resolve/tests/resolution_file_reads.rs new file mode 100644 index 00000000..9c68ef0d --- /dev/null +++ b/crates/codegraph-resolve/tests/resolution_file_reads.rs @@ -0,0 +1,283 @@ +//! Resolution reads only what extraction would accept (upstream v1.6.1 #1553): +//! a regular file no larger than the extraction size limit, checked before any +//! byte is read, with the rejection cached for the pass like any other read. + +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +use codegraph_core::config::DEFAULT_MAX_FILE_SIZE; +use codegraph_resolve::{ + ReferenceResolver, ResolutionContext, SnapshotResolutionContext, StoreResolutionContext, +}; +use codegraph_store::Store; + +static NONCE: AtomicU64 = AtomicU64::new(0); + +const LIMIT: usize = DEFAULT_MAX_FILE_SIZE as usize; + +struct Project { + root: PathBuf, + store: Option, +} + +impl Drop for Project { + fn drop(&mut self) { + drop(self.store.take()); + let _ = std::fs::remove_dir_all(&self.root); + } +} + +impl Project { + fn new(tag: &str) -> Self { + let root = std::env::temp_dir().join(format!( + "codegraph-resolution-reads-{tag}-{}-{}", + std::process::id(), + NONCE.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&root).expect("create fixture root"); + let store = Store::open(&root.join("index.db")).expect("open store"); + Self { + root, + store: Some(store), + } + } + + fn store(&self) -> &Store { + self.store.as_ref().expect("store") + } + + fn root_str(&self) -> String { + self.root.to_string_lossy().to_string() + } + + fn write(&self, relative: &str, content: impl AsRef<[u8]>) { + let path = self.root.join(relative); + std::fs::create_dir_all(path.parent().expect("parent")).expect("create parent"); + std::fs::write(path, content).expect("write fixture"); + } + + /// A 2 MiB sparse package archive with a gzip header, as `file:` package + /// metadata can point an import at. + fn sparse_archive(&self) -> &'static str { + let relative = "node_modules/example/react_native_openharmony.har"; + let path = self.root.join(relative); + std::fs::create_dir_all(path.parent().expect("parent")).expect("create parent"); + std::fs::write(&path, [0x1f, 0x8b]).expect("write archive header"); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .and_then(|file| file.set_len(2 * 1024 * 1024)) + .expect("extend archive"); + relative + } +} + +fn contexts(project: &Project) -> (StoreResolutionContext<'_>, SnapshotResolutionContext) { + ( + StoreResolutionContext::new(project.store(), project.root_str()), + SnapshotResolutionContext::from_store(project.store(), project.root_str()) + .expect("snapshot"), + ) +} + +#[test] +fn reads_stop_at_the_extraction_size_limit_in_both_contexts() { + let project = Project::new("limit"); + let at_limit = "a".repeat(LIMIT); + project.write("small.ts", "export const answer = 42;\n"); + project.write("boundary.ts", &at_limit); + project.write("bundle.js", "a".repeat(LIMIT + 1)); + std::fs::create_dir_all(project.root.join("directory")).expect("mkdir"); + let archive = project.sparse_archive(); + + let (store_context, snapshot) = contexts(&project); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert_eq!( + context.read_file("small.ts").as_deref(), + Some("export const answer = 42;\n") + ); + assert!(context.read_file("boundary.ts") == Some(at_limit.clone())); + for rejected in ["bundle.js", archive, "directory", "missing.ts"] { + assert_eq!( + context.read_file(rejected).map(|content| content.len()), + None, + "{rejected}" + ); + assert!(!context.is_file_readable(rejected), "{rejected}"); + } + } +} + +/// Resolution decodes a file exactly as extraction does (upstream #1910): bytes +/// that are not UTF-8 become U+FFFD, and a video clip named `.ts` is not +/// source, so neither context reads it. +#[test] +fn reads_decode_like_extraction_and_skip_video_clips() { + let project = Project::new("decode"); + project.write("legacy.c", b"/* caf\xe9 */\n"); + let mut clip = Vec::new(); + for packet in 0..64_u32 { + clip.push(0x47); + clip.extend((0..187_u32).map(|byte| ((packet * 31 + byte * 7) % 32) as u8)); + } + project.write("clip.ts", &clip); + let (store_context, snapshot) = contexts(&project); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert_eq!( + context.read_file("legacy.c").as_deref(), + Some("/* caf\u{fffd} */\n") + ); + assert_eq!(context.read_file("clip.ts"), None); + } +} + +#[test] +fn a_rejection_is_cached_for_the_pass_and_the_next_pass_rereads() { + let project = Project::new("cache"); + let archive = project.sparse_archive(); + let (store_context, snapshot) = contexts(&project); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert_eq!(context.read_file(archive), None); + } + + project.write(archive, "export const repaired = true;"); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert_eq!(context.read_file(archive), None); + } + let (store_context, snapshot) = contexts(&project); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert_eq!( + context.read_file(archive).as_deref(), + Some("export const repaired = true;") + ); + } +} + +/// A project that raises `max_file_size` indexes larger files, so resolution +/// must keep reading them: the limit follows the project, through every context +/// the resolver builds. +#[test] +fn a_raised_max_file_size_reaches_every_context() { + let project = Project::new("raised"); + let generated = "a".repeat(LIMIT + 1); + project.write("generated.ts", &generated); + let raised = 2 * DEFAULT_MAX_FILE_SIZE; + + let store_context = + StoreResolutionContext::new(project.store(), project.root_str()).with_max_file_size(raised); + let snapshot = SnapshotResolutionContext::from_store(project.store(), project.root_str()) + .expect("snapshot") + .with_max_file_size(raised); + let resolver = ReferenceResolver::new(project.root_str()).with_max_file_size(raised); + let resolver_context = resolver.store_context(project.store()); + for context in [ + &store_context as &dyn ResolutionContext, + &snapshot, + &resolver_context, + ] { + assert!(context.read_file("generated.ts") == Some(generated.clone())); + } + let default_context = ReferenceResolver::new(project.root_str()).store_context(project.store()); + assert_eq!( + default_context + .read_file("generated.ts") + .map(|content| content.len()), + None + ); +} + +/// A FIFO would block a plain read forever; it is not a regular file, so it is +/// rejected before it is opened. +#[cfg(unix)] +#[test] +fn a_fifo_is_rejected_without_blocking() { + let project = Project::new("fifo"); + let fifo = project.root.join("pipe.ts"); + let status = std::process::Command::new("mkfifo") + .arg(&fifo) + .status() + .expect("run mkfifo"); + assert!(status.success(), "mkfifo failed"); + + let root = project.root_str(); + let db = project.root.join("index.db"); + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let store = Store::open(&db).expect("open store"); + let store_read = StoreResolutionContext::new(&store, root.clone()).read_file("pipe.ts"); + let snapshot_read = SnapshotResolutionContext::from_store(&store, root) + .expect("snapshot") + .read_file("pipe.ts"); + let _ = sender.send((store_read, snapshot_read)); + }); + let reads = receiver + .recv_timeout(std::time::Duration::from_secs(10)) + .expect("reading a FIFO blocked"); + assert_eq!(reads, (None, None)); +} + +/// The framework pass reads project files through the same guard, so a file +/// extraction skipped for its size yields no framework output either. +#[test] +fn framework_extraction_skips_a_file_too_large_to_extract() { + let project = Project::new("framework"); + project.write( + "src-tauri/src/main.rs", + "#[tauri::command]\nfn save_config() {}\n", + ); + project.write( + "src-tauri/tauri.conf.json", + "{ \"productName\": \"x\", \"identifier\": \"dev.x\", \"build\": {}, \"app\": {} }\n", + ); + let call = "import { invoke } from '@tauri-apps/api/core';\ninvoke('save_config');\n"; + project.write("src/app.ts", call); + project.write("src/bundle.ts", format!("{call}{}", " ".repeat(LIMIT))); + + let mut resolver = ReferenceResolver::new(project.root_str()); + { + let context = resolver.store_context(project.store()); + resolver.initialize(&context); + } + assert!(resolver.has_framework_resolvers()); + let collection = resolver.collect_framework_extraction_with( + &["src/app.ts".to_string(), "src/bundle.ts".to_string()], + &codegraph_resolve::framework::FrameworkExtractionContext::without_config( + project.root_str(), + ), + &codegraph_extract::ExtensionOverrides::default(), + ); + let sources = collection + .unresolved_references + .iter() + .map(|reference| reference.file_path.as_str()) + .collect::>(); + assert_eq!(sources, vec!["src/app.ts"]); +} + +/// Import resolution probes the same candidate paths (every extension of a +/// specifier) from every file that imports it, and the tree does not change +/// within a pass, so a pass answers each existence probe once (upstream +/// v1.6.1's `fileExistsMemo`); the next pass probes afresh. +#[test] +fn existence_probes_are_answered_once_per_pass() { + let project = Project::new("exists-memo"); + project.write("present.ts", "export const a = 1;\n"); + let (store_context, snapshot) = contexts(&project); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert!(context.file_exists("present.ts")); + assert!(!context.file_exists("later.ts")); + } + + project.write("later.ts", "export const b = 2;\n"); + std::fs::remove_file(project.root.join("present.ts")).expect("remove"); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert!(context.file_exists("present.ts"), "memoized for the pass"); + assert!(!context.file_exists("later.ts"), "memoized for the pass"); + } + + let (store_context, snapshot) = contexts(&project); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert!(!context.file_exists("present.ts")); + assert!(context.file_exists("later.ts")); + } +} diff --git a/crates/codegraph-store/src/index_state.rs b/crates/codegraph-store/src/index_state.rs index 08aa794a..089357a2 100644 --- a/crates/codegraph-store/src/index_state.rs +++ b/crates/codegraph-store/src/index_state.rs @@ -115,7 +115,7 @@ pub const CURRENT_STORAGE_PROTOCOL: u64 = 2; /// The extraction-pipeline version this binary produces. This is the single /// source of truth: no other crate may define its own copy. -pub const CURRENT_EXTRACTION_VERSION: u64 = 15; +pub const CURRENT_EXTRACTION_VERSION: u64 = 16; /// The `project_metadata` key under which a built index records the extraction /// version it was produced with. Single source of truth for the key spelling. diff --git a/crates/codegraph-store/src/lib.rs b/crates/codegraph-store/src/lib.rs index cc97cef4..255dfcdd 100644 --- a/crates/codegraph-store/src/lib.rs +++ b/crates/codegraph-store/src/lib.rs @@ -10,6 +10,7 @@ pub mod schema; #[cfg(feature = "test-hooks")] pub mod test_support; pub mod uninit; +pub mod wsl_shared_index; pub use connection::{ ExtractionStampIssue, Store, StoreError, StoreStatusOpen, StoreWriteAuthorization, @@ -33,3 +34,4 @@ pub use rebuild::{ resume_full_rebuild, }; pub use uninit::{UninitError, UninitOutcome, uninit_index, uninit_index_with_drain}; +pub use wsl_shared_index::wsl_shared_index_guidance; diff --git a/crates/codegraph-store/src/wsl_shared_index.rs b/crates/codegraph-store/src/wsl_shared_index.rs new file mode 100644 index 00000000..801f12b5 --- /dev/null +++ b/crates/codegraph-store/src/wsl_shared_index.rs @@ -0,0 +1,109 @@ +//! Windows and WSL sharing one index on a Windows drive (upstream #995). +//! +//! A repo under `/mnt//` can be used from both sides: Windows-native +//! CodeGraph and WSL CodeGraph then open the same `.codegraph/codegraph.db` +//! across the 9p/DrvFs bridge, where SQLite's file locking and its `-shm` +//! shared memory do not hold between the two systems, and WSL's connection +//! fails with a bare "disk I/O error". A fresh WSL index there already gets its +//! own `.codegraph-wsl` (`IndexPaths::resolve`); an index kept in `.codegraph` +//! that fails this way gets the instruction below instead of an unexplained +//! I/O failure. + +use std::error::Error; +use std::path::Path; + +use codegraph_core::index_paths::{DEFAULT_CURRENT_DIR, WSL_CURRENT_DIR}; + +use crate::StoreError; + +/// Guidance for `error` when a SQLite I/O failure hit the default-named index +/// of a project on a Windows drive under WSL; `None` otherwise. Only the +/// default `.codegraph` counts: it is the directory Windows CodeGraph opens +/// too, and an I/O failure on any other root has some other cause. +pub fn wsl_shared_index_guidance(error: &(dyn Error + 'static)) -> Option { + guidance_for(error, codegraph_core::wsl::is_wsl_windows_drive) +} + +fn guidance_for( + error: &(dyn Error + 'static), + on_wsl_windows_drive: impl Fn(&Path) -> bool, +) -> Option { + let mut cause = Some(error); + let db = std::iter::from_fn(|| { + let current = cause?; + cause = current.source(); + Some(current) + }) + .find_map(|error| io_failure_database(error.downcast_ref::()?))?; + let data_dir = db.parent()?; + if data_dir.file_name()? != DEFAULT_CURRENT_DIR || !on_wsl_windows_drive(db) { + return None; + } + Some(format!( + "This project is on a Windows drive, where SQLite's file locking doesn't work across \ + the Windows/WSL boundary, so this usually means CodeGraph on Windows is using the same \ + index at {}. Windows and WSL can't share one index on a Windows drive; give WSL its own:\n\ + \x20 1. Set CODEGRAPH_DIR={WSL_CURRENT_DIR} in the WSL environment (your shell profile, \ + or the env of the MCP server your agent starts).\n\ + \x20 2. Run \"codegraph init\" in WSL to build that index. Windows keeps using \ + {DEFAULT_CURRENT_DIR}.\n", + data_dir.display() + )) +} + +/// The database a SQLite I/O failure was reported against, if `error` is one. +fn io_failure_database(error: &StoreError) -> Option<&Path> { + let (path, source) = match error { + StoreError::Open { path, source } + | StoreError::Configure { path, source } + | StoreError::Migrate { path, source } + | StoreError::ReadExtractionStamp { path, source } + | StoreError::Close { path, source } + | StoreError::CheckpointWal { path, source } => (path, source), + _ => return None, + }; + is_sqlite_io_error(source).then_some(path.as_path()) +} + +/// SQLite reports an I/O failure as `SQLITE_IOERR`, whose extended codes keep +/// the primary code in their low byte. +fn is_sqlite_io_error(error: &rusqlite::Error) -> bool { + matches!( + error, + rusqlite::Error::SqliteFailure(failure, _) if failure.extended_code & 0xff == 10 + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + fn io_error(path: &str) -> StoreError { + StoreError::Configure { + path: PathBuf::from(path), + // SQLITE_IOERR_SHMMAP: the -shm mapping the bridge cannot share. + source: rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(5130), None), + } + } + + #[test] + fn an_io_error_on_a_shared_default_index_explains_the_wsl_fix() { + let error = io_error("/mnt/c/repo/.codegraph/codegraph.db"); + let guidance = guidance_for(&error, |_| true).expect("guidance"); + assert!( + guidance.contains("CODEGRAPH_DIR=.codegraph-wsl"), + "{guidance}" + ); + assert!(guidance.contains("/mnt/c/repo/.codegraph"), "{guidance}"); + + assert_eq!(guidance_for(&error, |_| false), None, "not WSL"); + let own_root = io_error("/mnt/c/repo/.codegraph-wsl/codegraph.db"); + assert_eq!(guidance_for(&own_root, |_| true), None, "WSL's own index"); + let not_io = StoreError::Configure { + path: PathBuf::from("/mnt/c/repo/.codegraph/codegraph.db"), + source: rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(5), None), + }; + assert_eq!(guidance_for(¬_io, |_| true), None, "SQLITE_BUSY"); + } +} diff --git a/crates/codegraph-store/tests/index_state.rs b/crates/codegraph-store/tests/index_state.rs index df760d91..dba33b6f 100644 --- a/crates/codegraph-store/tests/index_state.rs +++ b/crates/codegraph-store/tests/index_state.rs @@ -119,7 +119,7 @@ fn assert_corrupt( fn index_state_constants_and_canonical_payload_are_exact() { /// The extraction version the immediately preceding release wrote. Read from /// `index_state.rs` at implementation time; see plan decision C1 step 0. - const PREVIOUS_EXTRACTION_VERSION: u64 = 14; + const PREVIOUS_EXTRACTION_VERSION: u64 = 15; assert_eq!(CURRENT_STORAGE_PROTOCOL, 2); assert_eq!(CURRENT_EXTRACTION_VERSION, PREVIOUS_EXTRACTION_VERSION + 1); diff --git a/crates/codegraph-watch/Cargo.toml b/crates/codegraph-watch/Cargo.toml index f52b5205..a3496769 100644 --- a/crates/codegraph-watch/Cargo.toml +++ b/crates/codegraph-watch/Cargo.toml @@ -13,6 +13,11 @@ rust-version.workspace = true description = "Debounced file watcher and incremental sync for CodeGraph-rs." readme = "../../README.md" +[features] +# Test seams for dependents' test builds (e.g. publishing a watcher health +# without a running watcher). NEVER enabled in the shipped binary. +test-hooks = [] + [dependencies] serde = { workspace = true } anyhow = { workspace = true } diff --git a/crates/codegraph-watch/src/lib.rs b/crates/codegraph-watch/src/lib.rs index 2ac7b8bd..97080c2d 100644 --- a/crates/codegraph-watch/src/lib.rs +++ b/crates/codegraph-watch/src/lib.rs @@ -17,8 +17,11 @@ pub use sync::{ sync_project_once, sync_project_once_cancellable, sync_project_once_with_progress, }; pub use watcher::{ - PendingFile, ProjectWatcher, WatchOptions, start_serve_watcher, watch_options_for_project, + LOCK_RECOVERY_INTERVAL, PendingFile, ProjectWatcher, WatchHealth, WatchOptions, + start_serve_watcher, watch_health, watch_options_for_project, }; +#[cfg(feature = "test-hooks")] +pub use watcher::{WatchHealthGuard, register_watch_health_for_tests}; pub use worktree::{ WorktreeIndexMismatch, detect_worktree_index_mismatch, git_worktree_root, worktree_mismatch_notice, worktree_mismatch_warning, diff --git a/crates/codegraph-watch/src/migrate.rs b/crates/codegraph-watch/src/migrate.rs index 4ddce4cf..1299ed1f 100644 --- a/crates/codegraph-watch/src/migrate.rs +++ b/crates/codegraph-watch/src/migrate.rs @@ -37,7 +37,7 @@ use codegraph_core::IndexPaths; use codegraph_core::generated_header::detect_generated_file; use codegraph_core::node_id::hash_content; use codegraph_core::types::{Edge, FileRecord, Node, UnresolvedRef}; -use codegraph_extract::{detect_language_with, extract_source_with}; +use codegraph_extract::detect_language_with; use codegraph_resolve::ReferenceResolver; use codegraph_store::StoreWriteAuthorization; @@ -93,30 +93,21 @@ pub(crate) fn migrate_project( for (done, relative) in candidates.iter().enumerate() { let full = project_root.join(relative); - // One metadata + one source read per file, mirroring the CLI producer so - // the oversized-file skip message is byte-identical. - let metadata = std::fs::metadata(&full) - .with_context(|| format!("reading metadata for {}", full.display()))?; - let source = std::fs::read_to_string(&full) - .with_context(|| format!("reading source file {}", full.display()))?; - let mut result = if metadata.len() > options.max_file_size { - codegraph_core::types::ExtractionResult { - nodes: Vec::new(), - edges: Vec::new(), - unresolved_references: Vec::new(), - errors: vec![format!( - "File exceeds max size ({} > {}): {relative}", - metadata.len(), - options.max_file_size - )], - duration_ms: 0, - } - } else { - extract_source_with(relative, &source, None, &options.extensions) + // One bounded read per file through the reader the CLI producer uses, so + // the oversized-file skip message and the size stamp are byte-identical. + let (metadata, source) = + codegraph_extract::read_source_file(&full, relative, options.max_file_size) + .with_context(|| format!("reading source file {}", full.display()))?; + // A video clip named `.ts` is not source, so it gets no record (#1910). + let Some(hash_input) = source.hash_input() else { + on_progress(done + 1, total); + continue; }; + let mut result = + codegraph_extract::engine::extraction_of(relative, &source, options, |_| {}); let file = FileRecord { path: relative.clone(), - content_hash: hash_content(&source), + content_hash: hash_content(&hash_input), language: detect_language_with(relative, &options.extensions), size: metadata.len() as i64, modified_at: modified_millis(&metadata), @@ -127,7 +118,7 @@ pub(crate) fn migrate_project( .filter(|node| node.file_path == *relative) .count() as i64, errors: result.errors.clone(), - generated: detect_generated_file(relative, &source), + generated: detect_generated_file(relative, &hash_input), }; rebuild.store().upsert_file(&file)?; @@ -164,12 +155,10 @@ pub(crate) fn migrate_project( } drop(refs); - let mut resolver = ReferenceResolver::new(project_root.to_string_lossy()); + let mut resolver = ReferenceResolver::new(project_root.to_string_lossy()) + .with_max_file_size(options.max_file_size); { - let context = codegraph_resolve::StoreResolutionContext::new( - rebuild.store(), - project_root.to_string_lossy(), - ); + let context = resolver.store_context(rebuild.store()); resolver.initialize(&context); } if resolver.has_framework_resolvers() { diff --git a/crates/codegraph-watch/src/policy.rs b/crates/codegraph-watch/src/policy.rs index 9b0623d9..21c38406 100644 --- a/crates/codegraph-watch/src/policy.rs +++ b/crates/codegraph-watch/src/policy.rs @@ -1,8 +1,7 @@ -use std::fs; use std::path::{Path, PathBuf}; use std::sync::Arc; -use codegraph_extract::{ExtensionOverrides, detect_language_with}; +use codegraph_extract::{ExtensionOverrides, RootGitignore, detect_language_with}; pub const CODEGRAPH_NO_WATCH: &str = "CODEGRAPH_NO_WATCH"; @@ -19,12 +18,6 @@ const WATCH_ONLY_DEFAULT_IGNORE_DIRS: &[&str] = &[ ".cache", ]; -#[derive(Debug, Clone)] -struct IgnoreRule { - pattern: String, - negated: bool, -} - #[derive(Debug, Clone)] pub struct WatchPolicy { root: PathBuf, @@ -41,11 +34,12 @@ pub struct WatchPolicy { /// uses, never [`rule_matches`] — whose `*` form compares BASENAMES, so it /// could never match a `res/values*` prefix against `res/values/strings.xml`. ignore_paths: Vec, - /// The root `.gitignore` rules in file order — the NEGOTIABLE tail of the - /// last-match-wins stream (`ignore_paths` and `exclude` first, these last, - /// mirroring the scan's `pattern_sets`), so a `!pattern` line here re-includes - /// what an `ignore_paths`, `exclude`, or earlier `.gitignore` line dropped. - gitignore_rules: Vec, + /// The root `.gitignore` — the NEGOTIABLE tail of the last-match-wins stream + /// (`ignore_paths` and `exclude` first, it last, mirroring the scan), so a + /// `!pattern` line here re-includes what an `ignore_paths`, `exclude`, or + /// earlier `.gitignore` line dropped. It is the scan's own git-rules + /// matcher, so the two read every `.gitignore` line identically. + gitignore: RootGitignore, include: Vec, exclude: Vec, /// The addressed project's custom extension→language overrides, so a file the @@ -95,12 +89,12 @@ impl WatchPolicy { .map(str::to_string), ) .collect::>(); - let gitignore_rules = read_gitignore_rules(&root); + let gitignore = RootGitignore::load(&root); Self { root, structural_ignores, ignore_paths: ignore_paths.to_vec(), - gitignore_rules, + gitignore, include: include.to_vec(), exclude: exclude.to_vec(), extensions: ExtensionOverrides::empty(), @@ -191,9 +185,13 @@ impl WatchPolicy { } fn matches_structural(&self, relative: &str, is_dir: bool) -> bool { - self.structural_ignores - .iter() - .any(|pattern| rule_matches(pattern, relative, is_dir)) + self.structural_ignores.iter().any(|pattern| { + if pattern == "build/" { + build_segment_is_output(relative, is_dir) + } else { + rule_matches(pattern, relative, is_dir) + } + }) } /// The watcher's port of the scan's `is_path_ignored` fold over its ordered @@ -204,20 +202,16 @@ impl WatchPolicy { /// `gen/helper.ts` — and the set order is observable, so a `!` in `exclude` /// re-includes an `ignore_paths` match but not the reverse. /// - /// The ONLY deviation from the scan is which matcher each set uses, and it - /// is LOAD-BEARING — do not "simplify" this into a single matcher: - /// the two CONFIG sets use the SHARED whole-path - /// [`codegraph_extract::include_exclude_pattern_matches`] the scan itself - /// uses (its trailing-`*` form is a whole-path prefix, so `res/values*` - /// matches `res/values/strings.xml`), while `.gitignore` keeps the - /// watcher's basename-glob [`rule_matches`]. A `!` in a config set is - /// stripped and then re-matched with that set's OWN matcher, mirroring how - /// the scan strips then calls `pattern_matches`. The config sets are also - /// matched WITHOUT `is_dir`, exactly as the scan feeds a bare `relative` to - /// `is_path_ignored`; only the `.gitignore` fold is `is_dir`-aware. The - /// `.gitignore` rules arrive pre-parsed into [`IgnoreRule`] by - /// [`read_gitignore_rules`], so their `!` is already split off, whereas the - /// config sets are raw patterns with the `!` still inline. + /// Each set uses the scan's own matcher: the two CONFIG sets the SHARED + /// whole-path [`codegraph_extract::include_exclude_pattern_matches`] (its + /// trailing-`*` form is a whole-path prefix, so `res/values*` matches + /// `res/values/strings.xml`), and `.gitignore` the SHARED git-rules + /// [`RootGitignore`]. A `!` in a config set is stripped and then re-matched + /// with that set's OWN matcher, mirroring how the scan strips then calls + /// `pattern_matches`. The config sets are also matched WITHOUT `is_dir`, + /// exactly as the scan feeds a bare `relative` to `is_path_ignored`; only + /// the `.gitignore` verdict is `is_dir`-aware, and it walks the directories + /// above `relative` the way the scan carries them down its walk. fn matches_negotiable(&self, relative: &str, is_dir: bool) -> bool { let mut ignored = false; for set in [&self.ignore_paths, &self.exclude] { @@ -231,12 +225,7 @@ impl WatchPolicy { } } } - for rule in &self.gitignore_rules { - if rule_matches(&rule.pattern, relative, is_dir) { - ignored = !rule.negated; - } - } - ignored + self.gitignore.verdict(relative, is_dir).unwrap_or(ignored) } /// The scan's `IncludeSet::forces`/`wants_descend` knockout check, which is a @@ -311,7 +300,7 @@ pub fn watch_disabled_reason(project_root: impl AsRef, no_watch: bool) -> if std::env::var("CODEGRAPH_FORCE_WATCH").as_deref() == Ok("1") { return None; } - if detect_wsl() && is_windows_drive_mount(project_root.as_ref()) { + if codegraph_core::wsl::is_wsl_windows_drive(project_root.as_ref()) { return Some( "project is on a WSL2 /mnt/ drive, where recursive fs.watch is too slow to be reliable" .to_string(), @@ -428,24 +417,23 @@ pub fn normalize_path(path: impl AsRef) -> String { .replace('\\', "/") } -fn read_gitignore_rules(root: &Path) -> Vec { - fs::read_to_string(root.join(".gitignore")) - .unwrap_or_default() - .lines() - .filter_map(|line| { - let trimmed = line.trim(); - if trimmed.is_empty() || trimmed.starts_with('#') { - return None; - } - let (negated, pattern) = trimmed - .strip_prefix('!') - .map_or((false, trimmed), |pattern| (true, pattern)); - Some(IgnoreRule { - pattern: pattern.trim_start_matches('/').to_string(), - negated, - }) - }) - .collect() +/// The `build/` structural rule, segment by segment: a `build` directory on the +/// path prunes it unless it is a JVM package under a conventional source root +/// (#1642, mirroring `scan_dir`). +fn build_segment_is_output(relative: &str, is_dir: bool) -> bool { + let segments = relative + .trim_end_matches('/') + .split('/') + .collect::>(); + let directories = if is_dir { + segments.len() + } else { + segments.len().saturating_sub(1) + }; + (0..directories).any(|index| { + segments[index] == "build" + && !codegraph_core::config::is_jvm_source_build_dir(&segments[..=index].join("/")) + }) } fn rule_matches(pattern: &str, relative: &str, is_dir: bool) -> bool { @@ -474,34 +462,12 @@ fn rule_matches(pattern: &str, relative: &str, is_dir: bool) -> bool { relative == pattern || relative.ends_with(&format!("/{pattern}")) } -fn detect_wsl() -> bool { - if !cfg!(target_os = "linux") { - return false; - } - if std::env::var_os("WSL_DISTRO_NAME").is_some() || std::env::var_os("WSL_INTEROP").is_some() { - return true; - } - fs::read_to_string("/proc/version") - .map(|version| { - let version = version.to_ascii_lowercase(); - version.contains("microsoft") || version.contains("wsl") - }) - .unwrap_or(false) -} - -fn is_windows_drive_mount(path: &Path) -> bool { - let normalized = normalize_path(path); - let mut parts = normalized.split('/'); - matches!( - (parts.next(), parts.next(), parts.next()), - (Some(""), Some("mnt"), Some(drive)) if drive.len() == 1 && drive.as_bytes()[0].is_ascii_alphabetic() - ) -} - #[cfg(test)] mod tests { use super::*; use crate::test_env::{EnvGuard, env_guard}; + use codegraph_core::wsl::is_windows_drive_mount; + use std::fs; #[test] fn watch_disabled_when_root_is_home() { @@ -667,7 +633,8 @@ mod tests { fs::write(dir.path().join(".gitignore"), "a/b/\n").unwrap(); let policy = WatchPolicy::new(dir.path()); assert!(!policy.should_watch_dir("a/b")); - assert!(!policy.should_watch_dir("x/a/b")); + // An inner `/` anchors the rule to the root, in git and in the scan. + assert!(policy.should_watch_dir("x/a/b")); assert!(!policy.should_watch_dir("a/b/c")); assert!(policy.should_watch_dir("a/bb")); assert!(policy.should_watch_dir("za/b")); @@ -896,6 +863,20 @@ mod tests { assert!(policy.should_handle_file("src/app.ts")); } + #[test] + fn jvm_packages_named_build_are_watched_like_the_scan_indexes_them() { + let dir = crate::sync::tests::TestDir::new("watch-policy-jvm-build"); + let policy = WatchPolicy::new(dir.path()); + assert!(policy.should_watch_dir("src/main/java/com/acme/build")); + assert!(policy.should_handle_file("src/main/java/com/acme/build/Builder.java")); + assert!(policy.should_handle_file("app/src/test/kotlin/build/BuildTest.kt")); + assert!(!policy.should_watch_dir("build")); + assert!(!policy.should_watch_dir("app/build")); + assert!(!policy.should_handle_file("app/build/classes/Out.java")); + assert!(!policy.should_handle_file("src/main/resources/build/Res.java")); + assert!(!policy.should_handle_file("src/main/java/build/node_modules/dep.js")); + } + #[test] fn should_handle_file_requires_known_language() { let dir = crate::sync::tests::TestDir::new("watch-policy-handle"); diff --git a/crates/codegraph-watch/src/sync.rs b/crates/codegraph-watch/src/sync.rs index 63dd177d..780b9b13 100644 --- a/crates/codegraph-watch/src/sync.rs +++ b/crates/codegraph-watch/src/sync.rs @@ -13,7 +13,8 @@ use codegraph_core::generated_header::detect_generated_file; use codegraph_core::node_id::hash_content; use codegraph_core::types::FileRecord; use codegraph_extract::{ - ExtensionOverrides, ExtractOptions, detect_language_with, extract_file_with_options, + ExtensionOverrides, ExtractOptions, SourceText, detect_language_with, + extract_file_with_options, is_source_file, read_source_file, }; use codegraph_resolve::ReferenceResolver; use codegraph_resolve::framework::FrameworkExtractionContext; @@ -101,6 +102,12 @@ pub struct SyncOutcome { /// this remains populated when another writer (for example startup catch-up) /// indexed the same event first and this sync consequently skips it unchanged. pub trigger_paths: Vec, + /// References the orphan sweep attempted, resolved and left unresolved + /// when it ran (upstream #1360), so a sync that healed an interrupted index + /// does not read as a no-op. Zero on a healthy index. + pub pending_refs_processed: usize, + pub pending_refs_resolved: usize, + pub pending_refs_unresolved: usize, } /// Read-only difference between the current project scope and the persisted @@ -148,22 +155,33 @@ pub fn pending_project_changes( .collect::>(); let on_disk_set = on_disk.iter().cloned().collect::>(); + let max_file_size = scope.options.max_file_size; let mut pending = PendingChanges::default(); for relative in on_disk { + let full = project_root.join(&relative); let Some(stored) = tracked.get(&relative) else { - pending.added.push(relative); + // An untracked video clip named `.ts` is not source sync would add. + if is_source_file(&full, &relative, max_file_size) + .with_context(|| format!("read pending source {}", full.display()))? + { + pending.added.push(relative); + } continue; }; - let full = project_root.join(&relative); let metadata = fs::metadata(&full).with_context(|| format!("stat {}", full.display()))?; if stored.size == metadata.len() as i64 && stored.modified_at == modified_millis(&metadata) { continue; } - let source = fs::read_to_string(&full) + let (_, source) = read_source_file(&full, &relative, max_file_size) .with_context(|| format!("read pending source {}", full.display()))?; - if stored.content_hash != hash_content(&source) { - pending.modified.push(relative); + match source.hash_input() { + // A tracked file that became a video clip leaves the index. + None => pending.removed.push(relative), + Some(hash_input) if stored.content_hash != hash_content(&hash_input) => { + pending.modified.push(relative); + } + Some(_) => {} } } for relative in tracked.keys() { @@ -496,12 +514,10 @@ fn sync_paths_with_store( merge_dependent_site(&mut dependent_sites, &mut dependent_fallbacks, affected); } } - let mut resolver = ReferenceResolver::new(project_root.to_string_lossy()); + let mut resolver = ReferenceResolver::new(project_root.to_string_lossy()) + .with_max_file_size(scope.options.max_file_size); { - let context = codegraph_resolve::StoreResolutionContext::new( - store, - project_root.to_string_lossy(), - ); + let context = resolver.store_context(store); resolver.initialize(&context); } // Re-run framework per-file extract for reindexed files whose framework @@ -546,7 +562,10 @@ fn sync_paths_with_store( // healthy index (marker absent) skips this entirely, so an ordinary sync is // byte-for-byte unchanged. if store.is_resolution_incomplete()? { - sweep_orphaned_refs(project_root, store)?; + let stats = sweep_orphaned_refs(project_root, store, scope.options.max_file_size)?; + outcome.pending_refs_processed = stats.total; + outcome.pending_refs_resolved = stats.resolved; + outcome.pending_refs_unresolved = stats.unresolved; } outcome.duration_ms = started.elapsed().as_millis(); @@ -687,11 +706,15 @@ fn merge_dependent_site( /// batched pass re-arms and clears the marker itself, so on success the index is /// no longer flagged partial. Framework per-file extract is re-run first so any /// framework refs an interrupted run never re-injected are present for the sweep. -fn sweep_orphaned_refs(project_root: &Path, store: &mut Store) -> Result<()> { - let mut resolver = ReferenceResolver::new(project_root.to_string_lossy()); +fn sweep_orphaned_refs( + project_root: &Path, + store: &mut Store, + max_file_size: u64, +) -> Result { + let mut resolver = + ReferenceResolver::new(project_root.to_string_lossy()).with_max_file_size(max_file_size); { - let context = - codegraph_resolve::StoreResolutionContext::new(store, project_root.to_string_lossy()); + let context = resolver.store_context(store); resolver.initialize(&context); } // Framework extraction must not run in the sweep. The marker is set only @@ -701,9 +724,9 @@ fn sweep_orphaned_refs(project_root: &Path, store: &mut Store) -> Result<()> { // duplicate rows (`unresolved_refs` has no UNIQUE constraint), which // accumulate across interrupt→heal cycles and diverge from `index --force`. // Re-resolving the existing rows alone heals the interrupted pass byte-equal. - resolver.resolve_and_persist_batched(store, ORPHAN_SWEEP_BATCH_ROWS)?; + let result = resolver.resolve_and_persist_batched(store, ORPHAN_SWEEP_BATCH_ROWS)?; resolver.run_post_extract(store)?; - Ok(()) + Ok(result.stats) } /// Batch size for the #1187 orphan sweep, matching the CLI full-index batch @@ -736,21 +759,14 @@ fn sync_one( let metadata = match (!force_absent).then(|| fs::metadata(&full)) { Some(Ok(metadata)) if metadata.is_file() => metadata, _ => { - let was_tracked = store.file_by_path(relative)?.is_some(); - if !was_tracked { - outcome.files_ignored += 1; - return Ok(false); - } - for dependent in store.reference_sites_dependent_on_file(relative)? { - merge_dependent_site(dependent_sites, dependent_fallbacks, dependent); - } - for name in node_names_in_file(store, relative)? { - changed_names.insert(name); - } - delete_unresolved_refs_by_file(store, relative)?; - store.delete_file_record(relative)?; - outcome.files_removed += 1; - return Ok(true); + return remove_tracked_file( + store, + relative, + outcome, + dependent_sites, + dependent_fallbacks, + changed_names, + ); } }; @@ -771,8 +787,21 @@ fn sync_one( return Ok(false); } - let source = fs::read_to_string(&full).with_context(|| format!("read {}", full.display()))?; - let content_hash = hash_content(&source); + let (metadata, source) = read_source_file(&full, relative, scope.options.max_file_size) + .with_context(|| format!("read {}", full.display()))?; + // A file that became a video clip named `.ts` is no longer source: it leaves + // the index like a deletion, and an untracked one is ignored (#1910). + let Some(hash_input) = source.hash_input() else { + return remove_tracked_file( + store, + relative, + outcome, + dependent_sites, + dependent_fallbacks, + changed_names, + ); + }; + let content_hash = hash_content(&hash_input); // Authoritative content gate, port of the upstream hash gate in // `upstream extraction/index.ts:1326-1337,1465-1483`. if stored.is_some_and(|file| file.content_hash == content_hash) { @@ -783,22 +812,55 @@ fn sync_one( for dependent in store.reference_sites_dependent_on_file(relative)? { merge_dependent_site(dependent_sites, dependent_fallbacks, dependent); } - reextract_into_store(project_root, store, relative, scope, changed_names)?; + reextract_into_store( + store, + relative, + scope, + changed_names, + (&metadata, &source, content_hash), + )?; outcome.files_reindexed += 1; Ok(true) } +/// Drop `relative`'s record when it is tracked, collecting what its removal +/// invalidates; count it as ignored otherwise. +fn remove_tracked_file( + store: &mut Store, + relative: &str, + outcome: &mut SyncOutcome, + dependent_sites: &mut BTreeMap>, + dependent_fallbacks: &mut BTreeSet, + changed_names: &mut HashSet, +) -> Result { + let was_tracked = store.file_by_path(relative)?.is_some(); + if !was_tracked { + outcome.files_ignored += 1; + return Ok(false); + } + for dependent in store.reference_sites_dependent_on_file(relative)? { + merge_dependent_site(dependent_sites, dependent_fallbacks, dependent); + } + for name in node_names_in_file(store, relative)? { + changed_names.insert(name); + } + delete_unresolved_refs_by_file(store, relative)?; + store.delete_file_record(relative)?; + outcome.files_removed += 1; + Ok(true) +} + +/// Re-extract `relative` from the bytes `sync_one` already read and hashed, so +/// the record, its hash and its symbols all describe one read of the file. fn reextract_into_store( - project_root: &Path, store: &mut Store, relative: &str, scope: &ProjectScope, changed_names: &mut HashSet, + (metadata, source, content_hash): (&fs::Metadata, &SourceText, String), ) -> Result<()> { - let full = project_root.join(relative); - let source = fs::read_to_string(&full).with_context(|| format!("read {}", full.display()))?; - let metadata = fs::metadata(&full).with_context(|| format!("stat {}", full.display()))?; - let result = extract_file_with_options(project_root, relative, &scope.options)?; + let result = codegraph_extract::engine::extraction_of(relative, source, &scope.options, |_| {}); + let hash_input = source.hash_input().unwrap_or_default(); let node_ids = result .nodes .iter() @@ -811,10 +873,10 @@ fn reextract_into_store( .collect::>(); let file = FileRecord { path: relative.to_string(), - content_hash: hash_content(&source), + content_hash, language: detect_language_with(relative, &scope.options.extensions), size: metadata.len() as i64, - modified_at: modified_millis(&metadata), + modified_at: modified_millis(metadata), indexed_at: now_millis(), node_count: result .nodes @@ -822,7 +884,7 @@ fn reextract_into_store( .filter(|node| node.file_path == relative) .count() as i64, errors: result.errors, - generated: detect_generated_file(relative, &source), + generated: detect_generated_file(relative, &hash_input), }; // A name's resolution outcomes (confidence, chosen target) depend on the set @@ -1515,6 +1577,16 @@ pub(crate) mod tests { // When: a bare no-change sync runs (no path is dirty on disk). let outcome = sync_changed_paths(dir.path(), &db, Vec::::new()).unwrap(); assert_eq!(outcome.files_reindexed, 0, "no file changed"); + // The sweep's work is reported, so a recovering sync is not a no-op + // (upstream #1360). + assert_eq!( + ( + outcome.pending_refs_processed, + outcome.pending_refs_resolved, + outcome.pending_refs_unresolved + ), + (1, 1, 0) + ); // Then: the orphaned ref is swept into a Calls edge and the marker clears. let store = Store::open(&db).unwrap(); @@ -1558,6 +1630,7 @@ pub(crate) mod tests { // Then: no marker was set, nothing was swept, and the retained // unresolvable rows are untouched (#1240 retry state preserved). assert_eq!(outcome.files_reindexed, 0); + assert_eq!(outcome.pending_refs_processed, 0, "no sweep ran"); assert!(!before.0, "a healthy index must not carry the marker"); let store = Store::open(&db).unwrap(); assert!(!store.is_resolution_incomplete().unwrap()); diff --git a/crates/codegraph-watch/src/watcher.rs b/crates/codegraph-watch/src/watcher.rs index 51fe0c2e..fe67c6d4 100644 --- a/crates/codegraph-watch/src/watcher.rs +++ b/crates/codegraph-watch/src/watcher.rs @@ -1,10 +1,10 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fs; use std::io; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::{self, RecvTimeoutError, Sender}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock}; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -477,24 +477,148 @@ pub fn next_backoff(prev: Duration) -> Duration { #[derive(Default)] struct DegradedState { degraded: AtomicBool, + /// Set while lock contention paused auto-sync: the watcher still collects + /// changes and retries a full reconcile, which clears the whole state. + recovering: AtomicBool, reason: Mutex>, } impl DegradedState { + /// Auto-sync stopped for good (until a restart). fn mark(&self, reason: String) { if let Ok(mut guard) = self.reason.lock() { *guard = Some(reason); } + self.recovering.store(false, Ordering::SeqCst); self.degraded.store(true, Ordering::SeqCst); } + /// Auto-sync paused by lock contention, recovering on its own. + fn mark_recovering(&self, reason: String) { + if let Ok(mut guard) = self.reason.lock() { + *guard = Some(reason); + } + self.recovering.store(true, Ordering::SeqCst); + self.degraded.store(true, Ordering::SeqCst); + } + + /// A committed full reconcile ends a recovery; a stopped watcher stays stopped. + fn recover(&self) { + if self.recovering.swap(false, Ordering::SeqCst) { + if let Ok(mut guard) = self.reason.lock() { + *guard = None; + } + self.degraded.store(false, Ordering::SeqCst); + } + } + fn is_degraded(&self) -> bool { self.degraded.load(Ordering::SeqCst) } + fn is_recovering(&self) -> bool { + self.recovering.load(Ordering::SeqCst) + } + fn reason(&self) -> Option { self.reason.lock().ok().and_then(|guard| guard.clone()) } + + fn health(&self) -> WatchHealth { + let reason = self.reason().unwrap_or_default(); + if !self.is_degraded() { + WatchHealth::Healthy + } else if self.is_recovering() { + WatchHealth::Recovering { reason } + } else { + WatchHealth::Disabled { reason } + } + } +} + +/// Auto-sync health of a project's live watcher in this process, for the hosts +/// that answer from its index (upstream #876, #1959). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WatchHealth { + /// Watching, and every change is synced or queued for a retry. + Healthy, + /// Another writer held the index past the contention budget. Changes are + /// still collected and a full reconcile is retried, but until one commits + /// the index may be stale. + Recovering { reason: String }, + /// Watching stopped (watch resources exhausted, or sync failing + /// persistently): the index is frozen until `codegraph sync` and a restart. + Disabled { reason: String }, +} + +type HealthRegistry = Mutex>>; + +fn health_registry() -> &'static HealthRegistry { + static REGISTRY: OnceLock = OnceLock::new(); + REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// The auto-sync health of the live watcher this process runs for +/// `project_root`, or `None` when it runs none. +pub fn watch_health(project_root: &Path) -> Option { + let key = project_root + .canonicalize() + .unwrap_or_else(|_| project_root.to_path_buf()); + let registry = health_registry().lock().ok()?; + registry + .get(&key) + .and_then(std::sync::Weak::upgrade) + .map(|state| state.health()) +} + +/// Test seam: publish `health` for `project_root` as a watcher there would, +/// for as long as the returned guard lives. +#[cfg(feature = "test-hooks")] +pub fn register_watch_health_for_tests( + project_root: &Path, + health: WatchHealth, +) -> WatchHealthGuard { + let key = project_root + .canonicalize() + .unwrap_or_else(|_| project_root.to_path_buf()); + let state = Arc::new(DegradedState::default()); + match health { + WatchHealth::Healthy => {} + WatchHealth::Recovering { reason } => state.mark_recovering(reason), + WatchHealth::Disabled { reason } => state.mark(reason), + } + register_health(&key, &state); + WatchHealthGuard { key, state } +} + +/// Keeps a [`register_watch_health_for_tests`] registration alive. +#[cfg(feature = "test-hooks")] +pub struct WatchHealthGuard { + key: PathBuf, + state: Arc, +} + +#[cfg(feature = "test-hooks")] +impl Drop for WatchHealthGuard { + fn drop(&mut self) { + unregister_health(&self.key, &self.state); + } +} + +fn register_health(project_root: &Path, state: &Arc) { + if let Ok(mut registry) = health_registry().lock() { + registry.insert(project_root.to_path_buf(), Arc::downgrade(state)); + } +} + +fn unregister_health(project_root: &Path, state: &Arc) { + if let Ok(mut registry) = health_registry().lock() + && registry + .get(project_root) + .is_some_and(|registered| std::ptr::eq(registered.as_ptr(), Arc::as_ptr(state))) + { + registry.remove(project_root); + } } #[derive(Clone)] @@ -528,8 +652,19 @@ pub struct WatchOptions { /// shutdown can refuse queued lease loops and interrupt a running one /// (frozen plan lines 598-601). cancel: SyncCancellation, + /// Cumulative backoff a sync spends waiting out another writer before + /// auto-sync pauses as RECOVERING. Defaults to [`MAX_BACKOFF`]. + pub lock_contention_budget: Duration, + /// How often a RECOVERING watcher retries its full reconcile. Defaults to + /// [`LOCK_RECOVERY_INTERVAL`]. + pub lock_recovery_interval: Duration, } +/// How often a watcher paused by lock contention retries its full reconcile, +/// so a long-lived writer is not polled in a loop (upstream #1959's re-arm +/// cooldown). +pub const LOCK_RECOVERY_INTERVAL: Duration = Duration::from_secs(30); + impl Default for WatchOptions { fn default() -> Self { let indexing = codegraph_core::config::IndexingConfig::default(); @@ -551,6 +686,8 @@ impl Default for WatchOptions { sync_fn: None, full_sync_fn: None, cancel: SyncCancellation::new(), + lock_contention_budget: MAX_BACKOFF, + lock_recovery_interval: LOCK_RECOVERY_INTERVAL, } } } @@ -597,6 +734,8 @@ pub struct ProjectWatcher { thread: Option>, watcher: SharedWatcher, degraded: Arc, + /// The canonical project root its health is registered under. + health_key: PathBuf, cancel: SyncCancellation, /// Set by the event-loop thread as its LAST action. Lets a caller observe /// completion without joining, so a bounded shutdown never blocks on a @@ -669,6 +808,9 @@ impl ProjectWatcher { }); let (tx, rx) = mpsc::channel(); let degraded = Arc::new(DegradedState::default()); + let health_key = project_root.clone(); + let lock_contention_budget = options.lock_contention_budget; + let lock_recovery_interval = options.lock_recovery_interval; // Capture directory identity before registering the backend. Windows emits // `RemoveKind::Any` after deletion, so this pre-event snapshot is the only // deterministic way to distinguish a known directory from an extensionless @@ -781,15 +923,19 @@ impl ProjectWatcher { degraded: loop_degraded, watcher: loop_watcher, known_dirs, + lock_contention_budget, + lock_recovery_interval, }); loop_finished.store(true, Ordering::SeqCst); }); + register_health(&health_key, °raded); Ok(Some(Self { tx, thread: Some(thread), watcher, degraded, + health_key, cancel, finished, })) @@ -847,6 +993,12 @@ impl ProjectWatcher { self.degraded.reason() } + /// Auto-sync health; [`WatchHealth::Recovering`] while lock contention + /// paused it and a full reconcile has not committed yet. + pub fn health(&self) -> WatchHealth { + self.degraded.health() + } + pub fn ingest_event_for_tests(&self, relative: impl Into) { let _ = self.tx.send(LoopMessage::Event(WatchEventBatch::paths(vec![ relative.into(), @@ -893,6 +1045,7 @@ impl ProjectWatcher { // Signal first (never join without cancelling: the event-loop thread may be // inside a bounded lease acquisition), then join. self.begin_shutdown(); + unregister_health(&self.health_key, &self.degraded); if let Some(thread) = self.thread.take() { let _ = thread.join(); } @@ -991,6 +1144,8 @@ struct EventLoopCtx { degraded: Arc, watcher: SharedWatcher, known_dirs: BTreeSet, + lock_contention_budget: Duration, + lock_recovery_interval: Duration, } fn event_loop(ctx: EventLoopCtx) { @@ -1008,6 +1163,8 @@ fn event_loop(ctx: EventLoopCtx) { degraded, watcher, mut known_dirs, + lock_contention_budget, + lock_recovery_interval, } = ctx; let mut pending = BTreeMap::::new(); let mut deadline = None::; @@ -1134,7 +1291,9 @@ fn event_loop(ctx: EventLoopCtx) { }); } } - if !pending.is_empty() { + // While lock contention paused auto-sync, the recovery retry keeps + // its own cadence; a burst of edits must not poll the other writer. + if !pending.is_empty() && !degraded.is_recovering() { // Resetting the timer on every event ports the upstream exactly-once // burst semantics (`upstream sync/watcher.ts:529-540`). deadline = Some(Instant::now() + runtime_scope.debounce); @@ -1158,18 +1317,27 @@ fn event_loop(ctx: EventLoopCtx) { } Some(LoopMessage::Stop) => break, None => { - let paths = pending.keys().cloned().collect::>(); - pending.clear(); + // Taken for the attempt; a failed or paused sync puts it back. + let mut batch = Some(std::mem::take(&mut pending)); + let paths = batch + .as_ref() + .map(|batch| batch.keys().cloned().collect::>()) + .unwrap_or_default(); deadline = None; let full_sync = std::mem::take(&mut full_sync_pending); let attempt = if full_sync { - run_full_sync_with_backoff(&full_sync_fn) + run_full_sync_with_backoff(&full_sync_fn, lock_contention_budget) } else { - run_sync_with_backoff(&sync_fn, paths.clone()) + run_sync_with_backoff(&sync_fn, paths.clone(), lock_contention_budget) }; let decision = classify_persistent_failure(&attempt, &mut consecutive_sync_errors); match attempt { SyncAttempt::Done(outcome) => { + // A committed full reconcile is what ends a recovery: the + // edits missed while another writer held the index are in. + if full_sync { + degraded.recover(); + } // Preserve the event batch independently of actual DB // mutations. Startup catch-up can win the writer lease and // index the same file first; the watcher still handled this @@ -1183,13 +1351,41 @@ fn event_loop(ctx: EventLoopCtx) { if let Some(cb) = &on_sync_error { cb(reason); } + // The failed batch is still unindexed: keep its paths + // and any full reconcile it owed (a directory removal + // adds no pending file), and retry after a backoff + // rather than drop them (upstream #1964). + pending = batch.take().unwrap_or_default(); + full_sync_pending |= full_sync; + deadline = Some( + Instant::now() + + sync_retry_delay(runtime_scope.debounce, consecutive_sync_errors), + ); } SyncAttempt::Degraded(_) => {} } - if let PersistentFailure::Degrade(reason) | PersistentFailure::Disable(reason) = - decision - { - if !degraded.is_degraded() { + if let PersistentFailure::Degrade(reason) = decision { + // Another writer held the index past the contention budget. + // Keep watching and collecting changes, report RECOVERING, and + // retry a FULL reconcile on a slow cadence until one commits + // (upstream #1959). + if !degraded.is_recovering() + && let Some(cb) = &on_sync_error + { + cb(format!( + "auto-sync paused: {reason}; retrying a full reconcile every {}s", + lock_recovery_interval.as_secs() + )); + } + degraded.mark_recovering(reason); + pending = batch.take().unwrap_or_default(); + full_sync_pending = true; + deadline = Some(Instant::now() + lock_recovery_interval); + continue; + } + if let PersistentFailure::Disable(reason) = decision { + // A recovery that keeps failing is a stop like any other. + if !degraded.is_degraded() || degraded.is_recovering() { degraded.mark(reason.clone()); if let Some(cb) = &on_degraded { cb(reason); @@ -1287,14 +1483,14 @@ fn classify_persistent_failure( /// Run `sync_fn`, retrying on write-lock contention with bounded exponential /// backoff capped at [`MAX_BACKOFF`]. Once the cumulative sleep budget is spent /// the watcher degrades; any non-contention error is surfaced as a sync error. -fn run_sync_with_backoff(sync_fn: &SyncFn, paths: Vec) -> SyncAttempt { - run_sync_with_backoff_inner(sync_fn, paths, MAX_BACKOFF, thread::sleep) +fn run_sync_with_backoff(sync_fn: &SyncFn, paths: Vec, budget: Duration) -> SyncAttempt { + run_sync_with_backoff_inner(sync_fn, paths, budget, thread::sleep) } /// Same bounded-backoff contract as [`run_sync_with_backoff`], for the /// whole-project sync a removed directory escalates to. -fn run_full_sync_with_backoff(full_sync_fn: &FullSyncFn) -> SyncAttempt { - run_with_backoff(MAX_BACKOFF, thread::sleep, || full_sync_fn()) +fn run_full_sync_with_backoff(full_sync_fn: &FullSyncFn, budget: Duration) -> SyncAttempt { + run_with_backoff(budget, thread::sleep, || full_sync_fn()) } /// Inner retry loop with an injectable budget and sleeper so the cap can be @@ -1339,11 +1535,32 @@ fn run_with_backoff( } } -/// A sync error is "lock contention" iff its chain mentions a busy/locked DB, -/// which is the only error worth retrying with backoff. +/// A sync error is "lock contention" iff another writer holds the index: its +/// chain mentions a busy/locked DB, or the bounded permanent-index-lock +/// acquisition timed out behind a long foreground `index`. Only contention is +/// retried in place with backoff; a timed-out lease counted as a sync failure +/// would drop the batch and push auto-sync toward being disabled. fn is_lock_contention(err: &anyhow::Error) -> bool { + use codegraph_store::{IndexLeaseError, StoreError}; + let lease_timed_out = err.chain().any(|cause| { + matches!( + cause.downcast_ref::(), + Some(IndexLeaseError::TimedOut { .. }) + ) || matches!( + cause.downcast_ref::(), + Some(StoreError::Lease(IndexLeaseError::TimedOut { .. })) + ) + }); let text = format!("{err:#}").to_ascii_lowercase(); - text.contains("locked") || text.contains("busy") + lease_timed_out || text.contains("locked") || text.contains("busy") +} + +/// How long the event loop waits before retrying a batch whose sync failed: +/// the debounce doubled per consecutive failure, capped at [`MAX_BACKOFF`] +/// (upstream `debounceMs * 2 ** (n - 1)`). +fn sync_retry_delay(debounce: Duration, consecutive_errors: u32) -> Duration { + let doublings = consecutive_errors.saturating_sub(1).min(16); + debounce.saturating_mul(1_u32 << doublings).min(MAX_BACKOFF) } fn snapshot(pending: &BTreeMap) -> Vec { @@ -2587,6 +2804,205 @@ mod tests { ); } + /// A failed sync leaves its batch unindexed. A directory removal owes a + /// full reconcile and adds no pending file, so dropping the batch on the + /// failure lost the reconcile and the removed files kept their nodes + /// (upstream #1964): both the paths and the owed full sync are kept and + /// retried after a backoff. + #[test] + fn a_failed_sync_keeps_its_batch_and_owed_full_reconcile_and_retries() { + let _env = crate::test_env::env_guard(); + let dir = crate::sync::tests::TestDir::new("watch-retry-owed-full"); + let full_calls = Arc::new(AtomicUsize::new(0)); + let full_counter = Arc::clone(&full_calls); + let full_sync_fn: FullSyncFn = Arc::new(move || { + if full_counter.fetch_add(1, AtomicOrdering::SeqCst) == 0 { + return Err(anyhow::anyhow!("disk full while removing nodes")); + } + Ok(SyncOutcome { + files_removed: 2, + ..Default::default() + }) + }); + let incremental_paths = Arc::new(Mutex::new(Vec::>::new())); + let recorded = Arc::clone(&incremental_paths); + let sync_fn: SyncFn = Arc::new(move |paths| { + let mut calls = recorded.lock().unwrap(); + calls.push(paths); + if calls.len() == 1 { + return Err(anyhow::anyhow!("parse error while re-extracting")); + } + Ok(SyncOutcome::default()) + }); + let (outcome_tx, outcome_rx) = mpsc::channel(); + let watcher = ProjectWatcher::start( + dir.path(), + WatchOptions { + debounce: Duration::from_millis(20), + inert_for_tests: true, + sync_fn: Some(sync_fn), + full_sync_fn: Some(full_sync_fn), + on_sync_complete: Some(Arc::new(move |outcome| { + outcome_tx.send(outcome).unwrap(); + })), + ..WatchOptions::default() + }, + ) + .unwrap() + .unwrap(); + + watcher.ingest_removed_dir_for_tests("src/feature"); + watcher.flush_for_tests(); + let outcome = outcome_rx + .recv_timeout(Duration::from_secs(5)) + .expect("the owed full reconcile is retried"); + assert_eq!(outcome.files_removed, 2); + assert_eq!(full_calls.load(AtomicOrdering::SeqCst), 2); + assert!(incremental_paths.lock().unwrap().is_empty()); + + watcher.ingest_event_for_tests("src/app.ts"); + watcher.flush_for_tests(); + outcome_rx + .recv_timeout(Duration::from_secs(5)) + .expect("the failed batch is retried"); + assert_eq!( + *incremental_paths.lock().unwrap(), + vec![ + vec!["src/app.ts".to_string()], + vec!["src/app.ts".to_string()] + ] + ); + assert!(watcher.pending_files().is_empty()); + watcher.stop(); + } + + /// Lock contention past the budget pauses auto-sync as RECOVERING rather + /// than stopping it: changes are still collected, a FULL reconcile is + /// retried on a slow cadence, and the one that commits restores health + /// (upstream #1959). + #[test] + fn lock_contention_pauses_auto_sync_until_a_full_reconcile_commits() { + let _env = crate::test_env::env_guard(); + let dir = crate::sync::tests::TestDir::new("watch-recover-lock"); + let held = Arc::new(AtomicBool::new(true)); + let full_calls = Arc::new(AtomicUsize::new(0)); + let full_held = Arc::clone(&held); + let full_counter = Arc::clone(&full_calls); + let full_sync_fn: FullSyncFn = Arc::new(move || { + full_counter.fetch_add(1, AtomicOrdering::SeqCst); + if full_held.load(AtomicOrdering::SeqCst) { + return Err(anyhow::anyhow!("database is locked")); + } + Ok(SyncOutcome { + files_reindexed: 2, + ..Default::default() + }) + }); + let held_for_sync = Arc::clone(&held); + let sync_fn: SyncFn = Arc::new(move |_paths| { + if held_for_sync.load(AtomicOrdering::SeqCst) { + return Err(anyhow::anyhow!("database is locked")); + } + Ok(SyncOutcome::default()) + }); + let notices = Arc::new(Mutex::new(Vec::::new())); + let recorded = Arc::clone(¬ices); + let stops = Arc::new(AtomicUsize::new(0)); + let stop_counter = Arc::clone(&stops); + let (outcome_tx, outcome_rx) = mpsc::channel(); + let watcher = ProjectWatcher::start( + dir.path(), + WatchOptions { + debounce: Duration::from_millis(20), + inert_for_tests: true, + sync_fn: Some(sync_fn), + full_sync_fn: Some(full_sync_fn), + lock_contention_budget: Duration::from_millis(1), + lock_recovery_interval: Duration::from_millis(50), + on_sync_error: Some(Arc::new(move |msg| recorded.lock().unwrap().push(msg))), + on_degraded: Some(Arc::new(move |_| { + stop_counter.fetch_add(1, AtomicOrdering::SeqCst); + })), + on_sync_complete: Some(Arc::new(move |outcome| { + outcome_tx.send(outcome).unwrap(); + })), + ..WatchOptions::default() + }, + ) + .unwrap() + .unwrap(); + assert_eq!(watch_health(dir.path()), Some(WatchHealth::Healthy)); + + watcher.ingest_event_for_tests("src/app.ts"); + watcher.flush_for_tests(); + let mut recovering = false; + for _ in 0..80 { + if matches!(watcher.health(), WatchHealth::Recovering { .. }) { + recovering = true; + break; + } + std::thread::sleep(Duration::from_millis(25)); + } + assert!(recovering, "contention must pause auto-sync as recovering"); + assert!(matches!( + watch_health(dir.path()), + Some(WatchHealth::Recovering { .. }) + )); + assert!(watcher.is_degraded()); + assert_eq!( + stops.load(AtomicOrdering::SeqCst), + 0, + "recovering is not a stop" + ); + assert!( + notices.lock().unwrap()[0].contains("auto-sync paused"), + "{:?}", + notices.lock().unwrap() + ); + watcher.ingest_event_for_tests("src/other.ts"); + + held.store(false, AtomicOrdering::SeqCst); + let outcome = outcome_rx + .recv_timeout(Duration::from_secs(5)) + .expect("the full reconcile commits once the writer is gone"); + assert_eq!(outcome.files_reindexed, 2); + assert_eq!(watcher.health(), WatchHealth::Healthy); + assert_eq!(watch_health(dir.path()), Some(WatchHealth::Healthy)); + assert!( + full_calls.load(AtomicOrdering::SeqCst) >= 1, + "the recovery ran the full reconcile" + ); + assert!(watcher.pending_files().is_empty()); + watcher.stop(); + assert_eq!( + watch_health(dir.path()), + None, + "a stopped watcher is unregistered" + ); + } + + /// A long foreground `index` holds the permanent index lock past a sync's + /// bounded acquisition: that is contention to back off from, not a sync + /// failure counted toward disabling auto-sync. + #[test] + fn an_index_lease_timeout_is_lock_contention() { + let timed_out = codegraph_store::IndexLeaseError::TimedOut { + path: std::path::PathBuf::from("/p/.codegraph/codegraph.lock"), + }; + assert!(is_lock_contention(&anyhow::Error::new( + codegraph_store::StoreError::Lease(timed_out) + ))); + assert!(is_lock_contention( + &anyhow::Error::new(codegraph_store::IndexLeaseError::TimedOut { + path: std::path::PathBuf::from("/p/.codegraph/codegraph.lock"), + }) + .context("opening the sync writer") + )); + assert!(!is_lock_contention(&anyhow::anyhow!( + "schema version mismatch" + ))); + } + #[test] fn event_loop_disables_auto_sync_after_persistent_errors() { let _env = crate::test_env::env_guard(); diff --git a/crates/codegraph-watch/tests/include_scan_watch_parity.rs b/crates/codegraph-watch/tests/include_scan_watch_parity.rs index 36093260..81933117 100644 --- a/crates/codegraph-watch/tests/include_scan_watch_parity.rs +++ b/crates/codegraph-watch/tests/include_scan_watch_parity.rs @@ -531,3 +531,65 @@ fn config_set_order_decides_which_negation_wins() { watcher either" ); } + +/// The root `.gitignore` is read by ONE git-rules matcher on both sides, so the +/// watcher handles exactly the files the scan indexes and watches every +/// directory above them: anchored and any-depth rules, globs, `**`, no +/// re-inclusion below an ignored directory, and JVM packages named `build` +/// under bare, Android and Spring Initializr rules (#1642). +#[test] +fn scan_and_watch_agree_on_root_gitignore_rules() { + let files = [ + "src/app.ts", + "artifacts/a.ts", + "src/artifacts/b.ts", + "src/types.gen.ts", + "docs/guide/draft/c.ts", + "docs/guide/final/d.ts", + "src/scratch/e.ts", + "logs/keep.ts", + "src/main/java/com/acme/build/Hidden.java", + "src/main/java/build/nested/build/Example.java", + "app/build/generated/Out.java", + ]; + for gitignore in [ + "/artifacts/\n*.gen.ts\ndocs/**/draft/\nscratch/\nlogs/\n!logs/keep.ts\n", + "build/\n", + "/build\n", + "build/\n!**/src/main/**/build/\n!**/src/test/**/build/\n", + ] { + let project = unique_project("gitignore_rules"); + touch(&project, ".gitignore", gitignore); + for file in files { + touch(&project, file, "x"); + } + let options = ExtractOptions::default(); + let scanned = scan_project(&project, &options).expect("scan"); + let policy = WatchPolicy::with_config( + &project, + &options.ignore_dirs, + &options.ignore_paths, + &[], + &[], + ); + for file in files { + let in_scan = scanned.iter().any(|f| f == file); + assert_eq!( + in_scan, + policy.should_handle_file(file), + "scan⇔watch parity broken for .gitignore={gitignore:?} file={file}" + ); + if in_scan { + let segments = file.split('/').collect::>(); + for depth in 1..segments.len() { + let dir = segments[..depth].join("/"); + assert!( + policy.should_watch_dir(&dir), + ".gitignore={gitignore:?}: {dir} holds indexed {file} but is not watched" + ); + } + } + } + fs::remove_dir_all(&project).ok(); + } +} diff --git a/docs/cli.md b/docs/cli.md index b2958b9e..393fceaf 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1028,13 +1028,23 @@ daemon mode when several agents or editor windows need the same live index. The daemon watches the project for file changes and re-indexes automatically. Changes are debounced before the re-index triggers. On WSL2, watching files under `/mnt/` is automatically disabled because recursive `fs.watch` is too slow on -those paths; the reason is surfaced in the log. +those paths; the reason is surfaced in the log. On such a Windows drive WSL also +keeps its own index: SQLite's locking does not hold across the Windows/WSL +bridge, so with `CODEGRAPH_DIR` unset an existing `.codegraph-wsl/` is used, then +an existing `.codegraph/codegraph.db` is kept, and otherwise a fresh index goes +in `.codegraph-wsl/` (`init` says so). A disk I/O error on a shared +`.codegraph/` index explains how to give WSL its own. The watcher registers per-directory watches only on non-ignored directories, pruning `node_modules`, `.venv`, `__pycache__`, `target`, `dist`, `.godot`, -`.cache`, `.git`, `.codegraph`, and everything else in the -default ignore set, plus any paths matched by the root `.gitignore`. This pruning applies at any -nesting depth, so an `node_modules` buried several levels deep is never walked. +`.cache`, `.git`, `.codegraph`, and everything else in the default ignore set at +any nesting depth, so a `node_modules` buried several levels deep is never walked. +A `build` directory that is a Java, Kotlin or Scala package under a source root +(`src//{java,kotlin,scala}/…/build`) is source rather than build +output, so it stays indexed and watched. The root `.gitignore` prunes the index +and the watcher alike, with git's own rules: a slash-less rule applies at any +depth, a leading or inner `/` anchors it to the project root, `*` and `**` glob, +and `!` re-includes a path unless a directory above it is ignored. This keeps the total watch count well inside the OS inotify limit on large trees and makes daemon startup fast. A newly-created non-ignored directory is picked up automatically on its create event — no restart required. @@ -1073,19 +1083,19 @@ Three escape hatches: ### Environment variable reference -| Variable | Default | Clamp range | Meaning | -| ---------------------------------- | ------------ | ------------------- | ------------------------------------------------------------------------------------------------------------------ | -| `CODEGRAPH_NO_DAEMON` | — | — | Force foreground Direct mode; one indexed-project writer only, enforced by `writer.pid` | -| `CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS` | `300000` | 1000–3600000 | Exit after this long with no connected clients | -| `CODEGRAPH_DAEMON_MAX_IDLE_MS` | `1800000` | 1000–3600000 | Hard cap on total daemon lifetime when idle | -| `CODEGRAPH_DAEMON_CLIENT_SWEEP_MS` | `30000` | 50–600000 | How often the daemon sweeps for dead clients | -| `CODEGRAPH_WATCH_DEBOUNCE_MS` | `2000` | 100–60000 | File-change debounce window before a re-index triggers | -| `CODEGRAPH_NO_WATCH` | — | — | Disable the live file watcher (equivalent to `serve --no-watch`) | -| `CODEGRAPH_FORCE_WATCH` | — | — | Override WSL2 `/mnt/` auto-disable; does not override `NO_WATCH` | -| `CODEGRAPH_NO_WAL_DEFER` | — | `1` enables opt-out | Keep SQLite's default WAL autocheckpoint interval during bulk indexing | -| `CODEGRAPH_WAL_VALVE_MB` | `256` | >0; invalid→default | Shared MB threshold for the active WAL valve, resetting `journal_size_limit`, and `status` WAL warning | -| `CODEGRAPH_MCP_REGISTRY_DIR` | — | — | Override the stdio MCP registry directory read by `mcp list` | -| `CODEGRAPH_DIR` | `.codegraph` | — | Select one non-empty project-local directory name; absolute paths, separators, `.`, `..`, and aliases are rejected | +| Variable | Default | Clamp range | Meaning | +| ---------------------------------- | ------------ | ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `CODEGRAPH_NO_DAEMON` | — | — | Force foreground Direct mode; one indexed-project writer only, enforced by `writer.pid` | +| `CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS` | `300000` | 1000–3600000 | Exit after this long with no connected clients | +| `CODEGRAPH_DAEMON_MAX_IDLE_MS` | `1800000` | 1000–3600000 | Hard cap on total daemon lifetime when idle | +| `CODEGRAPH_DAEMON_CLIENT_SWEEP_MS` | `30000` | 50–600000 | How often the daemon sweeps for dead clients | +| `CODEGRAPH_WATCH_DEBOUNCE_MS` | `2000` | 100–60000 | File-change debounce window before a re-index triggers | +| `CODEGRAPH_NO_WATCH` | — | — | Disable the live file watcher (equivalent to `serve --no-watch`) | +| `CODEGRAPH_FORCE_WATCH` | — | — | Override WSL2 `/mnt/` auto-disable; does not override `NO_WATCH` | +| `CODEGRAPH_NO_WAL_DEFER` | — | `1` enables opt-out | Keep SQLite's default WAL autocheckpoint interval during bulk indexing | +| `CODEGRAPH_WAL_VALVE_MB` | `256` | >0; invalid→default | Shared MB threshold for the active WAL valve, resetting `journal_size_limit`, and `status` WAL warning | +| `CODEGRAPH_MCP_REGISTRY_DIR` | — | — | Override the stdio MCP registry directory read by `mcp list` | +| `CODEGRAPH_DIR` | `.codegraph` | — | Select one non-empty project-local directory name; absolute paths, separators, `.`, `..`, and aliases are rejected. Unset on a WSL Windows drive (`/mnt//`), the default is `.codegraph-wsl` unless `.codegraph/codegraph.db` already exists, so WSL never shares Windows CodeGraph's index | Timeout/debounce values outside their clamp range are silently clamped to the nearest bound. `CODEGRAPH_WAL_VALVE_MB` instead falls back to `256` when it is @@ -1179,6 +1189,15 @@ synthetic file node, `init`/`index`/`sync` still succeed but print and persist `parse produced no symbols (tree has errors)`. Files with useful surviving symbols stay quiet even when the grammar tree contains recoverable error nodes. +A file over `indexing.max_file_size` (1 MiB by default) is recorded without +being read: no symbols, a `File exceeds max size` error, and a size stamp in +place of its content hash, so a same-size rewrite is not a change while crossing +the limit in either direction is. A `.ts` file whose head is an MPEG transport +stream is video rather than TypeScript: it is neither indexed nor counted, and a +tracked file that turns into one leaves the index. Bytes that are not valid +UTF-8 are decoded with U+FFFD replacements, so a Latin-1 or binary source file +is indexed like any other instead of failing the run. + ### Index diagnostics `init`, `index`, and `sync` share `--debug` and `--debug-log `. diff --git a/docs/mcp.md b/docs/mcp.md index 70476e31..9683731a 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -696,3 +696,23 @@ gets **no** banner, which is what makes "trust everything not listed" a real guarantee rather than an assumption — the agent instructions had described this banner before anything produced it. Run `codegraph sync`, or wait for the watcher, if you see it on a hot codebase; ordinary drift does not require a full rebuild. + +### Auto-sync health + +When the server runs the project's live watcher in-process, every answer also +reports its health, since edits the index never heard about cannot reach the +per-file banner above: + +- **RECOVERING** — another process held the index past the sync's contention + budget (a long foreground `index`). Watching continues and changes are still + collected; a full reconcile is retried every 30 s, and until one commits each + response starts with `⚠️ CodeGraph auto-sync is RECOVERING …`. +- **DISABLED** — watching stopped (watch resources exhausted, or syncs failing + persistently). Responses start with `⚠️ CodeGraph auto-sync is DISABLED …` and + the reason; run `codegraph sync` and restart the server. + +In either state `codegraph_search`, `codegraph_callers`, `codegraph_callees` +and `codegraph_impact` check every indexed file they would name against disk, +and when one changed or disappeared since its last sync they name those files +instead of answering from the frozen graph. `codegraph_status` reports the +state as `Auto-sync:`. diff --git a/docs/upstream-sync/V1_6_1_AUDIT_2026-09-30.md b/docs/upstream-sync/V1_6_1_AUDIT_2026-09-30.md index fa994fb8..798d9318 100644 --- a/docs/upstream-sync/V1_6_1_AUDIT_2026-09-30.md +++ b/docs/upstream-sync/V1_6_1_AUDIT_2026-09-30.md @@ -20,38 +20,38 @@ documented divergence). ## Resolution and extraction -| Issue / PR | Upstream | Behavior | Disposition | State | -| --------------------------------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| #1862 (a) | `4297b8e` | an identifier-rooted TS/JS chain of three or more segments keeps its qualified call site; only a framework resolver may bind it | CORRECT | landed in #277 | -| #1862 (b) | `4297b8e` | typed method lookup matches within a language family (`.tsx` reaches `.ts`, `.js` reaches `.jsx`) | PORT | landed in #275 | -| #1862 (c), #647 | `4297b8e`, `68eaf0d` | Zustand `get()`, `useStore.getState()`, destructured and selector bindings resolve inside one identified store; requires the store-initializer action extraction the port never had | PORT | landed in #277 | -| #1987 | `b65e05b` | `this.#field.m()` resolves on the private field's type; JS built-in method names need receiver evidence | PORT | landed in #277 | -| #1986 | `9649ce1` | code references crossing code families need bridge evidence; TS/JS receiver wrappers peel and untyped expression receivers emit nothing | PORT | landed in #277 | -| #1857 | `53fe53d` | a receiver-less Go call never binds a method | CORRECT | landed in #275 | -| #1882 | `8f80819` | Rust `self.m()` with same-named owners in several modules pins the caller's file | CORRECT | landed in #275 | -| #1885 / #1840 | `d0996a2` | awaited receiver inference: block scope, reassignment, call tail, imported or reachable factory, `Promise` | CORRECT | landed in #275 | -| #1824 | `e27d6da` | a Scala object is never a supertype target but owns methods and supertypes | CORRECT | landed in #275 | -| #2040 | `9be3f1a` | an unknown member of a named Python import stays unresolved | PORT | landed in #277 | -| #2055 | `4960720` | an inheritance ref that lands on a TS value moves to the same-file same-named type | PORT | landed in #277 | -| #1932 | `17d09bf` | object-literal members that alias a function follow the binding lexically (last own property, spread/computed invalidation, shadowing) | CORRECT | landed in #277 | -| #1747 | `9fb2c25` | a curried-wrapper function that is an object member is named by its key | CORRECT | landed in #277 | -| #1905 | `bd993b1` | Python module/class/function body docstrings (PEP 257), joined after a preceding comment | PORT | landed in #277 | -| #1373 | `9b5c8e4` | single-argument C/C++ function macros recover the defined function name with local `#define` evidence | PORT | landed in #277 | -| #1820 | `5eaa6fe` | Python attribute and Go selector method values keep their receiver and resolve through type/import scope | CORRECT | receiver-qualified JS/TS/Python values landed in #275, the Python named-import module fallback in #277, Go selector values and Python receiver scoping in #278 | -| #1838 / #1839 / #2069 / #2070 | `fa25883`, `15a5d1d` | translation-unit-ordered macro visibility; `extern`, array and `initializer_list` constructor rules; macro-name calls bind only functions | CORRECT | constructor/macro basics landed in #275; translation-unit macro visibility, the newer constructor rules, #2069 and #2070 planned | -| #1974 | `52df4d2` | depth-limited traversal re-expands a node reached again at a shallower depth | PORT | landed in #278 | -| #1349 | `86708bf` | a later same-line, same-name declaration gets a column-suffixed node id | PORT, touches the node-id invariant | landed in #278 (approved by the owner on 2026-10-01) | -| #2072 | `fef3776` | performance: memoised gates and caches, graph output unchanged | partial PORT | `this.` declaration memo landed in #277; per-file source facts in #275; the `file_exists` miss cache is planned | -| #1719 | (09-15 row) | sealed-module markers read after blanking | CORRECT | landed in #275 | -| #1642 | `4ebd0d6` | `src/*/{java,kotlin,scala}/**/build/` is source, not build output | PORT | planned | -| #1553 | `276df3a` | resolver file reads stat first and skip non-regular or oversized files | PORT | planned | -| #1910 | `04ec5f5` | bounded extraction reads and MPEG-TS sniffing for `.ts` | PORT | planned | -| #1209 / #1222 | `9e4b8e7` | MyBatis closing tags tolerate whitespace | PORT (refid half ALREADY-HAVE) | planned | -| #1906 | `c505d0d` | Liquid `{% liquid %}` blocks | PORT | planned | -| #1360 | `6184610` | sync reports recovered orphan references | PORT | planned | -| #1829 / #1878 | `4871114` | git-stamped fast path for pending status | KEEP-RUST (full inventory); fast path DEFER pending measurement | — | -| #1833 / #1849, #1830, #1784, #1823, #1519, #1350, #1348, #1359, #1361 | various | already implemented by the port | ALREADY-HAVE | — | -| #1827, #1773, #1864, #1973 / #1543, #1355 / #1988 / #2038 | various | Spring DI heuristics, Node `indexAll` spin, dominant-file SQL, dead-code, callback synthesis storage | N/A | — | +| Issue / PR | Upstream | Behavior | Disposition | State | +| --------------------------------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| #1862 (a) | `4297b8e` | an identifier-rooted TS/JS chain of three or more segments keeps its qualified call site; only a framework resolver may bind it | CORRECT | landed in #277 | +| #1862 (b) | `4297b8e` | typed method lookup matches within a language family (`.tsx` reaches `.ts`, `.js` reaches `.jsx`) | PORT | landed in #275 | +| #1862 (c), #647 | `4297b8e`, `68eaf0d` | Zustand `get()`, `useStore.getState()`, destructured and selector bindings resolve inside one identified store; requires the store-initializer action extraction the port never had | PORT | landed in #277 | +| #1987 | `b65e05b` | `this.#field.m()` resolves on the private field's type; JS built-in method names need receiver evidence | PORT | landed in #277 | +| #1986 | `9649ce1` | code references crossing code families need bridge evidence; TS/JS receiver wrappers peel and untyped expression receivers emit nothing | PORT | landed in #277 | +| #1857 | `53fe53d` | a receiver-less Go call never binds a method | CORRECT | landed in #275 | +| #1882 | `8f80819` | Rust `self.m()` with same-named owners in several modules pins the caller's file | CORRECT | landed in #275 | +| #1885 / #1840 | `d0996a2` | awaited receiver inference: block scope, reassignment, call tail, imported or reachable factory, `Promise` | CORRECT | landed in #275 | +| #1824 | `e27d6da` | a Scala object is never a supertype target but owns methods and supertypes | CORRECT | landed in #275 | +| #2040 | `9be3f1a` | an unknown member of a named Python import stays unresolved | PORT | landed in #277 | +| #2055 | `4960720` | an inheritance ref that lands on a TS value moves to the same-file same-named type | PORT | landed in #277 | +| #1932 | `17d09bf` | object-literal members that alias a function follow the binding lexically (last own property, spread/computed invalidation, shadowing) | CORRECT | landed in #277 | +| #1747 | `9fb2c25` | a curried-wrapper function that is an object member is named by its key | CORRECT | landed in #277 | +| #1905 | `bd993b1` | Python module/class/function body docstrings (PEP 257), joined after a preceding comment | PORT | landed in #277 | +| #1373 | `9b5c8e4` | single-argument C/C++ function macros recover the defined function name with local `#define` evidence | PORT | landed in #277 | +| #1820 | `5eaa6fe` | Python attribute and Go selector method values keep their receiver and resolve through type/import scope | CORRECT | receiver-qualified JS/TS/Python values landed in #275, the Python named-import module fallback in #277, Go selector values and Python receiver scoping in #278 | +| #1838 / #1839 / #2069 / #2070 | `fa25883`, `15a5d1d` | translation-unit-ordered macro visibility; `extern`, array and `initializer_list` constructor rules; macro-name calls bind only functions | CORRECT | constructor/macro basics landed in #275; translation-unit macro visibility, the newer constructor rules, #2069 and #2070 planned | +| #1974 | `52df4d2` | depth-limited traversal re-expands a node reached again at a shallower depth | PORT | landed in #278 | +| #1349 | `86708bf` | a later same-line, same-name declaration gets a column-suffixed node id | PORT, touches the node-id invariant | landed in #278 (approved by the owner on 2026-10-01) | +| #2072 | `fef3776` | performance: memoised gates and caches, graph output unchanged | partial PORT | `this.` declaration memo landed in #277; per-file source facts in #275; the `file_exists` memo in #279 | +| #1719 | (09-15 row) | sealed-module markers read after blanking | CORRECT | landed in #275 | +| #1642 | `4ebd0d6` | `src/*/{java,kotlin,scala}/**/build/` is source, not build output | PORT | landed in #279, with the root `.gitignore` read by git's rules in scan and watch (both matchers diverged from git before) | +| #1553 | `276df3a` | resolver file reads stat first and skip non-regular or oversized files | PORT | landed in #279 | +| #1910 | `04ec5f5` | bounded extraction reads and MPEG-TS sniffing for `.ts` | PORT | landed in #279, with lossy UTF-8 decoding: a non-UTF-8 source file failed the whole index before (a Rust-only defect) | +| #1209 / #1222 | `9e4b8e7` | MyBatis closing tags tolerate whitespace | PORT (refid half ALREADY-HAVE), CORRECT for #1182 | landed in #279 with the rest of #1182 (comments, quote styles, namespace-less ``, iBatis verbs, `databaseId`); KEEP-RUST: a self-closing `` is never a root | +| #1906 | `c505d0d` | Liquid `{% liquid %}` blocks | PORT | landed in #279 | +| #1360 | `6184610` | sync reports recovered orphan references | PORT | landed in #279 | +| #1829 / #1878 | `4871114` | git-stamped fast path for pending status | KEEP-RUST (full inventory); fast path DEFER pending measurement | — | +| #1833 / #1849, #1830, #1784, #1823, #1519, #1350, #1348, #1359, #1361 | various | already implemented by the port | ALREADY-HAVE | — | +| #1827, #1773, #1864, #1973 / #1543, #1355 / #1988 / #2038 | various | Spring DI heuristics, Node `indexAll` spin, dominant-file SQL, dead-code, callback synthesis storage | N/A | — | Findings outside the upstream range, recorded so the next audit starts from them: callers/callees never followed `instantiates` edges (upstream #774/#804, @@ -79,28 +79,28 @@ KEEP-RUST divergences taken in this PR: ## MCP, CLI, installer, lifecycle and explore -| Issue / PR | Upstream | Behavior | Disposition | State | -| ------------------------------------------------------------------------------------ | ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | -| #1832 | `cba44c9` | only a prompt that is entirely a task-notification envelope is skipped | CORRECT | landed in #275 | -| #1967 | `94ae04d` | dynamic-import boundaries: template or concatenated specifiers are runtime imports | PORT | planned | -| #1964 | `a9c8808` | a failed sync keeps the pending paths and owed full scan and retries | PORT | planned | -| #1959 | `2268a09` | restart a degraded watcher, RECOVERING/DISABLED banners, refuse drifted answers while degraded | PORT | planned | -| #1654 | `4fd3e6d` | `como`/`wie` leave the unconditional structural keyword list | PORT | planned | -| #1870 | `98718cf` | installer `detect()` is read-only; backups only when a config is rewritten | PORT | planned | -| #1839 (CLI) | `fa25883` | callers/callees JSON carries each node's `relationships` | PORT | landed in #278, with callers/callees following `instantiates` edges (upstream #774/#804, never ported before) | -| #1895 | `ee97a57` | MCP root discovery ignores a `codegraph.db` without a schema | PORT (CLI ALREADY-HAVE) | planned | -| #995 | `c13bd08` | WSL `/mnt/` defaults to `.codegraph-wsl` and maps SQLITE_IOERR to guidance | PORT | planned | -| #2053 / #1773 | `0a9f3de`, `c405ac5` | bounded retry when Windows sharing violations block `daemon.pid` replacement | PORT | planned | -| #1294 | `d3671fa` | `install.sh` under MINGW/MSYS/CYGWIN points at the PowerShell installer | PORT | planned | -| #1904, PR #2062, PR #2063, PR #2068, PR #2071, PR #2077 | `9ad6ee9`, `adad840`, `47840a6`, `724b5de`, `290e03f`, `003305f` | explore: empty-result lexical diagnostics, cluster budget order, line anchors as exact targets, single-file budget, same-basename pins, honest completion notes | PORT | planned | -| #1834, #1831, #1830, #1968, #1372, #1361, #1835, #1963 / #1356, #1325, #1902 | various | already implemented by the port | ALREADY-HAVE | — | -| #770 | `c897daf` | watch symlinked directories | DEFER until the scan's symlink policy is settled (the scan does not follow symlinked directories) | — | -| telemetry, npm/fnm, Node timers, `FILE_NOTIFY_CHANGE_LAST_ACCESS`, test-only commits | various | no Rust analogue | N/A | — | +| Issue / PR | Upstream | Behavior | Disposition | State | +| ------------------------------------------------------------------------------------ | ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| #1832 | `cba44c9` | only a prompt that is entirely a task-notification envelope is skipped | CORRECT | landed in #275 | +| #1967 | `94ae04d` | dynamic-import boundaries: template or concatenated specifiers are runtime imports | PORT | landed in #279 | +| #1964 | `a9c8808` | a failed sync keeps the pending paths and owed full scan and retries | PORT | landed in #279, with the index-lease timeout classified as contention | +| #1959 | `2268a09` | restart a degraded watcher, RECOVERING/DISABLED banners, refuse drifted answers while degraded | PORT | landed in #279 as an in-loop recovery (the watcher keeps collecting events and retries a full reconcile every 30 s), with the #876 banner never ported before | +| #1654 | `4fd3e6d` | `como`/`wie` leave the unconditional structural keyword list | PORT | landed in #279 | +| #1870 | `98718cf` | installer `detect()` is read-only; backups only when a config is rewritten | PORT | landed in #279 | +| #1839 (CLI) | `fa25883` | callers/callees JSON carries each node's `relationships` | PORT | landed in #278, with callers/callees following `instantiates` edges (upstream #774/#804, never ported before) | +| #1895 | `ee97a57` | MCP root discovery ignores a `codegraph.db` without a schema | PORT (CLI ALREADY-HAVE) | landed in #279 | +| #995 | `c13bd08` | WSL `/mnt/` defaults to `.codegraph-wsl` and maps SQLITE_IOERR to guidance | PORT | landed in #279 | +| #2053 / #1773 | `0a9f3de`, `c405ac5` | bounded retry when Windows sharing violations block `daemon.pid` replacement | PORT | landed in #279 | +| #1294 | `d3671fa` | `install.sh` under MINGW/MSYS/CYGWIN points at the PowerShell installer | PORT | landed in #279 | +| #1904, PR #2062, PR #2063, PR #2068, PR #2071, PR #2077 | `9ad6ee9`, `adad840`, `47840a6`, `724b5de`, `290e03f`, `003305f` | explore: empty-result lexical diagnostics, cluster budget order, line anchors as exact targets, single-file budget, same-basename pins, honest completion notes | PORT | planned | +| #1834, #1831, #1830, #1968, #1372, #1361, #1835, #1963 / #1356, #1325, #1902 | various | already implemented by the port | ALREADY-HAVE | — | +| #770 | `c897daf` | watch symlinked directories | DEFER until the scan's symlink policy is settled (the scan does not follow symlinked directories) | — | +| telemetry, npm/fnm, Node timers, `FILE_NOTIFY_CHANGE_LAST_ACCESS`, test-only commits | various | no Rust analogue | N/A | — | A Rust-only defect found by this pass: the watcher treats only `locked`/`busy` errors as lock contention, but an index-lease timeout reports `timed out acquiring permanent index lock`, so a long foreground `index` makes the watcher -drop the batch and count a failure. It is planned with #1964. +drop the batch and count a failure. It landed with #1964 in #279. ## UI family (re-triage) diff --git a/scripts/install.sh b/scripts/install.sh index 3d76bf12..7cc7adae 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -35,6 +35,13 @@ fi case "$(uname -s)" in Linux) os_part="unknown-linux-musl" ;; Darwin) os_part="apple-darwin" ;; +MINGW* | MSYS* | CYGWIN*) + # A Windows POSIX shell (Git Bash, MSYS2, Cygwin): the Windows build is + # installed by the PowerShell installer. + info "codegraph: on Windows, open PowerShell and run:" + info " irm https://raw.githubusercontent.com/${REPO}/main/scripts/install.ps1 | iex" + exit 1 + ;; *) err "unsupported OS: $(uname -s) (supported: Linux, Darwin)" ;; esac case "$(uname -m)" in diff --git a/scripts/tests/install-windows-shell.test.sh b/scripts/tests/install-windows-shell.test.sh new file mode 100644 index 00000000..e86ba9f9 --- /dev/null +++ b/scripts/tests/install-windows-shell.test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# install.sh under a Windows POSIX shell (Git Bash, MSYS2, Cygwin) points at +# the PowerShell installer and exits before downloading anything (upstream +# #1294); Linux and Darwin are unchanged. +set -euo pipefail +root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +installer="$root/scripts/install.sh" +work=$(mktemp -d); trap 'rm -rf "$work"' EXIT +pass=0; fail=0 + +bin="$work/bin"; mkdir -p "$bin" +for tool in sh tar gzip sed head mktemp cut awk tr rm mkdir mv chmod cat sha256sum; do + src=$(command -v "$tool") && ln -s "$src" "$bin/$tool" +done +cat >"$bin/uname" <<'SH' +#!/bin/sh +case "$1" in -s) echo "$CG_UNAME_S";; -m) echo x86_64;; *) echo "$CG_UNAME_S";; esac +SH +cat >"$bin/curl" <<'SH' +#!/bin/sh +echo "$*" >>"$CG_DOWNLOADS" +exit 22 +SH +chmod +x "$bin/uname" "$bin/curl" + +run() { + rm -f "$work/downloads" + set +e + env -i PATH="$bin" HOME="$work/home" CG_UNAME_S="$1" CG_DOWNLOADS="$work/downloads" \ + /bin/sh "$installer" >"$work/out" 2>"$work/err" + rc=$?; set -e +} +check() { + if [ "$2" -eq 1 ]; then echo "PASS: $1"; pass=$((pass+1)); + else echo "FAIL: $1 (exit $rc)" >&2; cat "$work/err" >&2; fail=$((fail+1)); fi +} + +for shell_os in MINGW64_NT-10.0-26100 MSYS_NT-10.0-26100 CYGWIN_NT-10.0-26100; do + run "$shell_os" + ok=1 + [ "$rc" -ne 0 ] || ok=0 + grep -Fq 'irm https://raw.githubusercontent.com/sunerpy/codegraph-rust/main/scripts/install.ps1 | iex' "$work/err" || ok=0 + [ ! -e "$work/downloads" ] || ok=0 + check "$shell_os" "$ok" +done + +# Linux still gets past the OS check to the (failing, fake) download. +run Linux +ok=1 +[ -e "$work/downloads" ] || ok=0 +! grep -q 'install.ps1' "$work/err" || ok=0 +check Linux "$ok" + +echo "windows-shell harness: $pass passed, $fail failed" +[ "$fail" -eq 0 ]