From 0be6e87c08a1ac5da464df88b8d2485df0ab15c5 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 09:11:10 +0800 Subject: [PATCH 01/19] fix(scan): keep JVM packages named build under source roots The default build prune hid Java, Kotlin and Scala packages named build (src//{java,kotlin,scala}/**/build). Exempt exactly those directories in the scan and the watcher's structural tier; module build output and the other defaults inside them stay pruned (upstream #1642). --- crates/codegraph-core/src/config.rs | 41 ++++++++++++++++++++ crates/codegraph-extract/src/engine.rs | 53 +++++++++++++++++++++++++- crates/codegraph-watch/src/policy.rs | 43 +++++++++++++++++++-- 3 files changed, 132 insertions(+), 5 deletions(-) diff --git a/crates/codegraph-core/src/config.rs b/crates/codegraph-core/src/config.rs index f90eb566..2565c75b 100644 --- a/crates/codegraph-core/src/config.rs +++ b/crates/codegraph-core/src/config.rs @@ -99,6 +99,24 @@ fn default_max_file_size() -> u64 { 1024 * 1024 } +/// Whether `relative_dir` — a project-relative, `/`-separated directory path — +/// is a `build` directory that names a JVM package under a conventional source +/// root (`**/src//{java,kotlin,scala}/**/build`) rather than build +/// output. The default `build` prune exempts exactly that directory, not its +/// subtree: the other defaults still apply inside it (upstream #1642). +pub fn is_jvm_source_build_dir(relative_dir: &str) -> bool { + let segments = relative_dir.split('/').collect::>(); + let Some(last) = segments.len().checked_sub(1) else { + return false; + }; + segments[last] == "build" + && (0..last).any(|index| { + segments[index] == "src" + && index + 2 < last + && matches!(segments[index + 2], "java" | "kotlin" | "scala") + }) +} + fn default_ignore_dirs() -> Vec { // upstream extraction/index.ts:117-145 // Directory names that are dependency, build, cache, or tooling output across the @@ -377,6 +395,29 @@ mod tests { } } + #[test] + fn jvm_source_build_dirs_are_package_segments_not_output() { + for kept in [ + "src/main/java/build", + "src/main/java/com/acme/build", + "module/src/test/kotlin/build", + "src/androidTest/scala/x/build", + ] { + assert!(is_jvm_source_build_dir(kept), "{kept}"); + } + for pruned in [ + "build", + "app/build", + "src/build", + "src/main/build", + "src/main/java", + "src/main/resources/build", + "lib/src/main/java/build/inner", + ] { + assert!(!is_jvm_source_build_dir(pruned), "{pruned}"); + } + } + #[test] fn test_default_config_parses() { let toml_str = r#" diff --git a/crates/codegraph-extract/src/engine.rs b/crates/codegraph-extract/src/engine.rs index 7577f918..d33f68dc 100644 --- a/crates/codegraph-extract/src/engine.rs +++ b/crates/codegraph-extract/src/engine.rs @@ -542,10 +542,15 @@ fn scan_dir( // while a same-basename user directory elsewhere is not. let is_reserved_root_here = (dir == root && name == ".git") || reserved_roots.contains(&path); - if is_reserved_root_here || ignored_dirs.contains(name.as_ref()) { + let relative = normalize_path(path.strip_prefix(root).unwrap_or(&path)); + // `build` is also a legal JVM package segment: keep it under a + // conventional source root while still pruning build output (#1642). + let jvm_package = name == "build" + && codegraph_core::config::is_jvm_source_build_dir(&relative) + && entry.file_type().is_ok_and(|kind| kind.is_dir()); + if is_reserved_root_here || (ignored_dirs.contains(name.as_ref()) && !jvm_package) { continue; } - let relative = normalize_path(path.strip_prefix(root).unwrap_or(&path)); let ignored = is_path_ignored(&relative, pattern_sets); let file_type = entry.file_type()?; if file_type.is_dir() { @@ -926,6 +931,50 @@ mod tests { fs::remove_dir_all(&project).ok(); } + /// `build` is a legal JVM package segment: a Java/Kotlin/Scala package + /// directory named `build` under a conventional source root stays indexed, + /// while module build output and the other defaults inside it stay pruned + /// (upstream #1642). + #[test] + fn scan_keeps_jvm_packages_named_build_but_prunes_build_output() { + let project = unique_project("jvm_build_package"); + touch( + &project, + "src/main/java/com/acme/build/Builder.java", + "class Builder {}", + ); + touch( + &project, + "app/src/test/kotlin/build/BuildTest.kt", + "class BuildTest", + ); + touch(&project, "src/main/scala/build/Tool.scala", "object Tool"); + touch(&project, "build/generated/Gen.java", "class Gen {}"); + touch(&project, "app/build/classes/Out.java", "class Out {}"); + touch( + &project, + "src/main/resources/build/Res.java", + "class Res {}", + ); + touch( + &project, + "src/main/java/build/node_modules/dep.js", + "export {}", + ); + + let files = scan_project(&project, &ExtractOptions::default()).expect("scan project"); + assert_eq!( + files, + vec![ + "app/src/test/kotlin/build/BuildTest.kt".to_string(), + "src/main/java/com/acme/build/Builder.java".to_string(), + "src/main/scala/build/Tool.scala".to_string(), + ] + ); + + fs::remove_dir_all(&project).ok(); + } + /// A team authoring first-party code under `addons/` can re-include it by /// overriding `indexing.ignore_dirs` (the same override surface a custom /// `.codegraph/config.toml` populates), proving the default is opt-out. diff --git a/crates/codegraph-watch/src/policy.rs b/crates/codegraph-watch/src/policy.rs index 9b0623d9..74d6f09d 100644 --- a/crates/codegraph-watch/src/policy.rs +++ b/crates/codegraph-watch/src/policy.rs @@ -191,9 +191,13 @@ impl WatchPolicy { } fn matches_structural(&self, relative: &str, is_dir: bool) -> bool { - self.structural_ignores - .iter() - .any(|pattern| rule_matches(pattern, relative, is_dir)) + self.structural_ignores.iter().any(|pattern| { + if pattern == "build/" { + build_segment_is_output(relative, is_dir) + } else { + rule_matches(pattern, relative, is_dir) + } + }) } /// The watcher's port of the scan's `is_path_ignored` fold over its ordered @@ -448,6 +452,25 @@ fn read_gitignore_rules(root: &Path) -> Vec { .collect() } +/// The `build/` structural rule, segment by segment: a `build` directory on the +/// path prunes it unless it is a JVM package under a conventional source root +/// (#1642, mirroring `scan_dir`). +fn build_segment_is_output(relative: &str, is_dir: bool) -> bool { + let segments = relative + .trim_end_matches('/') + .split('/') + .collect::>(); + let directories = if is_dir { + segments.len() + } else { + segments.len().saturating_sub(1) + }; + (0..directories).any(|index| { + segments[index] == "build" + && !codegraph_core::config::is_jvm_source_build_dir(&segments[..=index].join("/")) + }) +} + fn rule_matches(pattern: &str, relative: &str, is_dir: bool) -> bool { let candidate = if is_dir { format!("{}/", relative.trim_end_matches('/')) @@ -896,6 +919,20 @@ mod tests { assert!(policy.should_handle_file("src/app.ts")); } + #[test] + fn jvm_packages_named_build_are_watched_like_the_scan_indexes_them() { + let dir = crate::sync::tests::TestDir::new("watch-policy-jvm-build"); + let policy = WatchPolicy::new(dir.path()); + assert!(policy.should_watch_dir("src/main/java/com/acme/build")); + assert!(policy.should_handle_file("src/main/java/com/acme/build/Builder.java")); + assert!(policy.should_handle_file("app/src/test/kotlin/build/BuildTest.kt")); + assert!(!policy.should_watch_dir("build")); + assert!(!policy.should_watch_dir("app/build")); + assert!(!policy.should_handle_file("app/build/classes/Out.java")); + assert!(!policy.should_handle_file("src/main/resources/build/Res.java")); + assert!(!policy.should_handle_file("src/main/java/build/node_modules/dep.js")); + } + #[test] fn should_handle_file_requires_known_language() { let dir = crate::sync::tests::TestDir::new("watch-policy-handle"); From 27ad0eb9edff80e889e62a1b92d0aaefb2094b8b Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 09:21:24 +0800 Subject: [PATCH 02/19] fix(scan): read the root .gitignore with git's rules in scan and watch The scan matched root .gitignore lines literally: anchored rules, globs and ** never matched, and dir rules stopped at the root. The watcher read the same file differently, matching every dir rule at any depth. Both now share one matcher built on the ignore crate: a leading or inner / anchors a rule, a slash-less rule applies at any depth, * and ** glob, and nothing is re-included below an ignored directory. A JVM package named build therefore stays indexed and watched under Android's /build and Spring Initializr's build/ plus !**/src/main/**/build/, while a bare build/ hides it (upstream #1642). --- Cargo.lock | 1 + Cargo.toml | 1 + crates/codegraph-extract/Cargo.toml | 1 + crates/codegraph-extract/src/engine.rs | 191 ++++++++++++++++-- crates/codegraph-extract/src/lib.rs | 8 +- crates/codegraph-watch/src/policy.rs | 77 ++----- .../tests/include_scan_watch_parity.rs | 62 ++++++ docs/cli.md | 11 +- 8 files changed, 269 insertions(+), 83 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 61d82ab6..dfb923a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -495,6 +495,7 @@ version = "0.50.3" dependencies = [ "anyhow", "codegraph-core", + "ignore", "rayon", "regex", "serde", diff --git a/Cargo.toml b/Cargo.toml index 4a6f2b2c..0264354d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ sha2 = "0.10" sha1_smol = "1" rayon = "1.8" globset = "0.4" +ignore = "0.4" # Logging (rust-base-convention) tracing = "0.1" diff --git a/crates/codegraph-extract/Cargo.toml b/crates/codegraph-extract/Cargo.toml index 1f0eb0c7..bfe9a875 100644 --- a/crates/codegraph-extract/Cargo.toml +++ b/crates/codegraph-extract/Cargo.toml @@ -21,6 +21,7 @@ rayon = { workspace = true } tree-sitter = { workspace = true } streaming-iterator = { workspace = true } codegraph-core = { path = "../codegraph-core" } +ignore = { workspace = true } serde_json = { workspace = true } tracing = { workspace = true } diff --git a/crates/codegraph-extract/src/engine.rs b/crates/codegraph-extract/src/engine.rs index d33f68dc..4625a328 100644 --- a/crates/codegraph-extract/src/engine.rs +++ b/crates/codegraph-extract/src/engine.rs @@ -487,10 +487,10 @@ pub fn scan_project_with_stats(root: &Path, options: &ExtractOptions) -> Result< .iter() .map(String::as_str) .collect::>(); - let gitignore = read_root_gitignore(root); + let gitignore = RootGitignore::load(root); // Evaluated in order (default paths → config exclude → .gitignore), so a // later `!pattern` negation re-includes a path an earlier set excluded. - let pattern_sets: Vec<&[String]> = vec![&options.ignore_paths, &options.exclude, &gitignore]; + let pattern_sets: Vec<&[String]> = vec![&options.ignore_paths, &options.exclude]; let include = IncludeSet::new(&options.include, &options.exclude); // The EXACT resolved reserved index-root PATH for THIS project, resolved // once. scan_dir prunes a directory iff its FULL path equals this root, @@ -506,6 +506,7 @@ pub fn scan_project_with_stats(root: &Path, options: &ExtractOptions) -> Result< &ignored_dirs, &reserved_roots, &pattern_sets, + (&gitignore, GitignoreAncestry::default()), &include, &options.extensions, &mut files, @@ -525,6 +526,7 @@ fn scan_dir( ignored_dirs: &HashSet<&str>, reserved_roots: &std::collections::BTreeSet, pattern_sets: &[&[String]], + (gitignore, above): (&RootGitignore, GitignoreAncestry), include: &IncludeSet, overrides: &ExtensionOverrides, files: &mut Vec, @@ -551,8 +553,9 @@ fn scan_dir( if is_reserved_root_here || (ignored_dirs.contains(name.as_ref()) && !jvm_package) { continue; } - let ignored = is_path_ignored(&relative, pattern_sets); let file_type = entry.file_type()?; + let own = gitignore.matched(&relative, file_type.is_dir()); + let ignored = is_path_ignored(&relative, pattern_sets, above.decide(own)); if file_type.is_dir() { // A model-ignored dir is normally pruned before descent, so a FILE // include under a gitignored ancestor would never be reached. @@ -567,6 +570,7 @@ fn scan_dir( ignored_dirs, reserved_roots, pattern_sets, + (gitignore, above.child(own)), include, overrides, files, @@ -627,22 +631,96 @@ fn size_skip_result(file_path: &str, size: u64, max: u64) -> ExtractionResult { } } -fn read_root_gitignore(root: &Path) -> Vec { - fs::read_to_string(root.join(".gitignore")) - .unwrap_or_default() - .lines() - .map(str::trim) - .filter(|line| !line.is_empty() && !line.starts_with('#')) - .map(ToOwned::to_owned) - .collect() +/// The project-root `.gitignore`, read with git's own rules through the +/// `ignore` crate, as upstream reads it with the `ignore` package: a leading or +/// inner `/` anchors a rule to the root, a slash-less rule applies at any +/// depth, `*` and `**` glob, a trailing `/` matches directories only, and `!` +/// re-includes. Shared with `codegraph-watch`, so the watcher's `.gitignore` +/// verdict is the scan's by construction. An unreadable, non-UTF-8 or NUL-laden +/// file is treated as absent and an unparseable line is skipped, never fatal. +#[derive(Debug, Clone, Default)] +pub struct RootGitignore { + matcher: Option, } -/// Evaluate ordered `.gitignore`-style pattern sets with last-match-wins -/// negation: a `!pattern` line un-ignores a path an earlier pattern excluded. -/// Sets are scanned in order, patterns within a set in order, and the final -/// matching pattern decides — so a later `!res/values/` re-includes what a -/// default `res/values*` excluded. -fn is_path_ignored(relative: &str, pattern_sets: &[&[String]]) -> bool { +impl RootGitignore { + pub fn load(root: &Path) -> Self { + let Ok(text) = fs::read_to_string(root.join(".gitignore")) else { + return Self::default(); + }; + if text.contains('\0') { + return Self::default(); + } + // Rooted at `.` so a root-relative candidate is never prefix-stripped. + let mut builder = ignore::gitignore::GitignoreBuilder::new("."); + for line in text.lines() { + // An invalid glob drops only its own line; the rest still apply. + let _ = builder.add_line(None, line); + } + let matcher = builder.build().ok().filter(|matcher| !matcher.is_empty()); + Self { matcher } + } + + /// The root `.gitignore` verdict for `relative` (root-relative, + /// `/`-separated): `Some(true)` ignored, `Some(false)` re-included, `None` + /// undecided. Walks the directories above it; the scan instead carries + /// that state down its walk in a [`GitignoreAncestry`]. + pub fn verdict(&self, relative: &str, is_dir: bool) -> Option { + self.matcher.as_ref()?; + let above = relative + .match_indices('/') + .fold(GitignoreAncestry::default(), |above, (end, _)| { + above.child(self.matched(&relative[..end], true)) + }); + above.decide(self.matched(relative, is_dir)) + } + + /// The last rule matching `relative` itself, ignoring its ancestors. + fn matched(&self, relative: &str, is_dir: bool) -> Option { + match self.matcher.as_ref()?.matched(relative, is_dir) { + ignore::Match::None => None, + ignore::Match::Ignore(_) => Some(true), + ignore::Match::Whitelist(_) => Some(false), + } + } +} + +/// What the root `.gitignore` decided for the directories above a path. +#[derive(Debug, Clone, Copy, Default)] +struct GitignoreAncestry { + ignored: bool, + reincluded: bool, +} + +impl GitignoreAncestry { + /// The ancestry of an entry below a directory whose own verdict was `own`. + fn child(self, own: Option) -> Self { + Self { + ignored: self.ignored || own == Some(true), + reincluded: self.reincluded || own == Some(false), + } + } + + /// An ignored ancestor hides the path, since git never re-includes below an + /// excluded directory; otherwise the path's own last matching rule decides, + /// and failing that a re-included ancestor re-includes it, so `!res/values/` + /// still overrides a default path set that matched the files inside. + fn decide(self, own: Option) -> Option { + if self.ignored { + Some(true) + } else { + own.or(self.reincluded.then_some(false)) + } + } +} + +/// Evaluate the ordered `.gitignore`-style config pattern sets with last-match +/// wins negation, then let the root `.gitignore` verdict, when it has one, +/// decide last: a `!pattern` line un-ignores a path an earlier pattern +/// excluded. Sets are scanned in order, patterns within a set in order, and the +/// final matching pattern decides — so a later `!res/values/` re-includes what +/// a default `res/values*` excluded. +fn is_path_ignored(relative: &str, pattern_sets: &[&[String]], gitignore: Option) -> bool { let mut ignored = false; for set in pattern_sets { for pattern in set.iter() { @@ -655,7 +733,7 @@ fn is_path_ignored(relative: &str, pattern_sets: &[&[String]]) -> bool { } } } - ignored + gitignore.unwrap_or(ignored) } fn pattern_matches(relative: &str, pattern: &str) -> bool { @@ -1406,14 +1484,20 @@ mod tests { fn is_path_ignored_negation_is_last_match_wins() { let defaults = vec!["res/values*".to_string()]; let user = vec!["!res/values/".to_string()]; - assert!(is_path_ignored("res/values/strings.xml", &[&defaults])); + assert!(is_path_ignored( + "res/values/strings.xml", + &[&defaults], + None + )); assert!(!is_path_ignored( "res/values/strings.xml", - &[&defaults, &user] + &[&defaults, &user], + None )); assert!(is_path_ignored( "res/values-es/strings.xml", - &[&defaults, &user] + &[&defaults, &user], + None )); } @@ -1432,6 +1516,71 @@ mod tests { fs::remove_dir_all(&project).ok(); } + /// The root `.gitignore` is read with git's own rules, as upstream's `ignore` + /// matcher reads it: a leading or inner `/` anchors a rule to the root, a + /// slash-less rule applies at any depth, `*` and `**` glob, and nothing below + /// an ignored directory can be re-included. + #[test] + fn scan_project_reads_the_root_gitignore_with_git_rules() { + let project = unique_project("gitignore_git_rules"); + touch( + &project, + ".gitignore", + "/artifacts/\n*.gen.ts\ndocs/**/draft/\nscratch/\nlogs/\n!logs/keep.ts\n", + ); + for file in [ + "src/app.ts", + "artifacts/a.ts", + "src/artifacts/b.ts", + "src/types.gen.ts", + "docs/guide/draft/c.ts", + "docs/guide/final/d.ts", + "src/scratch/e.ts", + "logs/keep.ts", + ] { + touch(&project, file, "export const x = 1;\n"); + } + let files = scan_project(&project, &ExtractOptions::default()).expect("scan"); + assert_eq!( + files, + vec!["docs/guide/final/d.ts", "src/app.ts", "src/artifacts/b.ts"] + ); + fs::remove_dir_all(&project).ok(); + } + + /// An explicit root `.gitignore` still decides a JVM package named `build` + /// (upstream #1642): a bare `build/` hides it as git does, while Android's + /// anchored `/build` and Spring Initializr's `build/` plus + /// `!**/src/main/**/build/` keep it. Module build output stays pruned. + #[test] + fn root_gitignore_still_decides_jvm_packages_named_build() { + for (gitignore, kept) in [ + ("build/\n", false), + ("/build\n", true), + ( + "build/\n!**/src/main/**/build/\n!**/src/test/**/build/\n", + true, + ), + ] { + let project = unique_project("jvm_build_gitignore"); + touch(&project, ".gitignore", gitignore); + touch(&project, "src/main/java/App.java", "class App {}"); + touch( + &project, + "src/main/java/com/acme/build/Hidden.java", + "class Hidden {}", + ); + touch(&project, "app/build/generated/Out.java", "class Out {}"); + let files = scan_project(&project, &ExtractOptions::default()).expect("scan"); + let mut expected = vec!["src/main/java/App.java".to_string()]; + if kept { + expected.push("src/main/java/com/acme/build/Hidden.java".to_string()); + } + assert_eq!(files, expected, "{gitignore:?}"); + fs::remove_dir_all(&project).ok(); + } + } + #[test] fn scan_project_reports_visible_unsupported_extensions_in_the_same_walk() { let project = unique_project("unsupported-extensions"); diff --git a/crates/codegraph-extract/src/lib.rs b/crates/codegraph-extract/src/lib.rs index aa9d2b83..d1b2cfb4 100644 --- a/crates/codegraph-extract/src/lib.rs +++ b/crates/codegraph-extract/src/lib.rs @@ -17,9 +17,9 @@ pub mod spec; pub mod walker; pub use engine::{ - ExtractOptions, ExtractionStage, PARSE_COLLAPSE_WARNING, detect_language, detect_language_with, - extract_file, extract_file_with_options, extract_file_with_options_observer, extract_project, - extract_source, extract_source_with, extract_source_with_observer, - include_exclude_pattern_matches, is_extraction_warning, + ExtractOptions, ExtractionStage, PARSE_COLLAPSE_WARNING, RootGitignore, detect_language, + detect_language_with, extract_file, extract_file_with_options, + extract_file_with_options_observer, extract_project, extract_source, extract_source_with, + extract_source_with_observer, include_exclude_pattern_matches, is_extraction_warning, }; pub use ext_config::ExtensionOverrides; diff --git a/crates/codegraph-watch/src/policy.rs b/crates/codegraph-watch/src/policy.rs index 74d6f09d..69bd59f0 100644 --- a/crates/codegraph-watch/src/policy.rs +++ b/crates/codegraph-watch/src/policy.rs @@ -2,7 +2,7 @@ use std::fs; use std::path::{Path, PathBuf}; use std::sync::Arc; -use codegraph_extract::{ExtensionOverrides, detect_language_with}; +use codegraph_extract::{ExtensionOverrides, RootGitignore, detect_language_with}; pub const CODEGRAPH_NO_WATCH: &str = "CODEGRAPH_NO_WATCH"; @@ -19,12 +19,6 @@ const WATCH_ONLY_DEFAULT_IGNORE_DIRS: &[&str] = &[ ".cache", ]; -#[derive(Debug, Clone)] -struct IgnoreRule { - pattern: String, - negated: bool, -} - #[derive(Debug, Clone)] pub struct WatchPolicy { root: PathBuf, @@ -41,11 +35,12 @@ pub struct WatchPolicy { /// uses, never [`rule_matches`] — whose `*` form compares BASENAMES, so it /// could never match a `res/values*` prefix against `res/values/strings.xml`. ignore_paths: Vec, - /// The root `.gitignore` rules in file order — the NEGOTIABLE tail of the - /// last-match-wins stream (`ignore_paths` and `exclude` first, these last, - /// mirroring the scan's `pattern_sets`), so a `!pattern` line here re-includes - /// what an `ignore_paths`, `exclude`, or earlier `.gitignore` line dropped. - gitignore_rules: Vec, + /// The root `.gitignore` — the NEGOTIABLE tail of the last-match-wins stream + /// (`ignore_paths` and `exclude` first, it last, mirroring the scan), so a + /// `!pattern` line here re-includes what an `ignore_paths`, `exclude`, or + /// earlier `.gitignore` line dropped. It is the scan's own git-rules + /// matcher, so the two read every `.gitignore` line identically. + gitignore: RootGitignore, include: Vec, exclude: Vec, /// The addressed project's custom extension→language overrides, so a file the @@ -95,12 +90,12 @@ impl WatchPolicy { .map(str::to_string), ) .collect::>(); - let gitignore_rules = read_gitignore_rules(&root); + let gitignore = RootGitignore::load(&root); Self { root, structural_ignores, ignore_paths: ignore_paths.to_vec(), - gitignore_rules, + gitignore, include: include.to_vec(), exclude: exclude.to_vec(), extensions: ExtensionOverrides::empty(), @@ -208,20 +203,16 @@ impl WatchPolicy { /// `gen/helper.ts` — and the set order is observable, so a `!` in `exclude` /// re-includes an `ignore_paths` match but not the reverse. /// - /// The ONLY deviation from the scan is which matcher each set uses, and it - /// is LOAD-BEARING — do not "simplify" this into a single matcher: - /// the two CONFIG sets use the SHARED whole-path - /// [`codegraph_extract::include_exclude_pattern_matches`] the scan itself - /// uses (its trailing-`*` form is a whole-path prefix, so `res/values*` - /// matches `res/values/strings.xml`), while `.gitignore` keeps the - /// watcher's basename-glob [`rule_matches`]. A `!` in a config set is - /// stripped and then re-matched with that set's OWN matcher, mirroring how - /// the scan strips then calls `pattern_matches`. The config sets are also - /// matched WITHOUT `is_dir`, exactly as the scan feeds a bare `relative` to - /// `is_path_ignored`; only the `.gitignore` fold is `is_dir`-aware. The - /// `.gitignore` rules arrive pre-parsed into [`IgnoreRule`] by - /// [`read_gitignore_rules`], so their `!` is already split off, whereas the - /// config sets are raw patterns with the `!` still inline. + /// Each set uses the scan's own matcher: the two CONFIG sets the SHARED + /// whole-path [`codegraph_extract::include_exclude_pattern_matches`] (its + /// trailing-`*` form is a whole-path prefix, so `res/values*` matches + /// `res/values/strings.xml`), and `.gitignore` the SHARED git-rules + /// [`RootGitignore`]. A `!` in a config set is stripped and then re-matched + /// with that set's OWN matcher, mirroring how the scan strips then calls + /// `pattern_matches`. The config sets are also matched WITHOUT `is_dir`, + /// exactly as the scan feeds a bare `relative` to `is_path_ignored`; only + /// the `.gitignore` verdict is `is_dir`-aware, and it walks the directories + /// above `relative` the way the scan carries them down its walk. fn matches_negotiable(&self, relative: &str, is_dir: bool) -> bool { let mut ignored = false; for set in [&self.ignore_paths, &self.exclude] { @@ -235,12 +226,7 @@ impl WatchPolicy { } } } - for rule in &self.gitignore_rules { - if rule_matches(&rule.pattern, relative, is_dir) { - ignored = !rule.negated; - } - } - ignored + self.gitignore.verdict(relative, is_dir).unwrap_or(ignored) } /// The scan's `IncludeSet::forces`/`wants_descend` knockout check, which is a @@ -432,26 +418,6 @@ pub fn normalize_path(path: impl AsRef) -> String { .replace('\\', "/") } -fn read_gitignore_rules(root: &Path) -> Vec { - fs::read_to_string(root.join(".gitignore")) - .unwrap_or_default() - .lines() - .filter_map(|line| { - let trimmed = line.trim(); - if trimmed.is_empty() || trimmed.starts_with('#') { - return None; - } - let (negated, pattern) = trimmed - .strip_prefix('!') - .map_or((false, trimmed), |pattern| (true, pattern)); - Some(IgnoreRule { - pattern: pattern.trim_start_matches('/').to_string(), - negated, - }) - }) - .collect() -} - /// The `build/` structural rule, segment by segment: a `build` directory on the /// path prunes it unless it is a JVM package under a conventional source root /// (#1642, mirroring `scan_dir`). @@ -690,7 +656,8 @@ mod tests { fs::write(dir.path().join(".gitignore"), "a/b/\n").unwrap(); let policy = WatchPolicy::new(dir.path()); assert!(!policy.should_watch_dir("a/b")); - assert!(!policy.should_watch_dir("x/a/b")); + // An inner `/` anchors the rule to the root, in git and in the scan. + assert!(policy.should_watch_dir("x/a/b")); assert!(!policy.should_watch_dir("a/b/c")); assert!(policy.should_watch_dir("a/bb")); assert!(policy.should_watch_dir("za/b")); diff --git a/crates/codegraph-watch/tests/include_scan_watch_parity.rs b/crates/codegraph-watch/tests/include_scan_watch_parity.rs index 36093260..81933117 100644 --- a/crates/codegraph-watch/tests/include_scan_watch_parity.rs +++ b/crates/codegraph-watch/tests/include_scan_watch_parity.rs @@ -531,3 +531,65 @@ fn config_set_order_decides_which_negation_wins() { watcher either" ); } + +/// The root `.gitignore` is read by ONE git-rules matcher on both sides, so the +/// watcher handles exactly the files the scan indexes and watches every +/// directory above them: anchored and any-depth rules, globs, `**`, no +/// re-inclusion below an ignored directory, and JVM packages named `build` +/// under bare, Android and Spring Initializr rules (#1642). +#[test] +fn scan_and_watch_agree_on_root_gitignore_rules() { + let files = [ + "src/app.ts", + "artifacts/a.ts", + "src/artifacts/b.ts", + "src/types.gen.ts", + "docs/guide/draft/c.ts", + "docs/guide/final/d.ts", + "src/scratch/e.ts", + "logs/keep.ts", + "src/main/java/com/acme/build/Hidden.java", + "src/main/java/build/nested/build/Example.java", + "app/build/generated/Out.java", + ]; + for gitignore in [ + "/artifacts/\n*.gen.ts\ndocs/**/draft/\nscratch/\nlogs/\n!logs/keep.ts\n", + "build/\n", + "/build\n", + "build/\n!**/src/main/**/build/\n!**/src/test/**/build/\n", + ] { + let project = unique_project("gitignore_rules"); + touch(&project, ".gitignore", gitignore); + for file in files { + touch(&project, file, "x"); + } + let options = ExtractOptions::default(); + let scanned = scan_project(&project, &options).expect("scan"); + let policy = WatchPolicy::with_config( + &project, + &options.ignore_dirs, + &options.ignore_paths, + &[], + &[], + ); + for file in files { + let in_scan = scanned.iter().any(|f| f == file); + assert_eq!( + in_scan, + policy.should_handle_file(file), + "scan⇔watch parity broken for .gitignore={gitignore:?} file={file}" + ); + if in_scan { + let segments = file.split('/').collect::>(); + for depth in 1..segments.len() { + let dir = segments[..depth].join("/"); + assert!( + policy.should_watch_dir(&dir), + ".gitignore={gitignore:?}: {dir} holds indexed {file} but is not watched" + ); + } + } + } + fs::remove_dir_all(&project).ok(); + } +} diff --git a/docs/cli.md b/docs/cli.md index b2958b9e..bb0ee5fc 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1032,9 +1032,14 @@ those paths; the reason is surfaced in the log. The watcher registers per-directory watches only on non-ignored directories, pruning `node_modules`, `.venv`, `__pycache__`, `target`, `dist`, `.godot`, -`.cache`, `.git`, `.codegraph`, and everything else in the -default ignore set, plus any paths matched by the root `.gitignore`. This pruning applies at any -nesting depth, so an `node_modules` buried several levels deep is never walked. +`.cache`, `.git`, `.codegraph`, and everything else in the default ignore set at +any nesting depth, so a `node_modules` buried several levels deep is never walked. +A `build` directory that is a Java, Kotlin or Scala package under a source root +(`src//{java,kotlin,scala}/…/build`) is source rather than build +output, so it stays indexed and watched. The root `.gitignore` prunes the index +and the watcher alike, with git's own rules: a slash-less rule applies at any +depth, a leading or inner `/` anchors it to the project root, `*` and `**` glob, +and `!` re-includes a path unless a directory above it is ignored. This keeps the total watch count well inside the OS inotify limit on large trees and makes daemon startup fast. A newly-created non-ignored directory is picked up automatically on its create event — no restart required. From f9fae7a623d719b56a83dc9e41d233ac4c2c4fec Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 09:32:28 +0800 Subject: [PATCH 03/19] fix(resolve): read no file extraction would skip Resolution and the framework pass read files by path with no guard, so an import that followed package metadata into a dependency archive decoded it whole, and a FIFO blocked the pass. Stat first: only a regular file within the project's indexing.max_file_size is read, the read itself is bounded, and the rejection is cached like any read. The resolver hands the configured limit to every context it builds, so index and sync read the same files (upstream #1553). --- crates/codegraph-cli/src/main.rs | 6 +- crates/codegraph-core/src/config.rs | 12 +- crates/codegraph-resolve/src/context.rs | 33 ++- crates/codegraph-resolve/src/resolver.rs | 50 +++- .../codegraph-resolve/src/snapshot_context.rs | 15 +- .../tests/resolution_file_reads.rs | 232 ++++++++++++++++++ crates/codegraph-watch/src/migrate.rs | 8 +- crates/codegraph-watch/src/sync.rs | 18 +- 8 files changed, 337 insertions(+), 37 deletions(-) create mode 100644 crates/codegraph-resolve/tests/resolution_file_reads.rs diff --git a/crates/codegraph-cli/src/main.rs b/crates/codegraph-cli/src/main.rs index 1ea248a0..b5058457 100644 --- a/crates/codegraph-cli/src/main.rs +++ b/crates/codegraph-cli/src/main.rs @@ -5536,13 +5536,13 @@ fn index_project_inner( diagnostic_run.phase_start("framework_extract"); let framework_started = std::time::Instant::now(); let pb = phase_spinner("Detecting frameworks", quiet); - let mut resolver = ReferenceResolver::new(project.to_string_lossy()); + let mut resolver = + ReferenceResolver::new(project.to_string_lossy()).with_max_file_size(options.max_file_size); // Detect frameworks then run their per-file extract (route/component/handler // nodes + refs) BEFORE resolution, mirroring the upstream tree-sitter.ts:4796-4819 // framework-extraction pass feeding the resolution pipeline. { - let context = - codegraph_resolve::StoreResolutionContext::new(&store, project.to_string_lossy()); + let context = resolver.store_context(&store); resolver.initialize(&context); } if resolver.has_framework_resolvers() { diff --git a/crates/codegraph-core/src/config.rs b/crates/codegraph-core/src/config.rs index 2565c75b..13eff207 100644 --- a/crates/codegraph-core/src/config.rs +++ b/crates/codegraph-core/src/config.rs @@ -92,11 +92,15 @@ pub struct IndexingConfig { pub deprioritize: Vec, } +/// The largest source file extraction parses unless a project raises +/// `indexing.max_file_size`; resolution reads no file extraction would skip. +// upstream extraction/index.ts:101, src/file-limits.ts (`MAX_SOURCE_FILE_SIZE_BYTES`) +// Skip files larger than this (bytes). Generated bundles, minified JS, and +// vendored blobs blow the WASM heap. 1 MB covers essentially all hand-written source. +pub const DEFAULT_MAX_FILE_SIZE: u64 = 1024 * 1024; + fn default_max_file_size() -> u64 { - // upstream extraction/index.ts:101 - // Skip files larger than this (bytes). Generated bundles, minified JS, and - // vendored blobs blow the WASM heap. 1 MB covers essentially all hand-written source. - 1024 * 1024 + DEFAULT_MAX_FILE_SIZE } /// Whether `relative_dir` — a project-relative, `/`-separated directory path — diff --git a/crates/codegraph-resolve/src/context.rs b/crates/codegraph-resolve/src/context.rs index 912226e2..ffff3a78 100644 --- a/crates/codegraph-resolve/src/context.rs +++ b/crates/codegraph-resolve/src/context.rs @@ -32,6 +32,7 @@ pub(crate) const DEFAULT_CACHE_LIMIT: usize = 5_000; pub struct StoreResolutionContext<'a> { store: &'a Store, project_root: String, + max_file_size: u64, caches: RefCell, } @@ -82,10 +83,19 @@ impl<'a> StoreResolutionContext<'a> { Self { store, project_root: project_root.into(), + max_file_size: codegraph_core::config::DEFAULT_MAX_FILE_SIZE, caches: RefCell::new(Caches::new()), } } + /// Read files up to the addressed project's extraction limit + /// (`indexing.max_file_size`) instead of the default one. + #[must_use] + pub fn with_max_file_size(mut self, max_file_size: u64) -> Self { + self.max_file_size = max_file_size; + self + } + /// Drop all caches (`clearCaches`, `index.ts:313-324`). Called between /// resolution passes so the conformance pass sees freshly persisted edges. pub fn clear_caches(&self) { @@ -261,7 +271,8 @@ impl ResolutionContext for StoreResolutionContext<'_> { return c.file_cache.get(&file_path.to_string()).flatten(); } let full_path = Path::new(&self.project_root).join(file_path); - let content: Option> = std::fs::read_to_string(&full_path).ok().map(Arc::from); + let content: Option> = + read_source_file(&full_path, self.max_file_size).map(Arc::from); c.file_cache.set(file_path.to_string(), content.clone()); content } @@ -512,6 +523,26 @@ fn load_go_module(project_root: &str) -> Option { load_go_module_in(project_root, "") } +/// Read `path` for resolution only if extraction would accept it: a regular +/// file no larger than `max_bytes` (upstream #1553). The `stat` comes first, so +/// a dependency archive an import resolves to is never decoded and a FIFO is +/// never opened; the read is bounded as well, in case the file grows meanwhile. +pub(crate) fn read_source_file(path: &Path, max_bytes: u64) -> Option { + use std::io::Read; + + let metadata = std::fs::metadata(path).ok()?; + if !metadata.is_file() || metadata.len() > max_bytes { + return None; + } + let mut content = String::new(); + std::fs::File::open(path) + .ok()? + .take(max_bytes.saturating_add(1)) + .read_to_string(&mut content) + .ok()?; + (content.len() as u64 <= max_bytes).then_some(content) +} + /// Read the `module` line of `//go.mod`. fn load_go_module_in(project_root: &str, dir: &str) -> Option { let relative = if dir.is_empty() { diff --git a/crates/codegraph-resolve/src/resolver.rs b/crates/codegraph-resolve/src/resolver.rs index 051759ce..9e5a05dc 100644 --- a/crates/codegraph-resolve/src/resolver.rs +++ b/crates/codegraph-resolve/src/resolver.rs @@ -1284,6 +1284,9 @@ fn target_is_visible_across_files( /// `FrameworkResolver`s). pub struct ReferenceResolver { project_root: String, + /// Largest file resolution reads, the project's extraction limit + /// (`indexing.max_file_size`), handed to every context the resolver builds. + max_file_size: u64, /// Detected `FrameworkResolver` implementations (`frameworks` field, /// `index.ts:264`). Empty until [`Self::initialize`] detects react/vue/nestjs. framework_resolver_extensions: Vec>, @@ -1318,6 +1321,7 @@ impl ReferenceResolver { pub fn new(project_root: impl Into) -> Self { Self { project_root: project_root.into(), + max_file_size: codegraph_core::config::DEFAULT_MAX_FILE_SIZE, framework_resolver_extensions: Vec::new(), known_names: None, c_family_visible_macros: BTreeMap::new(), @@ -1327,6 +1331,25 @@ impl ReferenceResolver { } } + /// Read files up to the addressed project's extraction limit + /// (`indexing.max_file_size`) instead of the default one, so resolution + /// never reads a file extraction would skip and never skips one it indexed. + #[must_use] + pub fn with_max_file_size(mut self, max_file_size: u64) -> Self { + self.max_file_size = max_file_size; + self + } + + /// A store-backed resolution context for this resolver's project and file + /// size limit. + pub fn store_context<'a>( + &self, + store: &'a Store, + ) -> crate::context::StoreResolutionContext<'a> { + crate::context::StoreResolutionContext::new(store, &self.project_root) + .with_max_file_size(self.max_file_size) + } + /// Detect the project's frameworks and populate the resolver's /// `FrameworkResolver` list (`initialize`, `index.ts:263-266`). Must run /// before `resolve_all` so Strategy-1 dispatch sees the detected resolvers. @@ -1346,7 +1369,7 @@ impl ReferenceResolver { pub fn run_post_extract(&self, store: &mut Store) -> anyhow::Result { let mut all_updates: Vec = Vec::new(); { - let context = crate::context::StoreResolutionContext::new(store, &self.project_root); + let context = self.store_context(store); for resolver in &self.framework_resolver_extensions { if let Some(nodes) = resolver.post_extract(&context) { all_updates.extend(nodes); @@ -1414,8 +1437,10 @@ impl ReferenceResolver { context, extensions, |relative| { - std::fs::read_to_string(std::path::Path::new(&self.project_root).join(relative)) - .ok() + crate::context::read_source_file( + &std::path::Path::new(&self.project_root).join(relative), + self.max_file_size, + ) }, ) } @@ -2139,7 +2164,7 @@ impl ReferenceResolver { } let resolved: Vec = { - let context = crate::context::StoreResolutionContext::new(store, &self.project_root); + let context = self.store_context(store); deferred .iter() .filter_map(|reference| { @@ -2231,7 +2256,7 @@ impl ReferenceResolver { pub fn resolve_and_persist(&mut self, store: &mut Store) -> anyhow::Result { let unresolved_refs = store.all_unresolved_refs()?; let result = { - let context = crate::context::StoreResolutionContext::new(store, &self.project_root); + let context = self.store_context(store); self.resolve_all(&unresolved_refs, &context) }; @@ -2365,7 +2390,7 @@ impl ReferenceResolver { store.set_resolution_incomplete()?; { - let context = crate::context::StoreResolutionContext::new(store, &self.project_root); + let context = self.store_context(store); self.warm_caches(&context); } @@ -2428,18 +2453,17 @@ impl ReferenceResolver { // context. Prior batches' inheritance edges are already committed // (persisted at the bottom of this loop), so `get_supertypes` // reads them straight from the store — no snapshot, no adjacency. - let context = - crate::context::StoreResolutionContext::new(store, &self.project_root); + let context = self.store_context(store); self.resolve_all(&batch, &context) } else { let base = match &node_snapshot { Some(snapshot) => snapshot, None => { base_adjacency = (*build_edge_adjacency(store)?).clone(); - node_snapshot.insert(SnapshotResolutionContext::from_store( - store, - &self.project_root, - )?) + node_snapshot.insert( + SnapshotResolutionContext::from_store(store, &self.project_root)? + .with_max_file_size(self.max_file_size), + ) } }; // Install the adjacency from chunks 0..N-1 BEFORE resolving chunk N @@ -2614,7 +2638,7 @@ impl ReferenceResolver { let scoped = union_persisted_refs([by_files, by_sites, by_names])?; let result = { - let context = crate::context::StoreResolutionContext::new(store, &self.project_root); + let context = self.store_context(store); self.resolve_all(&scoped, &context) }; diff --git a/crates/codegraph-resolve/src/snapshot_context.rs b/crates/codegraph-resolve/src/snapshot_context.rs index 892143c8..83c2c3f1 100644 --- a/crates/codegraph-resolve/src/snapshot_context.rs +++ b/crates/codegraph-resolve/src/snapshot_context.rs @@ -230,6 +230,8 @@ struct NodeSnapshot { pub struct SnapshotResolutionContext { snapshot: Arc, caches: Arc, + /// Largest file read for resolution, the project's extraction limit. + max_file_size: u64, /// Per-chunk `implements`/`extends` adjacency for [`Self::get_supertypes`]. /// Empty until the resolver installs a chunk's map (T4); an empty map yields /// the same result the store context gives before any such edge exists. @@ -367,6 +369,7 @@ impl SnapshotResolutionContext { go_modules, }), caches: Arc::new(SnapshotCaches::new()), + max_file_size: codegraph_core::config::DEFAULT_MAX_FILE_SIZE, edges: Arc::new(HashMap::new()), }) } @@ -379,7 +382,7 @@ impl SnapshotResolutionContext { .file_loads .fetch_add(1, Ordering::Relaxed); let full_path = Path::new(&self.snapshot.project_root).join(file_path); - std::fs::read_to_string(full_path).ok().map(Arc::from) + crate::context::read_source_file(&full_path, self.max_file_size).map(Arc::from) }) } @@ -397,10 +400,20 @@ impl SnapshotResolutionContext { Self { snapshot: Arc::clone(&self.snapshot), caches: Arc::clone(&self.caches), + max_file_size: self.max_file_size, edges, } } + /// Read files up to the addressed project's extraction limit + /// (`indexing.max_file_size`) instead of the default one. Set it before the + /// first read: per-chunk clones share one file cache. + #[must_use] + pub fn with_max_file_size(mut self, max_file_size: u64) -> Self { + self.max_file_size = max_file_size; + self + } + #[cfg(test)] fn cache_counts(&self) -> (usize, usize, usize) { ( diff --git a/crates/codegraph-resolve/tests/resolution_file_reads.rs b/crates/codegraph-resolve/tests/resolution_file_reads.rs new file mode 100644 index 00000000..829d013b --- /dev/null +++ b/crates/codegraph-resolve/tests/resolution_file_reads.rs @@ -0,0 +1,232 @@ +//! Resolution reads only what extraction would accept (upstream v1.6.1 #1553): +//! a regular file no larger than the extraction size limit, checked before any +//! byte is read, with the rejection cached for the pass like any other read. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use codegraph_core::config::DEFAULT_MAX_FILE_SIZE; +use codegraph_resolve::{ + ReferenceResolver, ResolutionContext, SnapshotResolutionContext, StoreResolutionContext, +}; +use codegraph_store::Store; + +static NONCE: AtomicU64 = AtomicU64::new(0); + +const LIMIT: usize = DEFAULT_MAX_FILE_SIZE as usize; + +struct Project { + root: PathBuf, + store: Option, +} + +impl Drop for Project { + fn drop(&mut self) { + drop(self.store.take()); + let _ = std::fs::remove_dir_all(&self.root); + } +} + +impl Project { + fn new(tag: &str) -> Self { + let root = std::env::temp_dir().join(format!( + "codegraph-resolution-reads-{tag}-{}-{}", + std::process::id(), + NONCE.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&root).expect("create fixture root"); + let store = Store::open(&root.join("index.db")).expect("open store"); + Self { + root, + store: Some(store), + } + } + + fn store(&self) -> &Store { + self.store.as_ref().expect("store") + } + + fn root_str(&self) -> String { + self.root.to_string_lossy().to_string() + } + + fn write(&self, relative: &str, content: &str) { + let path = self.root.join(relative); + std::fs::create_dir_all(path.parent().expect("parent")).expect("create parent"); + std::fs::write(path, content).expect("write fixture"); + } + + /// A 2 MiB sparse package archive with a gzip header, as `file:` package + /// metadata can point an import at. + fn sparse_archive(&self) -> &'static str { + let relative = "node_modules/example/react_native_openharmony.har"; + let path = self.root.join(relative); + std::fs::create_dir_all(path.parent().expect("parent")).expect("create parent"); + std::fs::write(&path, [0x1f, 0x8b]).expect("write archive header"); + std::fs::OpenOptions::new() + .write(true) + .open(&path) + .and_then(|file| file.set_len(2 * 1024 * 1024)) + .expect("extend archive"); + relative + } +} + +fn contexts(project: &Project) -> (StoreResolutionContext<'_>, SnapshotResolutionContext) { + ( + StoreResolutionContext::new(project.store(), project.root_str()), + SnapshotResolutionContext::from_store(project.store(), project.root_str()) + .expect("snapshot"), + ) +} + +#[test] +fn reads_stop_at_the_extraction_size_limit_in_both_contexts() { + let project = Project::new("limit"); + let at_limit = "a".repeat(LIMIT); + project.write("small.ts", "export const answer = 42;\n"); + project.write("boundary.ts", &at_limit); + project.write("bundle.js", &"a".repeat(LIMIT + 1)); + std::fs::create_dir_all(project.root.join("directory")).expect("mkdir"); + let archive = project.sparse_archive(); + + let (store_context, snapshot) = contexts(&project); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert_eq!( + context.read_file("small.ts").as_deref(), + Some("export const answer = 42;\n") + ); + assert!(context.read_file("boundary.ts") == Some(at_limit.clone())); + for rejected in ["bundle.js", archive, "directory", "missing.ts"] { + assert_eq!( + context.read_file(rejected).map(|content| content.len()), + None, + "{rejected}" + ); + assert!(!context.is_file_readable(rejected), "{rejected}"); + } + } +} + +#[test] +fn a_rejection_is_cached_for_the_pass_and_the_next_pass_rereads() { + let project = Project::new("cache"); + let archive = project.sparse_archive(); + let (store_context, snapshot) = contexts(&project); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert_eq!(context.read_file(archive), None); + } + + project.write(archive, "export const repaired = true;"); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert_eq!(context.read_file(archive), None); + } + let (store_context, snapshot) = contexts(&project); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert_eq!( + context.read_file(archive).as_deref(), + Some("export const repaired = true;") + ); + } +} + +/// A project that raises `max_file_size` indexes larger files, so resolution +/// must keep reading them: the limit follows the project, through every context +/// the resolver builds. +#[test] +fn a_raised_max_file_size_reaches_every_context() { + let project = Project::new("raised"); + let generated = "a".repeat(LIMIT + 1); + project.write("generated.ts", &generated); + let raised = 2 * DEFAULT_MAX_FILE_SIZE; + + let store_context = + StoreResolutionContext::new(project.store(), project.root_str()).with_max_file_size(raised); + let snapshot = SnapshotResolutionContext::from_store(project.store(), project.root_str()) + .expect("snapshot") + .with_max_file_size(raised); + let resolver = ReferenceResolver::new(project.root_str()).with_max_file_size(raised); + let resolver_context = resolver.store_context(project.store()); + for context in [ + &store_context as &dyn ResolutionContext, + &snapshot, + &resolver_context, + ] { + assert!(context.read_file("generated.ts") == Some(generated.clone())); + } + let default_context = ReferenceResolver::new(project.root_str()).store_context(project.store()); + assert_eq!( + default_context + .read_file("generated.ts") + .map(|content| content.len()), + None + ); +} + +/// A FIFO would block a plain read forever; it is not a regular file, so it is +/// rejected before it is opened. +#[cfg(unix)] +#[test] +fn a_fifo_is_rejected_without_blocking() { + let project = Project::new("fifo"); + let fifo = project.root.join("pipe.ts"); + let status = std::process::Command::new("mkfifo") + .arg(&fifo) + .status() + .expect("run mkfifo"); + assert!(status.success(), "mkfifo failed"); + + let root = project.root_str(); + let db = project.root.join("index.db"); + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let store = Store::open(Path::new(&db)).expect("open store"); + let store_read = StoreResolutionContext::new(&store, root.clone()).read_file("pipe.ts"); + let snapshot_read = SnapshotResolutionContext::from_store(&store, root) + .expect("snapshot") + .read_file("pipe.ts"); + let _ = sender.send((store_read, snapshot_read)); + }); + let reads = receiver + .recv_timeout(std::time::Duration::from_secs(10)) + .expect("reading a FIFO blocked"); + assert_eq!(reads, (None, None)); +} + +/// The framework pass reads project files through the same guard, so a file +/// extraction skipped for its size yields no framework output either. +#[test] +fn framework_extraction_skips_a_file_too_large_to_extract() { + let project = Project::new("framework"); + project.write( + "src-tauri/src/main.rs", + "#[tauri::command]\nfn save_config() {}\n", + ); + project.write( + "src-tauri/tauri.conf.json", + "{ \"productName\": \"x\", \"identifier\": \"dev.x\", \"build\": {}, \"app\": {} }\n", + ); + let call = "import { invoke } from '@tauri-apps/api/core';\ninvoke('save_config');\n"; + project.write("src/app.ts", call); + project.write("src/bundle.ts", &format!("{call}{}", " ".repeat(LIMIT))); + + let mut resolver = ReferenceResolver::new(project.root_str()); + { + let context = resolver.store_context(project.store()); + resolver.initialize(&context); + } + assert!(resolver.has_framework_resolvers()); + let collection = resolver.collect_framework_extraction_with( + &["src/app.ts".to_string(), "src/bundle.ts".to_string()], + &codegraph_resolve::framework::FrameworkExtractionContext::without_config( + project.root_str(), + ), + &codegraph_extract::ExtensionOverrides::default(), + ); + let sources = collection + .unresolved_references + .iter() + .map(|reference| reference.file_path.as_str()) + .collect::>(); + assert_eq!(sources, vec!["src/app.ts"]); +} diff --git a/crates/codegraph-watch/src/migrate.rs b/crates/codegraph-watch/src/migrate.rs index 4ddce4cf..4d127cb9 100644 --- a/crates/codegraph-watch/src/migrate.rs +++ b/crates/codegraph-watch/src/migrate.rs @@ -164,12 +164,10 @@ pub(crate) fn migrate_project( } drop(refs); - let mut resolver = ReferenceResolver::new(project_root.to_string_lossy()); + let mut resolver = ReferenceResolver::new(project_root.to_string_lossy()) + .with_max_file_size(options.max_file_size); { - let context = codegraph_resolve::StoreResolutionContext::new( - rebuild.store(), - project_root.to_string_lossy(), - ); + let context = resolver.store_context(rebuild.store()); resolver.initialize(&context); } if resolver.has_framework_resolvers() { diff --git a/crates/codegraph-watch/src/sync.rs b/crates/codegraph-watch/src/sync.rs index 63dd177d..a6721960 100644 --- a/crates/codegraph-watch/src/sync.rs +++ b/crates/codegraph-watch/src/sync.rs @@ -496,12 +496,10 @@ fn sync_paths_with_store( merge_dependent_site(&mut dependent_sites, &mut dependent_fallbacks, affected); } } - let mut resolver = ReferenceResolver::new(project_root.to_string_lossy()); + let mut resolver = ReferenceResolver::new(project_root.to_string_lossy()) + .with_max_file_size(scope.options.max_file_size); { - let context = codegraph_resolve::StoreResolutionContext::new( - store, - project_root.to_string_lossy(), - ); + let context = resolver.store_context(store); resolver.initialize(&context); } // Re-run framework per-file extract for reindexed files whose framework @@ -546,7 +544,7 @@ fn sync_paths_with_store( // healthy index (marker absent) skips this entirely, so an ordinary sync is // byte-for-byte unchanged. if store.is_resolution_incomplete()? { - sweep_orphaned_refs(project_root, store)?; + sweep_orphaned_refs(project_root, store, scope.options.max_file_size)?; } outcome.duration_ms = started.elapsed().as_millis(); @@ -687,11 +685,11 @@ fn merge_dependent_site( /// batched pass re-arms and clears the marker itself, so on success the index is /// no longer flagged partial. Framework per-file extract is re-run first so any /// framework refs an interrupted run never re-injected are present for the sweep. -fn sweep_orphaned_refs(project_root: &Path, store: &mut Store) -> Result<()> { - let mut resolver = ReferenceResolver::new(project_root.to_string_lossy()); +fn sweep_orphaned_refs(project_root: &Path, store: &mut Store, max_file_size: u64) -> Result<()> { + let mut resolver = + ReferenceResolver::new(project_root.to_string_lossy()).with_max_file_size(max_file_size); { - let context = - codegraph_resolve::StoreResolutionContext::new(store, project_root.to_string_lossy()); + let context = resolver.store_context(store); resolver.initialize(&context); } // Framework extraction must not run in the sweep. The marker is set only From fc4ffd0fea0d652c1a6b87879deeb06a73834478 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 09:56:37 +0800 Subject: [PATCH 04/19] fix(extract): bound every source read and skip MPEG-TS clips named .ts Every indexing pass read a file in full before checking its size and decoded it strictly, so one oversize blob was read whole and one byte that is not UTF-8 failed the entire index, leaving it unreadable. One shared reader now serves index, sync, migration, the MCP freshness probe and resolution: it stats before reading and never holds more than the limit plus one byte; a file over the limit is recorded by a size stamp in place of its content hash, so a same-size rewrite is no change and crossing the limit is; an MPEG transport stream named .ts is video, neither indexed nor counted, and a tracked file that becomes one leaves the index; other bytes decode with U+FFFD as upstream does. Extraction version 15 -> 16 (upstream #1910). --- crates/codegraph-cli/src/main.rs | 80 +++-- crates/codegraph-cli/tests/source_reads.rs | 172 +++++++++++ crates/codegraph-core/src/lib.rs | 1 + crates/codegraph-core/src/source_file.rs | 278 ++++++++++++++++++ crates/codegraph-extract/src/engine.rs | 69 ++--- crates/codegraph-extract/src/lib.rs | 3 + crates/codegraph-mcp/src/engine.rs | 77 +++-- crates/codegraph-resolve/src/context.rs | 32 +- crates/codegraph-resolve/src/resolver.rs | 1 + .../codegraph-resolve/src/snapshot_context.rs | 3 +- .../tests/resolution_file_reads.rs | 29 +- crates/codegraph-store/src/index_state.rs | 2 +- crates/codegraph-store/tests/index_state.rs | 2 +- crates/codegraph-watch/src/migrate.rs | 37 +-- crates/codegraph-watch/src/sync.rs | 115 ++++++-- docs/cli.md | 9 + 16 files changed, 721 insertions(+), 189 deletions(-) create mode 100644 crates/codegraph-cli/tests/source_reads.rs create mode 100644 crates/codegraph-core/src/source_file.rs diff --git a/crates/codegraph-cli/src/main.rs b/crates/codegraph-cli/src/main.rs index b5058457..7cc2e24a 100644 --- a/crates/codegraph-cli/src/main.rs +++ b/crates/codegraph-cli/src/main.rs @@ -22,7 +22,7 @@ use codegraph_core::generated_header::detect_generated_file; use codegraph_core::logger::{LoggerConfig, init_logger}; use codegraph_core::node_id::hash_content; use codegraph_core::types::{Edge, ExtractionResult, FileRecord, Language, Node, NodeKind}; -use codegraph_extract::{ExtractOptions, detect_language_with, extract_source_with_observer}; +use codegraph_extract::{ExtractOptions, detect_language_with}; use codegraph_graph::graph::{GodotReach, GraphTraverser, group_definitions}; use codegraph_graph::query::{SearchOptions, search_nodes}; use codegraph_graph::{segment_match, segments}; @@ -5301,6 +5301,7 @@ fn index_project_inner( let mut files_indexed = 0; let mut files_skipped = 0; let mut files_errored = 0; + let mut files_not_source = 0_u64; let mut warnings = Vec::new(); // Stream the graph to the store in capped batches instead of holding the whole @@ -5350,7 +5351,8 @@ fn index_project_inner( diagnostic_run.phase_start("parse_write"); let parse_started = std::time::Instant::now(); - type ParsePayload = (String, FileRecord, ExtractionResult); + // `None` for an MPEG transport stream named `.ts`: video, not source (#1910). + type ParsePayload = Option<(String, FileRecord, ExtractionResult)>; let schedule_tracker = tracker.clone(); let parse_tracker = tracker.clone(); let buffer_tracker = tracker.clone(); @@ -5363,55 +5365,34 @@ fn index_project_inner( let relative = &files[index]; let full = project.join(relative); - parse_tracker.stage(index, "metadata"); - let metadata = match fs::metadata(&full) - .with_context(|| format!("reading metadata for {}", full.display())) - { - Ok(metadata) => metadata, - Err(error) => { - parse_tracker.failed(index, &error); - return Err(error); - } - }; - parse_tracker.stage(index, "read"); - let source = match fs::read_to_string(&full) - .with_context(|| format!("reading source file {}", full.display())) - { - Ok(source) => source, - Err(error) => { - parse_tracker.failed(index, &error); - return Err(error); - } + let (metadata, source) = + match codegraph_extract::read_source_file(&full, relative, options.max_file_size) + .with_context(|| format!("reading source file {}", full.display())) + { + Ok(read) => read, + Err(error) => { + parse_tracker.failed(index, &error); + return Err(error); + } + }; + // A file over the limit was never read: its size stamp stands in for + // its content, in the hash and in the generated-file check. A video + // clip named `.ts` is not source at all, so it is not recorded. + let Some(hash_input) = source.hash_input() else { + return Ok(None); }; parse_tracker.stage(index, "prepare"); let language = detect_language_with(relative, &options.extensions); parse_tracker.file_info(index, metadata.len(), language); - let result = if metadata.len() > options.max_file_size { - ExtractionResult { - nodes: Vec::new(), - edges: Vec::new(), - unresolved_references: Vec::new(), - errors: vec![format!( - "File exceeds max size ({} > {}): {relative}", - metadata.len(), - options.max_file_size - )], - duration_ms: 0, - } - } else { - extract_source_with_observer( - relative, - &source, - None, - &options.extensions, - |stage| parse_tracker.extraction_stage(index, stage), - ) - }; + let result = + codegraph_extract::engine::extraction_of(relative, &source, &options, |stage| { + parse_tracker.extraction_stage(index, stage) + }); let file = FileRecord { path: relative.clone(), - content_hash: hash_content(&source), + content_hash: hash_content(&hash_input), language, size: metadata.len() as i64, modified_at: modified_millis(&metadata), @@ -5422,13 +5403,18 @@ fn index_project_inner( .filter(|node| node.file_path == *relative) .count() as i64, errors: result.errors.clone(), - generated: detect_generated_file(relative, &source), + generated: detect_generated_file(relative, &hash_input), }; parse_tracker.parsed(index, &result); - Ok((relative.clone(), file, result)) + Ok(Some((relative.clone(), file, result))) }, |buffered| buffer_tracker.buffered(buffered), - |index, (_relative, file, mut result)| { + |index, payload| { + let Some((_relative, file, mut result)) = payload else { + files_not_source += 1; + persist_tracker.persisted(index); + return Ok(()); + }; let fatal_errors = file .errors .iter() @@ -5480,7 +5466,7 @@ fn index_project_inner( }), ); - let scan_files = bar.position(); + let scan_files = bar.position() - files_not_source; finish_phase( &bar, &format!("Indexed {} files", format_number(scan_files as i64)), diff --git a/crates/codegraph-cli/tests/source_reads.rs b/crates/codegraph-cli/tests/source_reads.rs new file mode 100644 index 00000000..0080e7f8 --- /dev/null +++ b/crates/codegraph-cli/tests/source_reads.rs @@ -0,0 +1,172 @@ +//! Source files are read as upstream reads them (v1.6.1 #1910): never past the +//! size limit, with a size stamp standing in for an oversize file's content; +//! an MPEG transport stream named `.ts` is video, not TypeScript; and bytes +//! that are not UTF-8 are decoded with replacement characters, so no single +//! file can fail an index. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use codegraph_core::node_id::hash_content; +use codegraph_store::Store; + +struct TestDir(PathBuf); + +impl TestDir { + fn new(tag: &str) -> Self { + let path = std::env::temp_dir().join(format!( + "codegraph-cli-source-reads-{tag}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + fs::create_dir_all(&path).unwrap(); + Self(path) + } + + fn write(&self, relative: &str, bytes: impl AsRef<[u8]>) { + let path = self.0.join(relative); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, bytes).unwrap(); + } +} + +impl Drop for TestDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn run(cwd: &Path, args: &[&str]) -> (String, String, bool) { + let output = Command::new(env!("CARGO_BIN_EXE_codegraph")) + .args(args) + .current_dir(cwd) + .env("CODEGRAPH_NO_DAEMON", "1") + .env("CODEGRAPH_NO_WATCH", "1") + .output() + .expect("run codegraph"); + ( + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + output.status.success(), + ) +} + +/// 64 transport-stream packets: the 0x47 sync byte, then a deterministic +/// pseudo-random payload standing in for compressed video. +fn mpeg_ts_clip() -> Vec { + let mut state = 0x2545_f491_u32; + let mut payload = move || { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state.to_le_bytes()[0] + }; + (0..64) + .flat_map(|_| { + std::iter::once(0x47) + .chain((0..187).map(|_| payload())) + .collect::>() + }) + .collect() +} + +const ONE_MIB: usize = 1024 * 1024; + +fn file_paths(store: &Store) -> Vec { + store + .all_files() + .unwrap() + .into_iter() + .map(|file| file.path) + .collect() +} + +#[test] +fn index_and_sync_treat_video_latin1_and_oversize_files_like_upstream() { + let dir = TestDir::new("index"); + dir.write("src/app.ts", "export function app() { return 1; }\n"); + dir.write("testdata/clip.ts", mpeg_ts_clip()); + dir.write( + "legacy/latin1.c", + b"/* caf\xe9 */\nint legacy_entry(void) { return 0; }\n", + ); + let oversize = format!("export const big = 1;\n{}", " ".repeat(ONE_MIB)); + dir.write("big/bundle.ts", &oversize); + let path = dir.0.to_str().unwrap(); + + let (stdout, stderr, ok) = run(&dir.0, &["init", path]); + assert!(ok, "init failed: {stdout} {stderr}"); + // The oversize file is recorded as skipped; the clip is not counted at all. + assert!(stdout.contains("Indexed 2 files"), "stdout={stdout}"); + assert!(stdout.contains("Skipped 1 files"), "stdout={stdout}"); + + let db = dir.0.join(".codegraph").join("codegraph.db"); + { + let store = Store::open(&db).unwrap(); + assert_eq!( + file_paths(&store), + vec!["big/bundle.ts", "legacy/latin1.c", "src/app.ts"] + ); + let bundle = store.file_by_path("big/bundle.ts").unwrap().unwrap(); + assert_eq!( + bundle.content_hash, + hash_content(&format!("codegraph:oversize:{}", oversize.len())) + ); + assert!( + store + .nodes_by_name("legacy_entry") + .unwrap() + .iter() + .any(|node| node.file_path == "legacy/latin1.c") + ); + } + + let (stdout, stderr, ok) = run(&dir.0, &["status", path, "--json"]); + assert!(ok, "status failed: {stdout} {stderr}"); + let status: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + assert_eq!( + status["pendingChanges"]["addedPaths"], + serde_json::json!([]), + "an untracked clip is not pending source: {stdout}" + ); + + // A same-size rewrite of an oversize file is not a change; a tracked `.ts` + // that became a clip is removed like a deletion. + let rewritten = format!("export const BIG = 2;\n{}", " ".repeat(ONE_MIB)); + assert_eq!(rewritten.len(), oversize.len()); + dir.write("big/bundle.ts", &rewritten); + dir.write("src/app.ts", mpeg_ts_clip()); + let (stdout, stderr, ok) = run(&dir.0, &["sync", path]); + assert!(ok, "sync failed: {stdout} {stderr}"); + { + let store = Store::open(&db).unwrap(); + assert_eq!(file_paths(&store), vec!["big/bundle.ts", "legacy/latin1.c"]); + } + let (stdout, stderr, ok) = run(&dir.0, &["status", path, "--json"]); + assert!(ok, "status failed: {stdout} {stderr}"); + let status: serde_json::Value = serde_json::from_str(&stdout).unwrap(); + for kind in ["addedPaths", "modifiedPaths", "removedPaths"] { + assert_eq!( + status["pendingChanges"][kind], + serde_json::json!([]), + "{kind}: {stdout}" + ); + } + + // Crossing the limit is a change in either direction. + dir.write("big/bundle.ts", "export const small = 3;\n"); + let (stdout, stderr, ok) = run(&dir.0, &["sync", path]); + assert!(ok, "sync failed: {stdout} {stderr}"); + let store = Store::open(&db).unwrap(); + assert!( + store + .nodes_by_name("small") + .unwrap() + .iter() + .any(|node| node.file_path == "big/bundle.ts") + ); +} diff --git a/crates/codegraph-core/src/lib.rs b/crates/codegraph-core/src/lib.rs index cef3be8b..d4beed05 100644 --- a/crates/codegraph-core/src/lib.rs +++ b/crates/codegraph-core/src/lib.rs @@ -8,6 +8,7 @@ pub mod generated_header; pub mod index_paths; pub mod logger; pub mod node_id; +pub mod source_file; pub mod traits; pub mod types; diff --git a/crates/codegraph-core/src/source_file.rs b/crates/codegraph-core/src/source_file.rs new file mode 100644 index 00000000..87c15b49 --- /dev/null +++ b/crates/codegraph-core/src/source_file.rs @@ -0,0 +1,278 @@ +//! Reading a project source file by path, the way every indexing pass does +//! (upstream v1.6.1 `file-limits.ts` and `grammars.ts`, #1910): never more than +//! the size limit plus one byte, with a size stamp standing in for a file over +//! it, and an MPEG transport stream named `.ts` recognised as video rather than +//! TypeScript before anything parses it. + +use std::borrow::Cow; +use std::fs; +use std::io::{self, Read}; +use std::path::Path; + +/// MPEG transport stream: fixed 188-byte packets, each opening with 0x47. +const MPEG_TS_PACKET_SIZE: usize = 188; +const MPEG_TS_SYNC_BYTE: u8 = 0x47; +/// Consecutive packets whose sync byte must line up before a file counts as +/// video — 3 KB of head. A shorter stream is cheap to parse anyway. +const MPEG_TS_MIN_PACKETS: usize = 16; +/// How much of a file's head [`is_mpeg_transport_stream`] looks at. +const MPEG_TS_SNIFF_BYTES: usize = MPEG_TS_PACKET_SIZE * MPEG_TS_MIN_PACKETS; + +/// A source file as an indexing pass sees it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SourceText { + /// The file's text, within the limit. Bytes that are not UTF-8 become + /// U+FFFD, as upstream's `toString('utf8')` decodes them, so a Latin-1 or + /// binary file is indexed like any other instead of failing the pass. + Text(String), + /// A file over the limit, never read: its size in bytes stands in for it. + Oversize(u64), + /// An MPEG transport stream named `.ts`: video, not source. + MpegTransportStream, +} + +impl SourceText { + /// What the index hashes for this file: its text within the limit, its + /// size stamp over it, and nothing for a file that is not source. + pub fn hash_input(&self) -> Option> { + match self { + Self::Text(text) => Some(Cow::Borrowed(text)), + Self::Oversize(size) => Some(Cow::Owned(oversize_stamp(*size))), + Self::MpegTransportStream => None, + } + } +} + +/// What stands in for the content of a file over the size limit. Such a file +/// is never parsed, so the stamp is a function of its size alone: a same-size +/// rewrite is not a change (nothing is indexed from it), while crossing the +/// limit in either direction is. +pub fn oversize_stamp(size: u64) -> String { + format!("codegraph:oversize:{size}") +} + +/// Read the source file at `path`, named `relative` in the project, without +/// ever holding more than `max_bytes` plus one byte. The size is checked by +/// `stat` before the file is opened, again on the open handle, and once more +/// after a read that itself stops one byte past the limit, so a file growing +/// meanwhile is still stamped rather than decoded. Returns the metadata the +/// file's record is built from. A path that is not a regular file is an error. +pub fn read_source_file( + path: &Path, + relative: &str, + max_bytes: u64, +) -> io::Result<(fs::Metadata, SourceText)> { + let initial = regular_file_metadata(fs::metadata(path)?)?; + if initial.len() > max_bytes { + let size = initial.len(); + return Ok((initial, SourceText::Oversize(size))); + } + let mut file = fs::File::open(path)?; + let opened = regular_file_metadata(file.metadata()?)?; + if opened.len() > max_bytes { + let size = opened.len(); + return Ok((opened, SourceText::Oversize(size))); + } + let mut bytes = Vec::with_capacity(usize::try_from(opened.len()).unwrap_or(0)); + (&mut file) + .take(max_bytes.saturating_add(1)) + .read_to_end(&mut bytes)?; + let read = file.metadata()?; + let read_len = bytes.len() as u64; + if read_len > max_bytes || read.len() > max_bytes { + let size = read.len().max(read_len); + return Ok((read, SourceText::Oversize(size))); + } + if has_mpeg_ts_extension(relative) && is_mpeg_transport_stream(&bytes) { + return Ok((read, SourceText::MpegTransportStream)); + } + let text = String::from_utf8(bytes) + .unwrap_or_else(|error| String::from_utf8_lossy(error.as_bytes()).into_owned()); + Ok((read, SourceText::Text(text))) +} + +/// Whether [`read_source_file`] would see source at `path`: everything except +/// an MPEG transport stream named `.ts` within the limit (a file over it is +/// still recorded, by its size stamp). Reads at most the head the check needs, +/// and only for a `.ts` name, so a pending-change scan stays cheap. +pub fn is_source_file(path: &Path, relative: &str, max_bytes: u64) -> io::Result { + if !has_mpeg_ts_extension(relative) { + return Ok(true); + } + let metadata = regular_file_metadata(fs::metadata(path)?)?; + if metadata.len() > max_bytes { + return Ok(true); + } + let mut head = Vec::with_capacity(MPEG_TS_SNIFF_BYTES); + fs::File::open(path)? + .take(MPEG_TS_SNIFF_BYTES as u64) + .read_to_end(&mut head)?; + Ok(!is_mpeg_transport_stream(&head)) +} + +fn regular_file_metadata(metadata: fs::Metadata) -> io::Result { + if metadata.is_file() { + Ok(metadata) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidInput, + "source path is not a regular file", + )) + } +} + +/// Whether `relative` carries the one extension MPEG-TS shares with a language. +fn has_mpeg_ts_extension(relative: &str) -> bool { + let bytes = relative.as_bytes(); + bytes.len() > 3 && bytes[bytes.len() - 3..].eq_ignore_ascii_case(b".ts") +} + +/// Whether a file starting with `head` is an MPEG transport stream, the other +/// thing a `.ts` file can be: tree-sitter takes tens of seconds on a clip for +/// zero symbols. Both conditions are required: +/// 1. the sync byte 0x47 opens each of the first [`MPEG_TS_MIN_PACKETS`] +/// 188-byte packets, as it opens every packet of a stream; +/// 2. the head is binary: at least 1/64 of it is control bytes other than +/// the whitespace ones, as any compressed payload carries. +/// +/// 0x47 is `G`, so source could satisfy (1) by putting one at every stride; +/// no source carries the dozens of control bytes (2) asks for. +fn is_mpeg_transport_stream(head: &[u8]) -> bool { + let head = &head[..head.len().min(MPEG_TS_SNIFF_BYTES)]; + let last_sync = MPEG_TS_PACKET_SIZE * (MPEG_TS_MIN_PACKETS - 1); + if head.len() <= last_sync { + return false; + } + if (0..=last_sync) + .step_by(MPEG_TS_PACKET_SIZE) + .any(|offset| head[offset] != MPEG_TS_SYNC_BYTE) + { + return false; + } + let control = head + .iter() + .filter(|&&byte| byte < 0x20 && !(0x09..=0x0d).contains(&byte)) + .count(); + control * 64 >= head.len() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_dir(tag: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!( + "codegraph-source-file-{tag}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + fs::create_dir_all(&dir).expect("create temp dir"); + dir + } + + /// `packets` transport-stream packets with a deterministic binary payload. + fn clip(packets: usize) -> Vec { + let mut state = 0x9e37_79b9_u32; + let mut payload = move || { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + state.to_le_bytes()[0] + }; + (0..packets) + .flat_map(|_| { + std::iter::once(MPEG_TS_SYNC_BYTE) + .chain((0..187).map(|_| payload())) + .collect::>() + }) + .collect() + } + + #[test] + fn a_transport_stream_needs_sixteen_aligned_packets_and_a_binary_head() { + assert!(is_mpeg_transport_stream(&clip(16))); + assert!(is_mpeg_transport_stream(&clip(64))); + assert!(!is_mpeg_transport_stream(&clip(15)), "too short to judge"); + let mut misaligned = clip(64); + misaligned[188 * 7] = b'x'; + assert!(!is_mpeg_transport_stream(&misaligned)); + + // TypeScript with a `G` at every stride and one raw NUL is still text. + let mut source = Vec::new(); + for _ in 0..16 { + let mut line = b"G".to_vec(); + line.extend(std::iter::repeat_n(b'a', 186)); + line.push(b'\n'); + source.extend(line); + } + source[100] = 0; + assert!(!is_mpeg_transport_stream(&source)); + } + + #[test] + fn the_extension_check_is_the_last_three_characters_case_blind() { + assert!(has_mpeg_ts_extension("clips/a.ts")); + assert!(has_mpeg_ts_extension("CLIP.TS")); + assert!(has_mpeg_ts_extension("types.d.ts")); + assert!(!has_mpeg_ts_extension(".ts")); + assert!(!has_mpeg_ts_extension("a.tsx")); + assert!(!has_mpeg_ts_extension("a.mts")); + } + + #[test] + fn reads_text_lossily_stamps_oversize_and_recognises_clips() { + let dir = temp_dir("read"); + fs::write(dir.join("app.ts"), "export const a = 1;\n").unwrap(); + fs::write(dir.join("latin1.c"), b"/* caf\xe9 */\n").unwrap(); + fs::write(dir.join("clip.ts"), clip(64)).unwrap(); + fs::write(dir.join("clip.bin.c"), clip(64)).unwrap(); + fs::write(dir.join("big.ts"), "a".repeat(65)).unwrap(); + fs::write(dir.join("edge.ts"), "a".repeat(64)).unwrap(); + fs::create_dir(dir.join("folder.ts")).unwrap(); + + let read = |name: &str| read_source_file(&dir.join(name), name, 64).map(|(_, text)| text); + assert_eq!( + read("app.ts").unwrap(), + SourceText::Text("export const a = 1;\n".to_string()) + ); + assert_eq!( + read("latin1.c").unwrap(), + SourceText::Text("/* caf\u{fffd} */\n".to_string()) + ); + assert_eq!(read("big.ts").unwrap(), SourceText::Oversize(65)); + assert_eq!(read("edge.ts").unwrap(), SourceText::Text("a".repeat(64))); + assert!(read("folder.ts").is_err()); + assert!(read("missing.ts").is_err()); + + let read_large = + |name: &str| read_source_file(&dir.join(name), name, 1024 * 1024).map(|(_, text)| text); + assert_eq!( + read_large("clip.ts").unwrap(), + SourceText::MpegTransportStream + ); + // Only a `.ts` name can be a transport stream; other binaries decode. + assert!(matches!( + read_large("clip.bin.c").unwrap(), + SourceText::Text(_) + )); + + assert_eq!( + SourceText::Oversize(65).hash_input().as_deref(), + Some("codegraph:oversize:65") + ); + assert_eq!(SourceText::MpegTransportStream.hash_input(), None); + + let is_source = |name: &str, max: u64| is_source_file(&dir.join(name), name, max).unwrap(); + assert!(!is_source("clip.ts", 1024 * 1024)); + assert!( + is_source("clip.ts", 64), + "an oversize clip is recorded by its stamp" + ); + assert!(is_source("clip.bin.c", 1024 * 1024)); + assert!(is_source("app.ts", 1024 * 1024)); + fs::remove_dir_all(&dir).ok(); + } +} diff --git a/crates/codegraph-extract/src/engine.rs b/crates/codegraph-extract/src/engine.rs index 4625a328..a40b808b 100644 --- a/crates/codegraph-extract/src/engine.rs +++ b/crates/codegraph-extract/src/engine.rs @@ -6,7 +6,6 @@ use anyhow::{Context, Result}; use codegraph_core::config::{Config, IndexingConfig}; -use codegraph_core::node_id::hash_content; use codegraph_core::types::{ExtractionResult, Language}; use rayon::prelude::*; use regex::Regex; @@ -20,6 +19,7 @@ use tree_sitter::Parser; use crate::ext_config::ExtensionOverrides; use crate::lang::{cpp_code_mask, spec_for_language}; use crate::walker::TreeSitterWalker; +use codegraph_core::source_file::{SourceText, read_source_file}; /// Stable diagnostic fragment used when a supported grammar reports parse /// errors and extraction collapses to only the synthetic file node. @@ -409,25 +409,33 @@ pub fn extract_file_with_options_observer( let root = root.as_ref(); let relative_path = normalize_path(relative_path.as_ref()); let full_path = root.join(&relative_path); - let metadata = fs::metadata(&full_path) - .with_context(|| format!("stat source file {}", full_path.display()))?; - if metadata.len() > options.max_file_size { - return Ok(size_skip_result( - &relative_path, - metadata.len(), - options.max_file_size, - )); - } - let source = fs::read_to_string(&full_path) + let (_, source) = read_source_file(&full_path, &relative_path, options.max_file_size) .with_context(|| format!("read source file {}", full_path.display()))?; - let _content_hash = hash_content(&source); - Ok(extract_source_with_observer( - &relative_path, - &source, - None, - &options.extensions, - observer, - )) + Ok(extraction_of(&relative_path, &source, options, observer)) +} + +/// The extraction result for one file as [`read_source_file`] saw it: a parse +/// of its text, the size-skip result over the limit, and nothing at all for an +/// MPEG transport stream, which is not source. +pub fn extraction_of( + relative_path: &str, + source: &SourceText, + options: &ExtractOptions, + observer: impl FnMut(ExtractionStage), +) -> ExtractionResult { + match source { + SourceText::Text(text) => { + extract_source_with_observer(relative_path, text, None, &options.extensions, observer) + } + SourceText::Oversize(size) => size_skip_result(relative_path, *size, options.max_file_size), + SourceText::MpegTransportStream => ExtractionResult { + nodes: Vec::new(), + edges: Vec::new(), + unresolved_references: Vec::new(), + errors: Vec::new(), + duration_ms: 0, + }, + } } pub fn extract_project( @@ -438,24 +446,9 @@ pub fn extract_project( let files = scan_project(root, options)?; let parse = |relative: &String| -> Result { let full = root.join(relative); - let metadata = - fs::metadata(&full).with_context(|| format!("stat source file {}", full.display()))?; - if metadata.len() > options.max_file_size { - return Ok(size_skip_result( - relative, - metadata.len(), - options.max_file_size, - )); - } - let source = fs::read_to_string(&full) + let (_, source) = read_source_file(&full, relative, options.max_file_size) .with_context(|| format!("read source file {}", full.display()))?; - let _content_hash = hash_content(&source); - Ok(extract_source_with( - relative, - &source, - None, - &options.extensions, - )) + Ok(extraction_of(relative, &source, options, |_| {})) }; let mut results = if options.parallel { @@ -619,7 +612,9 @@ fn merge_results(results: &mut [ExtractionResult]) -> Result { Ok(merged) } -fn size_skip_result(file_path: &str, size: u64, max: u64) -> ExtractionResult { +/// The result recorded for a file over the size limit: no symbols, and the +/// error every indexing path reports for it. +pub fn size_skip_result(file_path: &str, size: u64, max: u64) -> ExtractionResult { ExtractionResult { nodes: Vec::new(), edges: Vec::new(), diff --git a/crates/codegraph-extract/src/lib.rs b/crates/codegraph-extract/src/lib.rs index d1b2cfb4..4472355d 100644 --- a/crates/codegraph-extract/src/lib.rs +++ b/crates/codegraph-extract/src/lib.rs @@ -16,6 +16,9 @@ pub mod lang; pub mod spec; pub mod walker; +pub use codegraph_core::source_file::{ + SourceText, is_source_file, oversize_stamp, read_source_file, +}; pub use engine::{ ExtractOptions, ExtractionStage, PARSE_COLLAPSE_WARNING, RootGitignore, detect_language, detect_language_with, extract_file, extract_file_with_options, diff --git a/crates/codegraph-mcp/src/engine.rs b/crates/codegraph-mcp/src/engine.rs index 5988b96f..2bd03138 100644 --- a/crates/codegraph-mcp/src/engine.rs +++ b/crates/codegraph-mcp/src/engine.rs @@ -173,9 +173,10 @@ impl CodeGraphEngine { /// Freshness is fail-closed: it is proven only after successfully loading the /// file record and matching either size + millisecond mtime (the sync fast /// path, with no hash) or, after a stat mismatch, the indexed sha256 content - /// hash. A missing/unreadable record, read failure, or unhashable oversized - /// stat mismatch remains possibly drifted. The full probe is memoized for the - /// current handler. + /// hash — of the file's text, or of its size stamp for a file over the limit, + /// which is compared without being read, exactly as indexing hashed it. A + /// missing/unreadable record or a read failure remains possibly drifted. The + /// full probe is memoized for the current handler. fn project_source(&self, file_path: &str) -> SourceProbe { if let Some(cached) = self.source_probes.borrow().get(file_path).cloned() { return cached; @@ -193,34 +194,31 @@ impl CodeGraphEngine { && metadata_modified_millis(&metadata) .is_some_and(|mtime| file.modified_at == mtime) }); - if metadata.len() > self.config.indexing.max_file_size { - SourceProbe { - content: None, - freshness: if stat_matches { + match codegraph_core::source_file::read_source_file( + &abs, + file_path, + self.config.indexing.max_file_size, + ) { + Ok((_, source)) => { + let freshness = if stat_matches + || source.hash_input().is_some_and(|input| { + stored + .as_ref() + .is_some_and(|file| file.content_hash == hash_content(&input)) + }) { SourceFreshness::ProvenFresh } else { SourceFreshness::PossiblyDrifted - }, - } - } else { - match fs::read_to_string(&abs) { - Ok(content) => { - let freshness = if stat_matches - || stored - .as_ref() - .is_some_and(|file| file.content_hash == hash_content(&content)) - { - SourceFreshness::ProvenFresh - } else { - SourceFreshness::PossiblyDrifted - }; - SourceProbe { - content: Some(Arc::::from(content)), - freshness, + }; + let content = match source { + codegraph_core::source_file::SourceText::Text(text) => { + Some(Arc::::from(text)) } - } - Err(_) => SourceProbe::default(), + _ => None, + }; + SourceProbe { content, freshness } } + Err(_) => SourceProbe::default(), } } _ => SourceProbe::default(), @@ -5461,6 +5459,33 @@ mod tests { ); } + /// Indexing records a file over the limit by its size stamp, so the probe + /// compares it by that stamp, unread: a same-size rewrite is no drift + /// (upstream #1910), as it is no change to sync. + #[test] + fn project_source_compares_an_oversized_file_by_its_size_stamp() { + let mut engine = test_engine(); + let file = "src/oversized.rs"; + put_indexed_source(&engine, file, "fn oversized() {}\n", Language::Rust, 1); + let metadata = std::fs::metadata(engine.project_root.join(file)).unwrap(); + let mut stored = engine.store.file_by_path(file).unwrap().unwrap(); + stored.modified_at += 1; + stored.content_hash = + hash_content(&codegraph_core::source_file::oversize_stamp(metadata.len())); + put_file(&engine, &stored); + Arc::get_mut(&mut engine.config) + .unwrap() + .indexing + .max_file_size = metadata.len() - 1; + + let probe = engine.project_source(file); + assert_eq!(probe.freshness, SourceFreshness::ProvenFresh); + assert!( + probe.content.is_none(), + "oversized source bytes stay unread" + ); + } + #[test] fn ext_node_staleness_banner_is_only_emitted_for_drifted_files() { let mut engine = test_engine(); diff --git a/crates/codegraph-resolve/src/context.rs b/crates/codegraph-resolve/src/context.rs index ffff3a78..06e4c809 100644 --- a/crates/codegraph-resolve/src/context.rs +++ b/crates/codegraph-resolve/src/context.rs @@ -272,7 +272,7 @@ impl ResolutionContext for StoreResolutionContext<'_> { } let full_path = Path::new(&self.project_root).join(file_path); let content: Option> = - read_source_file(&full_path, self.max_file_size).map(Arc::from); + read_source_file(&full_path, file_path, self.max_file_size).map(Arc::from); c.file_cache.set(file_path.to_string(), content.clone()); content } @@ -523,24 +523,20 @@ fn load_go_module(project_root: &str) -> Option { load_go_module_in(project_root, "") } -/// Read `path` for resolution only if extraction would accept it: a regular -/// file no larger than `max_bytes` (upstream #1553). The `stat` comes first, so -/// a dependency archive an import resolves to is never decoded and a FIFO is -/// never opened; the read is bounded as well, in case the file grows meanwhile. -pub(crate) fn read_source_file(path: &Path, max_bytes: u64) -> Option { - use std::io::Read; - - let metadata = std::fs::metadata(path).ok()?; - if !metadata.is_file() || metadata.len() > max_bytes { - return None; - } - let mut content = String::new(); - std::fs::File::open(path) +/// Read `path` (`relative` in the project) for resolution only if extraction +/// would index it: a regular file no larger than `max_bytes` that is not a video +/// clip named `.ts` (upstream #1553, #1910). The shared bounded reader stats it +/// first, so a dependency archive an import resolves to is never decoded and a +/// FIFO is never opened, and it decodes the text exactly as extraction does. +pub(crate) fn read_source_file(path: &Path, relative: &str, max_bytes: u64) -> Option { + match codegraph_core::source_file::read_source_file(path, relative, max_bytes) .ok()? - .take(max_bytes.saturating_add(1)) - .read_to_string(&mut content) - .ok()?; - (content.len() as u64 <= max_bytes).then_some(content) + .1 + { + codegraph_core::source_file::SourceText::Text(text) => Some(text), + codegraph_core::source_file::SourceText::Oversize(_) + | codegraph_core::source_file::SourceText::MpegTransportStream => None, + } } /// Read the `module` line of `//go.mod`. diff --git a/crates/codegraph-resolve/src/resolver.rs b/crates/codegraph-resolve/src/resolver.rs index 9e5a05dc..57eed402 100644 --- a/crates/codegraph-resolve/src/resolver.rs +++ b/crates/codegraph-resolve/src/resolver.rs @@ -1439,6 +1439,7 @@ impl ReferenceResolver { |relative| { crate::context::read_source_file( &std::path::Path::new(&self.project_root).join(relative), + relative, self.max_file_size, ) }, diff --git a/crates/codegraph-resolve/src/snapshot_context.rs b/crates/codegraph-resolve/src/snapshot_context.rs index 83c2c3f1..bf297c3b 100644 --- a/crates/codegraph-resolve/src/snapshot_context.rs +++ b/crates/codegraph-resolve/src/snapshot_context.rs @@ -382,7 +382,8 @@ impl SnapshotResolutionContext { .file_loads .fetch_add(1, Ordering::Relaxed); let full_path = Path::new(&self.snapshot.project_root).join(file_path); - crate::context::read_source_file(&full_path, self.max_file_size).map(Arc::from) + crate::context::read_source_file(&full_path, file_path, self.max_file_size) + .map(Arc::from) }) } diff --git a/crates/codegraph-resolve/tests/resolution_file_reads.rs b/crates/codegraph-resolve/tests/resolution_file_reads.rs index 829d013b..9b064684 100644 --- a/crates/codegraph-resolve/tests/resolution_file_reads.rs +++ b/crates/codegraph-resolve/tests/resolution_file_reads.rs @@ -50,7 +50,7 @@ impl Project { self.root.to_string_lossy().to_string() } - fn write(&self, relative: &str, content: &str) { + fn write(&self, relative: &str, content: impl AsRef<[u8]>) { let path = self.root.join(relative); std::fs::create_dir_all(path.parent().expect("parent")).expect("create parent"); std::fs::write(path, content).expect("write fixture"); @@ -86,7 +86,7 @@ fn reads_stop_at_the_extraction_size_limit_in_both_contexts() { let at_limit = "a".repeat(LIMIT); project.write("small.ts", "export const answer = 42;\n"); project.write("boundary.ts", &at_limit); - project.write("bundle.js", &"a".repeat(LIMIT + 1)); + project.write("bundle.js", "a".repeat(LIMIT + 1)); std::fs::create_dir_all(project.root.join("directory")).expect("mkdir"); let archive = project.sparse_archive(); @@ -108,6 +108,29 @@ fn reads_stop_at_the_extraction_size_limit_in_both_contexts() { } } +/// Resolution decodes a file exactly as extraction does (upstream #1910): bytes +/// that are not UTF-8 become U+FFFD, and a video clip named `.ts` is not +/// source, so neither context reads it. +#[test] +fn reads_decode_like_extraction_and_skip_video_clips() { + let project = Project::new("decode"); + project.write("legacy.c", b"/* caf\xe9 */\n"); + let mut clip = Vec::new(); + for packet in 0..64_u32 { + clip.push(0x47); + clip.extend((0..187_u32).map(|byte| ((packet * 31 + byte * 7) % 32) as u8)); + } + project.write("clip.ts", &clip); + let (store_context, snapshot) = contexts(&project); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert_eq!( + context.read_file("legacy.c").as_deref(), + Some("/* caf\u{fffd} */\n") + ); + assert_eq!(context.read_file("clip.ts"), None); + } +} + #[test] fn a_rejection_is_cached_for_the_pass_and_the_next_pass_rereads() { let project = Project::new("cache"); @@ -208,7 +231,7 @@ fn framework_extraction_skips_a_file_too_large_to_extract() { ); let call = "import { invoke } from '@tauri-apps/api/core';\ninvoke('save_config');\n"; project.write("src/app.ts", call); - project.write("src/bundle.ts", &format!("{call}{}", " ".repeat(LIMIT))); + project.write("src/bundle.ts", format!("{call}{}", " ".repeat(LIMIT))); let mut resolver = ReferenceResolver::new(project.root_str()); { diff --git a/crates/codegraph-store/src/index_state.rs b/crates/codegraph-store/src/index_state.rs index 08aa794a..089357a2 100644 --- a/crates/codegraph-store/src/index_state.rs +++ b/crates/codegraph-store/src/index_state.rs @@ -115,7 +115,7 @@ pub const CURRENT_STORAGE_PROTOCOL: u64 = 2; /// The extraction-pipeline version this binary produces. This is the single /// source of truth: no other crate may define its own copy. -pub const CURRENT_EXTRACTION_VERSION: u64 = 15; +pub const CURRENT_EXTRACTION_VERSION: u64 = 16; /// The `project_metadata` key under which a built index records the extraction /// version it was produced with. Single source of truth for the key spelling. diff --git a/crates/codegraph-store/tests/index_state.rs b/crates/codegraph-store/tests/index_state.rs index df760d91..dba33b6f 100644 --- a/crates/codegraph-store/tests/index_state.rs +++ b/crates/codegraph-store/tests/index_state.rs @@ -119,7 +119,7 @@ fn assert_corrupt( fn index_state_constants_and_canonical_payload_are_exact() { /// The extraction version the immediately preceding release wrote. Read from /// `index_state.rs` at implementation time; see plan decision C1 step 0. - const PREVIOUS_EXTRACTION_VERSION: u64 = 14; + const PREVIOUS_EXTRACTION_VERSION: u64 = 15; assert_eq!(CURRENT_STORAGE_PROTOCOL, 2); assert_eq!(CURRENT_EXTRACTION_VERSION, PREVIOUS_EXTRACTION_VERSION + 1); diff --git a/crates/codegraph-watch/src/migrate.rs b/crates/codegraph-watch/src/migrate.rs index 4d127cb9..1299ed1f 100644 --- a/crates/codegraph-watch/src/migrate.rs +++ b/crates/codegraph-watch/src/migrate.rs @@ -37,7 +37,7 @@ use codegraph_core::IndexPaths; use codegraph_core::generated_header::detect_generated_file; use codegraph_core::node_id::hash_content; use codegraph_core::types::{Edge, FileRecord, Node, UnresolvedRef}; -use codegraph_extract::{detect_language_with, extract_source_with}; +use codegraph_extract::detect_language_with; use codegraph_resolve::ReferenceResolver; use codegraph_store::StoreWriteAuthorization; @@ -93,30 +93,21 @@ pub(crate) fn migrate_project( for (done, relative) in candidates.iter().enumerate() { let full = project_root.join(relative); - // One metadata + one source read per file, mirroring the CLI producer so - // the oversized-file skip message is byte-identical. - let metadata = std::fs::metadata(&full) - .with_context(|| format!("reading metadata for {}", full.display()))?; - let source = std::fs::read_to_string(&full) - .with_context(|| format!("reading source file {}", full.display()))?; - let mut result = if metadata.len() > options.max_file_size { - codegraph_core::types::ExtractionResult { - nodes: Vec::new(), - edges: Vec::new(), - unresolved_references: Vec::new(), - errors: vec![format!( - "File exceeds max size ({} > {}): {relative}", - metadata.len(), - options.max_file_size - )], - duration_ms: 0, - } - } else { - extract_source_with(relative, &source, None, &options.extensions) + // One bounded read per file through the reader the CLI producer uses, so + // the oversized-file skip message and the size stamp are byte-identical. + let (metadata, source) = + codegraph_extract::read_source_file(&full, relative, options.max_file_size) + .with_context(|| format!("reading source file {}", full.display()))?; + // A video clip named `.ts` is not source, so it gets no record (#1910). + let Some(hash_input) = source.hash_input() else { + on_progress(done + 1, total); + continue; }; + let mut result = + codegraph_extract::engine::extraction_of(relative, &source, options, |_| {}); let file = FileRecord { path: relative.clone(), - content_hash: hash_content(&source), + content_hash: hash_content(&hash_input), language: detect_language_with(relative, &options.extensions), size: metadata.len() as i64, modified_at: modified_millis(&metadata), @@ -127,7 +118,7 @@ pub(crate) fn migrate_project( .filter(|node| node.file_path == *relative) .count() as i64, errors: result.errors.clone(), - generated: detect_generated_file(relative, &source), + generated: detect_generated_file(relative, &hash_input), }; rebuild.store().upsert_file(&file)?; diff --git a/crates/codegraph-watch/src/sync.rs b/crates/codegraph-watch/src/sync.rs index a6721960..c20ab82a 100644 --- a/crates/codegraph-watch/src/sync.rs +++ b/crates/codegraph-watch/src/sync.rs @@ -13,7 +13,8 @@ use codegraph_core::generated_header::detect_generated_file; use codegraph_core::node_id::hash_content; use codegraph_core::types::FileRecord; use codegraph_extract::{ - ExtensionOverrides, ExtractOptions, detect_language_with, extract_file_with_options, + ExtensionOverrides, ExtractOptions, SourceText, detect_language_with, + extract_file_with_options, is_source_file, read_source_file, }; use codegraph_resolve::ReferenceResolver; use codegraph_resolve::framework::FrameworkExtractionContext; @@ -148,22 +149,33 @@ pub fn pending_project_changes( .collect::>(); let on_disk_set = on_disk.iter().cloned().collect::>(); + let max_file_size = scope.options.max_file_size; let mut pending = PendingChanges::default(); for relative in on_disk { + let full = project_root.join(&relative); let Some(stored) = tracked.get(&relative) else { - pending.added.push(relative); + // An untracked video clip named `.ts` is not source sync would add. + if is_source_file(&full, &relative, max_file_size) + .with_context(|| format!("read pending source {}", full.display()))? + { + pending.added.push(relative); + } continue; }; - let full = project_root.join(&relative); let metadata = fs::metadata(&full).with_context(|| format!("stat {}", full.display()))?; if stored.size == metadata.len() as i64 && stored.modified_at == modified_millis(&metadata) { continue; } - let source = fs::read_to_string(&full) + let (_, source) = read_source_file(&full, &relative, max_file_size) .with_context(|| format!("read pending source {}", full.display()))?; - if stored.content_hash != hash_content(&source) { - pending.modified.push(relative); + match source.hash_input() { + // A tracked file that became a video clip leaves the index. + None => pending.removed.push(relative), + Some(hash_input) if stored.content_hash != hash_content(&hash_input) => { + pending.modified.push(relative); + } + Some(_) => {} } } for relative in tracked.keys() { @@ -734,21 +746,14 @@ fn sync_one( let metadata = match (!force_absent).then(|| fs::metadata(&full)) { Some(Ok(metadata)) if metadata.is_file() => metadata, _ => { - let was_tracked = store.file_by_path(relative)?.is_some(); - if !was_tracked { - outcome.files_ignored += 1; - return Ok(false); - } - for dependent in store.reference_sites_dependent_on_file(relative)? { - merge_dependent_site(dependent_sites, dependent_fallbacks, dependent); - } - for name in node_names_in_file(store, relative)? { - changed_names.insert(name); - } - delete_unresolved_refs_by_file(store, relative)?; - store.delete_file_record(relative)?; - outcome.files_removed += 1; - return Ok(true); + return remove_tracked_file( + store, + relative, + outcome, + dependent_sites, + dependent_fallbacks, + changed_names, + ); } }; @@ -769,8 +774,21 @@ fn sync_one( return Ok(false); } - let source = fs::read_to_string(&full).with_context(|| format!("read {}", full.display()))?; - let content_hash = hash_content(&source); + let (metadata, source) = read_source_file(&full, relative, scope.options.max_file_size) + .with_context(|| format!("read {}", full.display()))?; + // A file that became a video clip named `.ts` is no longer source: it leaves + // the index like a deletion, and an untracked one is ignored (#1910). + let Some(hash_input) = source.hash_input() else { + return remove_tracked_file( + store, + relative, + outcome, + dependent_sites, + dependent_fallbacks, + changed_names, + ); + }; + let content_hash = hash_content(&hash_input); // Authoritative content gate, port of the upstream hash gate in // `upstream extraction/index.ts:1326-1337,1465-1483`. if stored.is_some_and(|file| file.content_hash == content_hash) { @@ -781,22 +799,55 @@ fn sync_one( for dependent in store.reference_sites_dependent_on_file(relative)? { merge_dependent_site(dependent_sites, dependent_fallbacks, dependent); } - reextract_into_store(project_root, store, relative, scope, changed_names)?; + reextract_into_store( + store, + relative, + scope, + changed_names, + (&metadata, &source, content_hash), + )?; outcome.files_reindexed += 1; Ok(true) } +/// Drop `relative`'s record when it is tracked, collecting what its removal +/// invalidates; count it as ignored otherwise. +fn remove_tracked_file( + store: &mut Store, + relative: &str, + outcome: &mut SyncOutcome, + dependent_sites: &mut BTreeMap>, + dependent_fallbacks: &mut BTreeSet, + changed_names: &mut HashSet, +) -> Result { + let was_tracked = store.file_by_path(relative)?.is_some(); + if !was_tracked { + outcome.files_ignored += 1; + return Ok(false); + } + for dependent in store.reference_sites_dependent_on_file(relative)? { + merge_dependent_site(dependent_sites, dependent_fallbacks, dependent); + } + for name in node_names_in_file(store, relative)? { + changed_names.insert(name); + } + delete_unresolved_refs_by_file(store, relative)?; + store.delete_file_record(relative)?; + outcome.files_removed += 1; + Ok(true) +} + +/// Re-extract `relative` from the bytes `sync_one` already read and hashed, so +/// the record, its hash and its symbols all describe one read of the file. fn reextract_into_store( - project_root: &Path, store: &mut Store, relative: &str, scope: &ProjectScope, changed_names: &mut HashSet, + (metadata, source, content_hash): (&fs::Metadata, &SourceText, String), ) -> Result<()> { - let full = project_root.join(relative); - let source = fs::read_to_string(&full).with_context(|| format!("read {}", full.display()))?; - let metadata = fs::metadata(&full).with_context(|| format!("stat {}", full.display()))?; - let result = extract_file_with_options(project_root, relative, &scope.options)?; + let result = codegraph_extract::engine::extraction_of(relative, source, &scope.options, |_| {}); + let hash_input = source.hash_input().unwrap_or_default(); let node_ids = result .nodes .iter() @@ -809,10 +860,10 @@ fn reextract_into_store( .collect::>(); let file = FileRecord { path: relative.to_string(), - content_hash: hash_content(&source), + content_hash, language: detect_language_with(relative, &scope.options.extensions), size: metadata.len() as i64, - modified_at: modified_millis(&metadata), + modified_at: modified_millis(metadata), indexed_at: now_millis(), node_count: result .nodes @@ -820,7 +871,7 @@ fn reextract_into_store( .filter(|node| node.file_path == relative) .count() as i64, errors: result.errors, - generated: detect_generated_file(relative, &source), + generated: detect_generated_file(relative, &hash_input), }; // A name's resolution outcomes (confidence, chosen target) depend on the set diff --git a/docs/cli.md b/docs/cli.md index bb0ee5fc..22cd0c5f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1184,6 +1184,15 @@ synthetic file node, `init`/`index`/`sync` still succeed but print and persist `parse produced no symbols (tree has errors)`. Files with useful surviving symbols stay quiet even when the grammar tree contains recoverable error nodes. +A file over `indexing.max_file_size` (1 MiB by default) is recorded without +being read: no symbols, a `File exceeds max size` error, and a size stamp in +place of its content hash, so a same-size rewrite is not a change while crossing +the limit in either direction is. A `.ts` file whose head is an MPEG transport +stream is video rather than TypeScript: it is neither indexed nor counted, and a +tracked file that turns into one leaves the index. Bytes that are not valid +UTF-8 are decoded with U+FFFD replacements, so a Latin-1 or binary source file +is indexed like any other instead of failing the run. + ### Index diagnostics `init`, `index`, and `sync` share `--debug` and `--debug-log `. From a1b029c75856ca9134b1a62cc985764ee38c9e21 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 10:01:35 +0800 Subject: [PATCH 05/19] fix(extract): read MyBatis and iBatis mappers as upstream does A close tag with whitespace before its bracket () let a statement run on and swallow the next one (upstream #1222). The rest of upstream's #1182 forms were also missing: commented-out statements and includes are now ignored (CDATA stays data), attribute values may use either quote style, a namespace-less qualifies statements by their Map.statement ids, iBatis maps add and , and databaseId joins the signature. A vendor-split pair written on one line keeps both nodes through the #1349 id allocator. KEEP-RUST: a self-closing reference is never a statement-map root. --- .../codegraph-extract/src/embedded/mybatis.rs | 275 ++++++++++++++---- .../tests/mybatis_mappers.rs | 166 +++++++++++ 2 files changed, 385 insertions(+), 56 deletions(-) create mode 100644 crates/codegraph-extract/tests/mybatis_mappers.rs diff --git a/crates/codegraph-extract/src/embedded/mybatis.rs b/crates/codegraph-extract/src/embedded/mybatis.rs index 89117597..999dfc23 100644 --- a/crates/codegraph-extract/src/embedded/mybatis.rs +++ b/crates/codegraph-extract/src/embedded/mybatis.rs @@ -1,3 +1,7 @@ +use std::cell::RefCell; +use std::sync::OnceLock; + +use codegraph_core::node_id::{NodeIdAllocator, utf16_column}; use codegraph_core::types::{EdgeKind, ExtractionResult, Language, NodeKind}; use regex::Regex; @@ -6,114 +10,181 @@ use crate::embedded::shared::{ unresolved_ref, }; +/// The two mapper dialects (upstream `mybatis-extractor.ts`, #1182). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Dialect { + /// MyBatis 3 ``. + MyBatis, + /// iBatis 2 ``, whose namespace is optional and which adds the + /// generic `` and `` verbs. + IBatis, +} + pub struct MyBatisExtractor<'a> { file_path: &'a str, - source: &'a str, + /// The source as written: the file node and identity columns read it. + original: &'a str, + /// The source with every XML comment blanked byte for byte, so the scans + /// never see a commented-out statement or include while every byte offset + /// and line still maps to [`Self::original`]. + source: String, line_starts: Vec, + node_ids: RefCell, } impl<'a> MyBatisExtractor<'a> { pub fn new(file_path: &'a str, source: &'a str) -> Self { + let stripped = strip_xml_comments(source); Self { file_path, - source, - line_starts: line_starts(source), + original: source, + line_starts: line_starts(&stripped), + source: stripped, + node_ids: RefCell::new(NodeIdAllocator::default()), } } pub fn extract(self) -> ExtractionResult { let start = std::time::Instant::now(); let mut result = empty_result(0); - let file_node = file_like_node(self.file_path, self.source, Language::Xml); + let file_node = file_like_node(self.file_path, self.original, Language::Xml); let file_id = file_node.id.clone(); result.nodes.push(file_node); - if let Some((namespace, body_start, body_end)) = self.find_mapper_root() { - self.extract_mapper(&mut result, &file_id, &namespace, body_start, body_end); + if let Some((dialect, namespace, body_start, body_end)) = self.find_mapper_root() { + self.extract_mapper( + &mut result, + &file_id, + dialect, + &namespace, + body_start, + body_end, + ); } result.duration_ms += start.elapsed().as_millis() as i64; result } - /// Locates the mapper root element and its namespace. + /// Locates the mapper root element, its dialect and its namespace. /// - /// Both dialect forms are accepted: the MyBatis `` - /// root and the older iBatis 2 `` root (upstream + /// A MyBatis `` needs a namespace. Failing that, an iBatis 2 + /// `` root is accepted with or without one: a namespace-less map + /// qualifies each statement through its `Map.statement` id (upstream /// #1182). `\b` keeps `` — the iBatis *config* file, which - /// holds no statements — from being mistaken for a statement map. The - /// closing tag is derived from whichever root matched, so the body window - /// ends at the real root close instead of a hard-coded ``. - fn find_mapper_root(&self) -> Option<(String, usize, usize)> { - let open_re = Regex::new(r#"<(mapper|sqlMap)\b([^>]*)>"#).unwrap(); - let ns_re = Regex::new(r#"\bnamespace\s*=\s*"([^"]+)""#).unwrap(); - let open = open_re.captures(self.source)?; - let matched = open.get(0)?; - let root_tag = open.get(1)?.as_str(); - let attrs = open.get(2).map_or("", |m| m.as_str()); - let namespace = ns_re - .captures(attrs) - .and_then(|cap| cap.get(1).map(|m| m.as_str().to_string()))?; - let body_start = matched.end(); - let close_tag = format!(""); - let body_end = self.source[body_start..] - .find(&close_tag) - .map_or(self.source.len(), |idx| body_start + idx); - Some((namespace, body_start, body_end)) + /// holds no statements — from being mistaken for a statement map. The body + /// window ends at the matching root close, or the end of the file. + fn find_mapper_root(&self) -> Option<(Dialect, String, usize, usize)> { + static MAPPER: OnceLock = OnceLock::new(); + static SQL_MAP: OnceLock = OnceLock::new(); + let body = |open: regex::Captures<'_>, close_tag: &str| { + let body_start = open.get(0).map_or(0, |m| m.end()); + let body_end = self.source[body_start..] + .find(close_tag) + .map_or(self.source.len(), |idx| body_start + idx); + (body_start, body_end) + }; + + let mapper = MAPPER.get_or_init(|| Regex::new(r"]*)>").unwrap()); + if let Some(open) = mapper.captures(&self.source) { + let attrs = open.get(1).map_or("", |m| m.as_str()); + if let Some(namespace) = attribute(attrs, Attribute::Namespace) { + let namespace = namespace.to_string(); + let (body_start, body_end) = body(open, ""); + return Some((Dialect::MyBatis, namespace, body_start, body_end)); + } + } + let sql_map = SQL_MAP.get_or_init(|| Regex::new(r"]*)>").unwrap()); + // KEEP-RUST: a self-closing `` is how a + // `` lists its maps, not a map; it holds no statements, + // so it is never the root (upstream would read the config's tail as one). + let open = sql_map.captures_iter(&self.source).find(|open| { + !open + .get(1) + .is_some_and(|attrs| attrs.as_str().trim_end().ends_with('/')) + })?; + let attrs = open.get(1).map_or("", |m| m.as_str()); + let namespace = attribute(attrs, Attribute::Namespace) + .unwrap_or_default() + .to_string(); + let (body_start, body_end) = body(open, ""); + Some((Dialect::IBatis, namespace, body_start, body_end)) } fn extract_mapper( &self, result: &mut ExtractionResult, file_id: &str, + dialect: Dialect, namespace: &str, body_start: usize, body_end: usize, ) { - let open_re = Regex::new(r#"<(select|insert|update|delete|sql)\b([^>]*)>"#).unwrap(); - let id_re = Regex::new(r#"\bid\s*=\s*"([^"]+)""#).unwrap(); + static MYBATIS_OPEN: OnceLock = OnceLock::new(); + static IBATIS_OPEN: OnceLock = OnceLock::new(); + static CLOSE: OnceLock = OnceLock::new(); + let open_re = match dialect { + Dialect::MyBatis => MYBATIS_OPEN.get_or_init(|| { + Regex::new(r"<(select|insert|update|delete|sql)\b([^>]*)>").unwrap() + }), + Dialect::IBatis => IBATIS_OPEN.get_or_init(|| { + Regex::new(r"<(select|insert|update|delete|sql|statement|procedure)\b([^>]*)>") + .unwrap() + }), + }; + // A close tag may carry whitespace before its `>` (``, legal + // XML); missing it let the statement run on to the next close and + // swallow the statement in between (upstream #1222). + let close_re = CLOSE.get_or_init(|| { + Regex::new(r"").unwrap() + }); let body = &self.source[body_start..body_end]; let mut search_start = 0; - while let Some(open) = open_re.find(&body[search_start..]) { + while let Some(captures) = open_re.captures(&body[search_start..]) { + let open = captures.get(0).unwrap(); let open_abs_body = search_start + open.start(); - let full_open = &body[search_start + open.start()..search_start + open.end()]; - let Some(captures) = open_re.captures(full_open) else { - search_start += open.end(); - continue; - }; let elem_type = captures.get(1).unwrap().as_str(); let attrs = captures.get(2).map_or("", |m| m.as_str()); - let close_tag = format!(""); let content_start_body = search_start + open.end(); - let Some(close_rel) = body[content_start_body..].find(&close_tag) else { + let Some(close) = close_re + .captures_iter(&body[content_start_body..]) + .find(|close| close.get(1).is_some_and(|verb| verb.as_str() == elem_type)) + .and_then(|close| close.get(0)) + else { search_start += open.end(); continue; }; - let close_abs_body = content_start_body + close_rel; - let full_end_body = close_abs_body + close_tag.len(); + let close_abs_body = content_start_body + close.start(); + let full_end_body = content_start_body + close.end(); search_start = full_end_body; - let Some(id) = id_re - .captures(attrs) - .and_then(|cap| cap.get(1).map(|m| m.as_str().to_string())) - else { + let Some(id) = attribute(attrs, Attribute::Id) else { continue; }; let absolute_index = body_start + open_abs_body; let start_line = line_number_for_offset(&self.line_starts, absolute_index); let end_line = line_number_for_offset(&self.line_starts, body_start + full_end_body); - let qualified = format!("{namespace}::{id}"); + let (qualified, name) = qualify_statement(namespace, id); let elem_body = &body[content_start_body..close_abs_body]; let mut node = default_node( self.file_path, Language::Xml, NodeKind::Method, - id, - qualified.clone(), + name, + qualified, start_line, end_line, 0, 0, ); + // A vendor-split pair (`databaseId`) written on one line shares a + // name and a line; the allocator keeps the second (#1349). + node.id = self.node_ids.borrow_mut().generate( + self.file_path, + node.kind, + &node.name, + start_line.max(1) as u32, + utf16_column(self.original, absolute_index), + ); node.signature = Some(build_signature(elem_type, attrs)); node.docstring = Some(preview_sql(elem_body)); let node_id = node.id.clone(); @@ -137,10 +208,15 @@ impl<'a> MyBatisExtractor<'a> { elem_body: &str, elem_body_abs: usize, ) { - let include_re = Regex::new(r#"]*\brefid\s*=\s*"([^"]+)""#).unwrap(); + static INCLUDE: OnceLock = OnceLock::new(); + let include_re = INCLUDE.get_or_init(|| { + Regex::new(r#"]*\brefid\s*=\s*(?:"([^"']+)"|'([^"']+)')"#).unwrap() + }); for cap in include_re.captures_iter(elem_body) { let full = cap.get(0).unwrap(); - let refid = cap.get(1).unwrap().as_str(); + let Some(refid) = cap.get(1).or_else(|| cap.get(2)).map(|m| m.as_str()) else { + continue; + }; let ref_qualified = qualify_refid(namespace, refid); let line = line_number_for_offset(&self.line_starts, elem_body_abs + full.start()); result.unresolved_references.push(unresolved_ref( @@ -156,6 +232,88 @@ impl<'a> MyBatisExtractor<'a> { } } +/// Blanks every `` comment byte for byte (newlines kept), so a +/// commented-out statement or include is never matched while offsets and +/// lines are unchanged. `` is skipped intact: a `", index + 4).map_or(bytes.len(), |end| end + 3); + for byte in &mut out[index..stop] { + if *byte != b'\n' { + *byte = b' '; + } + } + index = stop; + continue; + } + index += 1; + } + String::from_utf8(out).unwrap_or_else(|_| source.to_string()) +} + +fn find_bytes(haystack: &[u8], needle: &[u8], from: usize) -> Option { + haystack + .get(from..)? + .windows(needle.len()) + .position(|window| window == needle) + .map(|position| from + position) +} + +#[derive(Debug, Clone, Copy)] +enum Attribute { + Namespace, + Id, + ResultType, + ParameterType, + DatabaseId, +} + +/// The value of attribute `which` in `attrs`, in either quote style. The +/// identifier-shaped values read here are Java names, statement ids or type +/// aliases and never contain a quote, so the value excludes both. +fn attribute(attrs: &str, which: Attribute) -> Option<&str> { + static PATTERNS: OnceLock<[Regex; 5]> = OnceLock::new(); + let patterns = PATTERNS.get_or_init(|| { + [ + "namespace", + "id", + "resultType", + "parameterType", + "databaseId", + ] + .map(|name| Regex::new(&format!(r#"\b{name}\s*=\s*(?:"([^"']+)"|'([^"']+)')"#)).unwrap()) + }); + let captures = patterns[which as usize].captures(attrs)?; + captures + .get(1) + .or_else(|| captures.get(2)) + .map(|m| m.as_str()) +} + +/// The `{namespace}::{id}` qualified name the MyBatis synthesizer +/// suffix-matches against a Java `{Class}::{method}`, and the display name. A +/// namespace-less iBatis map carries the qualifier in the id itself +/// (`Account.getById` → `Account::getById`, named `getById`). +fn qualify_statement(namespace: &str, id: &str) -> (String, String) { + if !namespace.is_empty() { + return (format!("{namespace}::{id}"), id.to_string()); + } + match id.rsplit_once('.') { + Some((owner, name)) => (format!("{owner}::{name}"), name.to_string()), + None => (id.to_string(), id.to_string()), + } +} + /// Builds the `{namespace}::{id}` reference name a `` fragment node /// carries, from an `` value (upstream #1209). /// @@ -165,12 +323,14 @@ impl<'a> MyBatisExtractor<'a> { /// exactly the `qualified_name` the fragment node stores /// (`com.example.UserMapper` + `::` + `baseColumns`). Rewriting every dot to /// `::` would produce a name no node carries, and dropping the namespace would -/// let a same-`id` fragment in another namespace answer the include. +/// let a same-`id` fragment in another namespace answer the include. In a +/// namespace-less iBatis map a bare `refid` names its fragment as is. fn qualify_refid(namespace: &str, refid: &str) -> String { match refid.rsplit_once('.') { Some((owner, fragment)) if !owner.is_empty() && !fragment.is_empty() => { format!("{owner}::{fragment}") } + _ if namespace.is_empty() => refid.to_string(), _ => format!("{namespace}::{refid}"), } } @@ -179,14 +339,17 @@ fn build_signature(elem_type: &str, attrs: &str) -> String { if elem_type == "sql" { return "".to_string(); } - let result_re = Regex::new(r#"\bresultType\s*=\s*"([^"]+)""#).unwrap(); - let param_re = Regex::new(r#"\bparameterType\s*=\s*"([^"]+)""#).unwrap(); let mut parts = vec![elem_type.to_ascii_uppercase()]; - if let Some(param) = param_re.captures(attrs).and_then(|cap| cap.get(1)) { - parts.push(format!("param={}", param.as_str())); + if let Some(param) = attribute(attrs, Attribute::ParameterType) { + parts.push(format!("param={param}")); + } + if let Some(result) = attribute(attrs, Attribute::ResultType) { + parts.push(format!("result={result}")); } - if let Some(result) = result_re.captures(attrs).and_then(|cap| cap.get(1)) { - parts.push(format!("result={}", result.as_str())); + // A vendor-split statement carries `databaseId`; surface it so the two + // otherwise identical `{namespace}::{id}` nodes are distinguishable. + if let Some(database) = attribute(attrs, Attribute::DatabaseId) { + parts.push(format!("databaseId={database}")); } parts.join(" ") } diff --git a/crates/codegraph-extract/tests/mybatis_mappers.rs b/crates/codegraph-extract/tests/mybatis_mappers.rs new file mode 100644 index 00000000..50dc2aae --- /dev/null +++ b/crates/codegraph-extract/tests/mybatis_mappers.rs @@ -0,0 +1,166 @@ +//! MyBatis and iBatis mapper forms upstream v1.6.1 reads (#1182, #1222): +//! close tags with whitespace, commented-out statements, either quote style, +//! namespace-less `` ids, the iBatis-only verbs, and vendor-split +//! statements written on one line. + +use codegraph_core::types::{EdgeKind, ExtractionResult, Language, Node, NodeKind}; +use codegraph_extract::extract_source; + +fn extract(file: &str, source: &str) -> ExtractionResult { + let result = extract_source(file, source, Some(Language::Xml)); + assert!(result.errors.is_empty(), "{file}: {:?}", result.errors); + result +} + +fn statements(result: &ExtractionResult) -> Vec<&Node> { + result + .nodes + .iter() + .filter(|node| node.kind == NodeKind::Method) + .collect() +} + +fn qualified_names(result: &ExtractionResult) -> Vec<&str> { + statements(result) + .iter() + .map(|node| node.qualified_name.as_str()) + .collect() +} + +fn reference_names(result: &ExtractionResult) -> Vec<&str> { + result + .unresolved_references + .iter() + .filter(|reference| reference.reference_kind == EdgeKind::References) + .map(|reference| reference.reference_name.as_str()) + .collect() +} + +#[test] +fn a_close_tag_may_carry_whitespace_before_its_bracket() { + let result = extract( + "mapper.xml", + "\n\ + \x20 \n\ + \x20 \n\ + \n", + ); + assert_eq!( + qualified_names(&result), + vec!["com.example.M::first", "com.example.M::second"] + ); + assert_eq!(statements(&result)[0].end_line, 2); +} + +#[test] +fn commented_out_statements_and_includes_are_ignored_but_cdata_is_not_a_comment() { + let result = extract( + "mapper.xml", + "\n\ + \x20 \n\ + \x20 \n\ + \x20 \n\ + \x20 \n\ + \n", + ); + assert_eq!( + qualified_names(&result), + vec![ + "com.example.M::live", + "com.example.M::cols", + "com.example.M::after" + ] + ); + assert_eq!(reference_names(&result), vec!["com.example.M::cols"]); + let include = &result.unresolved_references[0]; + assert_eq!(include.line, 4, "comment blanking keeps original lines"); +} + +#[test] +fn ibatis_sqlmaps_without_a_namespace_qualify_by_their_ids() { + let result = extract( + "Account.xml", + "\n\ + \x20 \n\ + \x20 CALL raw()\n\ + \x20 {call p()}\n\ + \x20 id, name\n\ + \x20 x\n\ + \x20 \n\ + \n", + ); + assert_eq!( + qualified_names(&result), + vec![ + "Account::getById", + "Account::raw", + "Account::proc", + "Account::cols", + "bare", + "local" + ] + ); + let names = statements(&result) + .iter() + .map(|node| node.name.as_str()) + .collect::>(); + assert_eq!( + names, + vec!["getById", "raw", "proc", "cols", "bare", "local"] + ); + assert_eq!(reference_names(&result), vec!["Account::cols", "bare"]); +} + +#[test] +fn either_quote_style_is_read_and_ibatis_verbs_stay_ibatis_only() { + let ibatis = extract( + "Legacy.xml", + "\n CALL run()\n\n", + ); + assert_eq!(qualified_names(&ibatis), vec!["Legacy::run"]); + + let mybatis = extract( + "QueryMapper.xml", + "\n\ + \x20 \n\ + \x20 CALL x()\n\ + \x20 \n\ + \n", + ); + assert_eq!(qualified_names(&mybatis), vec!["com.example.Q::q"]); + assert_eq!( + statements(&mybatis)[0].signature.as_deref(), + Some("SELECT param=int result=Row databaseId=oracle") + ); +} + +#[test] +fn vendor_split_statements_on_one_line_keep_both_nodes() { + let result = extract( + "mapper.xml", + "\n", + ); + let nodes = statements(&result); + assert_eq!(nodes.len(), 2, "{nodes:#?}"); + assert_ne!(nodes[0].id, nodes[1].id); + assert_eq!( + nodes + .iter() + .map(|node| node.signature.as_deref()) + .collect::>(), + vec![ + Some("SELECT databaseId=oracle"), + Some("SELECT databaseId=mysql") + ] + ); + for node in nodes { + assert!( + result + .edges + .iter() + .any(|edge| edge.kind == EdgeKind::Contains && edge.target == node.id) + ); + } +} From a1b91a9ecb34f52c61c62e1d807babcdd8086204 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 10:05:26 +0800 Subject: [PATCH 06/19] fix(extract): read Liquid tags inside {% liquid %} blocks A {% liquid %} body holds tags without braces of their own, so render, include, section and assign written there were never extracted. Scan whole braced tags one at a time and the bare lines of each liquid body, anchored at the line start, so tag-like strings and inline comments are not read twice; {% comment %} and {% raw %} regions are skipped in both spellings (upstream #1906). --- .../codegraph-extract/src/embedded/liquid.rs | 182 +++++++++++-- .../codegraph-extract/tests/liquid_blocks.rs | 257 ++++++++++++++++++ 2 files changed, 412 insertions(+), 27 deletions(-) create mode 100644 crates/codegraph-extract/tests/liquid_blocks.rs diff --git a/crates/codegraph-extract/src/embedded/liquid.rs b/crates/codegraph-extract/src/embedded/liquid.rs index 05957051..1609c28a 100644 --- a/crates/codegraph-extract/src/embedded/liquid.rs +++ b/crates/codegraph-extract/src/embedded/liquid.rs @@ -1,4 +1,5 @@ use std::cell::RefCell; +use std::sync::OnceLock; use codegraph_core::node_id::{NodeIdAllocator, utf16_column}; use codegraph_core::types::{EdgeKind, ExtractionResult, Language, Node, NodeKind}; @@ -9,6 +10,51 @@ use crate::embedded::shared::{ line_start_for, line_starts, unresolved_ref, }; +/// The tags the extractor reads, each as its name pattern and the argument +/// pattern that follows it. +#[derive(Debug, Clone, Copy)] +enum LiquidTag { + /// `render 'name'` / `include 'name'`. + Snippet, + /// `section 'name'`. + Section, + /// `assign name =`. + Assign, +} + +impl LiquidTag { + /// The braced spelling (matched against one whole `{% … %}` tag) and the + /// bare spelling (matched against one line of a `{% liquid %}` body). + fn patterns(self) -> &'static (Regex, Regex) { + static PATTERNS: OnceLock<[(Regex, Regex); 3]> = OnceLock::new(); + let patterns = PATTERNS.get_or_init(|| { + [ + ("render|include", r#"['"]([^'"]+)['"]"#), + ("section", r#"['"]([^'"]+)['"]"#), + ("assign", r"([0-9A-Za-z_]+)\s*="), + ] + .map(|(name, argument)| { + ( + Regex::new(&format!(r"^\{{%-?\s*({name})\s+{argument}")).unwrap(), + Regex::new(&format!(r"^[ \t]*({name})[ \t]+{argument}")).unwrap(), + ) + }) + }); + &patterns[self as usize] + } +} + +/// One occurrence of a tag in either spelling. +struct TagOccurrence<'s> { + /// From the tag's start through its argument: `{% render 'card'` for a + /// braced tag, `render 'card'` for a bare line of a `{% liquid %}` block. + text: &'s str, + /// The tag name, then the argument's captures. + groups: Vec<&'s str>, + /// Byte offset of `text` in the source. + offset: usize, +} + pub struct LiquidExtractor<'a> { file_path: &'a str, source: &'a str, @@ -40,6 +86,95 @@ impl<'a> LiquidExtractor<'a> { node } + /// Every occurrence of `tag`, in both spellings Liquid allows (upstream + /// #1906). Inside a `{% liquid %}` tag each body line is a tag without + /// braces of its own: + /// + /// ```liquid + /// {% liquid + /// assign heading = section.settings.title + /// render 'card', title: heading + /// %} + /// ``` + /// + /// Whole braced tags are consumed one at a time, so a tag-like string or an + /// inline `{% # … %}` comment never starts a second match inside a tag, and + /// `{% comment %}` / `{% raw %}` regions are skipped in both spellings. + /// Bare tags are anchored at a body line's start, which keeps prose, + /// filters and `#` comment lines out. Sorted by offset. + fn tag_occurrences(&self, tag: LiquidTag) -> Vec> { + static TAGS: OnceLock = OnceLock::new(); + static FIRST_WORD: OnceLock = OnceLock::new(); + let tags = + TAGS.get_or_init(|| Regex::new(r"\{%-?\s*([0-9A-Za-z_]+|#)((?s:.*?))-?%\}").unwrap()); + let first_word = FIRST_WORD.get_or_init(|| Regex::new(r"^[ \t]*([0-9A-Za-z_]+)").unwrap()); + let (braced, bare) = tag.patterns(); + let groups = |captures: ®ex::Captures<'a>| { + captures + .iter() + .skip(1) + .map(|group| group.map_or("", |m| m.as_str())) + .collect::>() + }; + + let mut found = Vec::new(); + let mut blocks = Vec::new(); + let mut suppressed: Option<&str> = None; + for captures in tags.captures_iter(self.source) { + let whole = captures.get(0).unwrap(); + let name = captures.get(1).unwrap(); + if let Some(region) = suppressed { + if name.as_str().strip_prefix("end") == Some(region) { + suppressed = None; + } + continue; + } + match name.as_str() { + "comment" | "raw" => suppressed = Some(name.as_str()), + "liquid" => blocks.push((name.end(), captures.get(2).map_or("", |m| m.as_str()))), + _ => { + if let Some(matched) = braced.captures(whole.as_str()) { + found.push(TagOccurrence { + text: matched.get(0).unwrap().as_str(), + groups: groups(&matched), + offset: whole.start(), + }); + } + } + } + } + + for (body_start, body) in blocks { + let mut offset = body_start; + let mut suppressed: Option<&str> = None; + for line in body.split('\n') { + let name = first_word + .captures(line) + .and_then(|captures| captures.get(1)) + .map(|m| m.as_str()); + if let Some(region) = suppressed { + if name.and_then(|name| name.strip_prefix("end")) == Some(region) { + suppressed = None; + } + } else if let Some(region @ ("comment" | "raw")) = name { + suppressed = Some(region); + } else if let Some(matched) = bare.captures(line) { + let whole = matched.get(0).unwrap().as_str(); + let text = whole.trim_start(); + found.push(TagOccurrence { + text, + groups: groups(&matched), + offset: offset + (whole.len() - text.len()), + }); + } + offset += line.len() + 1; + } + } + + found.sort_by_key(|occurrence| occurrence.offset); + found + } + pub fn extract(self) -> ExtractionResult { let start = std::time::Instant::now(); let mut result = empty_result(0); @@ -85,14 +220,11 @@ impl<'a> LiquidExtractor<'a> { } fn extract_snippets(&self, result: &mut ExtractionResult, file_id: &str) { - let regex = Regex::new(r#"\{%[-]?\s*(render|include)\s+['"]([^'"]+)['"]"#).unwrap(); - for cap in regex.captures_iter(self.source) { - let full = cap.get(0).unwrap(); - let tag_type = cap.get(1).unwrap().as_str(); - let name = cap.get(2).unwrap().as_str(); - let line = line_number_for_offset(&self.line_starts, full.start()); - let col = full.start() as i64 - line_start_for(&self.line_starts, line) as i64; - self.push_import_node(result, file_id, name, full.as_str(), full.start()); + for occurrence in self.tag_occurrences(LiquidTag::Snippet) { + let (tag_type, name) = (occurrence.groups[0], occurrence.groups[1]); + let line = line_number_for_offset(&self.line_starts, occurrence.offset); + let col = occurrence.offset as i64 - line_start_for(&self.line_starts, line) as i64; + self.push_import_node(result, file_id, name, occurrence.text, occurrence.offset); let node = self.identify( default_node( self.file_path, @@ -103,9 +235,9 @@ impl<'a> LiquidExtractor<'a> { line, line, col, - col + full.as_str().len() as i64, + col + occurrence.text.len() as i64, ), - full.start(), + occurrence.offset, ); let node_id = node.id.clone(); result.nodes.push(node); @@ -123,13 +255,11 @@ impl<'a> LiquidExtractor<'a> { } fn extract_sections(&self, result: &mut ExtractionResult, file_id: &str) { - let regex = Regex::new(r#"\{%[-]?\s*section\s+['"]([^'"]+)['"]"#).unwrap(); - for cap in regex.captures_iter(self.source) { - let full = cap.get(0).unwrap(); - let name = cap.get(1).unwrap().as_str(); - let line = line_number_for_offset(&self.line_starts, full.start()); - let col = full.start() as i64 - line_start_for(&self.line_starts, line) as i64; - self.push_import_node(result, file_id, name, full.as_str(), full.start()); + for occurrence in self.tag_occurrences(LiquidTag::Section) { + let name = occurrence.groups[1]; + let line = line_number_for_offset(&self.line_starts, occurrence.offset); + let col = occurrence.offset as i64 - line_start_for(&self.line_starts, line) as i64; + self.push_import_node(result, file_id, name, occurrence.text, occurrence.offset); let node = self.identify( default_node( self.file_path, @@ -140,9 +270,9 @@ impl<'a> LiquidExtractor<'a> { line, line, col, - col + full.as_str().len() as i64, + col + occurrence.text.len() as i64, ), - full.start(), + occurrence.offset, ); let node_id = node.id.clone(); result.nodes.push(node); @@ -190,12 +320,10 @@ impl<'a> LiquidExtractor<'a> { } fn extract_assignments(&self, result: &mut ExtractionResult, file_id: &str) { - let regex = Regex::new(r"\{%[-]?\s*assign\s+(\w+)\s*=").unwrap(); - for cap in regex.captures_iter(self.source) { - let full = cap.get(0).unwrap(); - let name = cap.get(1).unwrap().as_str(); - let line = line_number_for_offset(&self.line_starts, full.start()); - let col = full.start() as i64 - line_start_for(&self.line_starts, line) as i64; + for occurrence in self.tag_occurrences(LiquidTag::Assign) { + let name = occurrence.groups[1]; + let line = line_number_for_offset(&self.line_starts, occurrence.offset); + let col = occurrence.offset as i64 - line_start_for(&self.line_starts, line) as i64; let node = self.identify( default_node( self.file_path, @@ -206,9 +334,9 @@ impl<'a> LiquidExtractor<'a> { line, line, col, - col + full.as_str().len() as i64, + col + occurrence.text.len() as i64, ), - full.start(), + occurrence.offset, ); let node_id = node.id.clone(); result.nodes.push(node); diff --git a/crates/codegraph-extract/tests/liquid_blocks.rs b/crates/codegraph-extract/tests/liquid_blocks.rs new file mode 100644 index 00000000..9088b57f --- /dev/null +++ b/crates/codegraph-extract/tests/liquid_blocks.rs @@ -0,0 +1,257 @@ +//! Tags inside a `{% liquid %}` block (upstream v1.6.1 #1906): each body line is +//! a tag without braces of its own. Ported from upstream `extraction.test.ts`. + +use codegraph_core::types::{ExtractionResult, Language, NodeKind}; +use codegraph_extract::extract_source; + +fn extract(file: &str, lines: &[&str]) -> ExtractionResult { + extract_with(file, lines, "\n") +} + +fn extract_with(file: &str, lines: &[&str], newline: &str) -> ExtractionResult { + let result = extract_source(file, &lines.join(newline), Some(Language::Liquid)); + assert!(result.errors.is_empty(), "{file}: {:?}", result.errors); + result +} + +fn names_of(result: &ExtractionResult, kind: NodeKind) -> Vec<&str> { + result + .nodes + .iter() + .filter(|node| node.kind == kind) + .map(|node| node.name.as_str()) + .collect() +} + +fn reference_names(result: &ExtractionResult) -> Vec<&str> { + result + .unresolved_references + .iter() + .map(|reference| reference.reference_name.as_str()) + .collect() +} + +#[test] +fn render_assign_and_section_inside_a_liquid_block_are_extracted() { + let result = extract( + "sections/featured.liquid", + &[ + "{% liquid", + " assign heading = section.settings.title", + " render 'card', title: heading", + "%}", + ], + ); + assert!(names_of(&result, NodeKind::Import).contains(&"card")); + assert!(names_of(&result, NodeKind::Variable).contains(&"heading")); + + let result = extract( + "layout/theme.liquid", + &["{% liquid", " section 'header'", "%}"], + ); + assert!(names_of(&result, NodeKind::Import).contains(&"header")); +} + +#[test] +fn a_tag_inside_a_liquid_block_reports_its_real_line() { + let result = extract( + "sections/featured.liquid", + &[ + "
", + "{% liquid", + " assign x = 1", + " render 'card'", + "%}", + ], + ); + let card = result + .nodes + .iter() + .find(|node| node.kind == NodeKind::Import && node.name == "card") + .expect("card import"); + assert_eq!(card.start_line, 4); +} + +#[test] +fn both_spellings_are_counted_once_each() { + let result = extract( + "sections/featured.liquid", + &["{% render 'card' %}", "{% liquid", " render 'card'", "%}"], + ); + let lines = result + .nodes + .iter() + .filter(|node| node.kind == NodeKind::Import && node.name == "card") + .map(|node| node.start_line) + .collect::>(); + assert_eq!(lines, vec![1, 3]); +} + +#[test] +fn a_bare_render_outside_a_liquid_block_is_prose() { + let result = extract( + "sections/featured.liquid", + &[ + "

We render the card below.

", + "{{ product | render_as: 'card' }}", + ], + ); + assert!(!names_of(&result, NodeKind::Import).contains(&"card")); +} + +#[test] +fn whitespace_control_on_the_liquid_tag_is_accepted() { + let result = extract( + "sections/featured.liquid", + &["{%- liquid", " render 'card'", "-%}"], + ); + assert!(names_of(&result, NodeKind::Import).contains(&"card")); +} + +#[test] +fn tag_like_strings_and_inline_comments_are_not_scanned_twice() { + let result = extract( + "sections/featured.liquid", + &[ + "{% assign example = \"{% render 'ghost'\" %}", + "{% # {% render 'ghost' %}", + "{% liquid", + " assign example = \"{% include 'ghost'\"", + " # section 'ghost'", + " echo 'render ghost'", + "%}", + "{% liquid render 'live' %}", + "{% liquid include 'after' %}", + ], + ); + let imports = result + .nodes + .iter() + .filter(|node| node.kind == NodeKind::Import) + .map(|node| (node.name.as_str(), node.start_line, node.start_column)) + .collect::>(); + assert_eq!(imports, vec![("live", 8, 10), ("after", 9, 10)]); + assert_eq!( + reference_names(&result), + vec!["snippets/live.liquid", "snippets/after.liquid"] + ); +} + +#[test] +fn liquid_block_positions_survive_both_line_endings() { + for newline in ["\n", "\r\n"] { + let result = extract_with( + "sections/featured.liquid", + &[ + "
", + "{%- liquid", + " assign heading = 'x'", + "\tinclude 'legacy'", + " render 'card'", + " section 'footer'", + "-%}", + " {% render 'after' %}", + ], + newline, + ); + let mut nodes = result + .nodes + .iter() + .filter(|node| matches!(node.kind, NodeKind::Variable | NodeKind::Import)) + .map(|node| { + ( + node.name.as_str(), + node.start_line, + node.start_column, + node.end_column, + ) + }) + .collect::>(); + nodes.sort(); + assert_eq!( + nodes, + vec![ + ("after", 8, 2, 19), + ("card", 5, 2, 15), + ("footer", 6, 2, 18), + ("heading", 3, 2, 18), + ("legacy", 4, 1, 17), + ], + "{newline:?}" + ); + let mut references = result + .unresolved_references + .iter() + .map(|reference| { + ( + reference.reference_name.as_str(), + reference.line, + reference.col, + ) + }) + .collect::>(); + references.sort(); + assert_eq!( + references, + vec![ + ("sections/footer.liquid", 6, 2), + ("snippets/after.liquid", 8, 2), + ("snippets/card.liquid", 5, 2), + ("snippets/legacy.liquid", 4, 1), + ], + "{newline:?}" + ); + } +} + +#[test] +fn comment_and_raw_regions_are_ignored_in_both_spellings() { + for tag in ["comment", "raw"] { + let open = format!("{{%- {tag} -%}}"); + let close = format!("{{%- end{tag} -%}}"); + let bare_open = format!(" {tag}"); + let bare_close = format!(" end{tag}"); + let result = extract( + "sections/featured.liquid", + &[ + &open, + "{% render 'ghost' %}", + "{% include 'ghost' %}", + "{% section 'ghost' %}", + "{% assign ghost = 1 %}", + "{% liquid", + " render 'ghost'", + "%}", + &close, + "{% liquid", + &bare_open, + " render 'ghost'", + " include 'ghost'", + " section 'ghost'", + " assign ghost = 1", + &bare_close, + " # render 'ghost'", + " render 'live'", + "%}", + "{% include 'after' %}", + ], + ); + let names = result + .nodes + .iter() + .filter(|node| node.kind != NodeKind::File) + .map(|node| node.name.as_str()) + .collect::>(); + assert_eq!(names, vec!["live", "live", "after", "after"], "{tag}"); + let references = result + .unresolved_references + .iter() + .map(|reference| (reference.reference_name.as_str(), reference.line)) + .collect::>(); + assert_eq!( + references, + vec![("snippets/live.liquid", 18), ("snippets/after.liquid", 20)], + "{tag}" + ); + } +} From 7514594b437c9d6809947bfc27bbd5aeaae9d8b2 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 10:08:38 +0800 Subject: [PATCH 07/19] fix(sync): report the pending references a healing sync resolved The orphan sweep's resolver statistics were dropped, so a sync that healed an interrupted index on unchanged files read as a no-op. Carry the attempted, resolved and unresolved counts in SyncOutcome, print them after the sync line, and add them to the diagnostics payload (upstream #1360). --- crates/codegraph-cli/src/main.rs | 19 +++ .../tests/sync_recovery_report.rs | 116 ++++++++++++++++++ crates/codegraph-watch/src/sync.rs | 32 ++++- 3 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 crates/codegraph-cli/tests/sync_recovery_report.rs diff --git a/crates/codegraph-cli/src/main.rs b/crates/codegraph-cli/src/main.rs index 7cc2e24a..04b29781 100644 --- a/crates/codegraph-cli/src/main.rs +++ b/crates/codegraph-cli/src/main.rs @@ -1774,6 +1774,22 @@ fn cmd_sync(path: Option, quiet: bool, diagnostics: DiagnosticArgs) -> format_number(outcome.files_removed as i64), format_duration(outcome.duration_ms as i64) ); + // A sync that healed an interrupted index reports the sweep's work, so + // recovering references on unchanged files is not a no-op (#1360). + if outcome.pending_refs_processed > 0 { + let unresolved = if outcome.pending_refs_unresolved > 0 { + format!( + " ({} unresolved)", + format_number(outcome.pending_refs_unresolved as i64) + ) + } else { + String::new() + }; + println!( + "Resolved {} pending references{unresolved}", + format_number(outcome.pending_refs_resolved as i64) + ); + } for warning in &sync_warnings { eprintln!("warning: {warning}"); } @@ -1784,6 +1800,9 @@ fn cmd_sync(path: Option, quiet: bool, diagnostics: DiagnosticArgs) -> "filesReindexed": outcome.files_reindexed, "filesSkipped": outcome.files_skipped_unchanged, "filesRemoved": outcome.files_removed, + "pendingRefsProcessed": outcome.pending_refs_processed, + "pendingRefsResolved": outcome.pending_refs_resolved, + "pendingRefsUnresolved": outcome.pending_refs_unresolved, "warnings": sync_warnings, })); Ok(()) diff --git a/crates/codegraph-cli/tests/sync_recovery_report.rs b/crates/codegraph-cli/tests/sync_recovery_report.rs new file mode 100644 index 00000000..3f9d317e --- /dev/null +++ b/crates/codegraph-cli/tests/sync_recovery_report.rs @@ -0,0 +1,116 @@ +//! A sync that heals an interrupted index says so (upstream v1.6.1 #1360): the +//! orphan sweep's work is reported instead of reading as a no-op. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use codegraph_core::types::{EdgeKind, Language, UnresolvedRef}; +use codegraph_store::Store; + +struct TestDir(PathBuf); + +impl Drop for TestDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn run(cwd: &Path, args: &[&str]) -> (String, String, bool) { + let output = Command::new(env!("CARGO_BIN_EXE_codegraph")) + .args(args) + .current_dir(cwd) + .env("CODEGRAPH_NO_DAEMON", "1") + .env("CODEGRAPH_NO_WATCH", "1") + .output() + .expect("run codegraph"); + ( + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + output.status.success(), + ) +} + +#[test] +fn a_healing_sync_reports_the_pending_references_it_resolved() { + let dir = TestDir(std::env::temp_dir().join(format!( + "codegraph-cli-sync-recovery-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + ))); + fs::create_dir_all(dir.0.join("src")).unwrap(); + fs::write( + dir.0.join("src/math.ts"), + "export function add(a: number, b: number): number { return a + b; }\n", + ) + .unwrap(); + fs::write( + dir.0.join("src/app.ts"), + "import { add } from './math';\nexport function run(): number { return add(1, 2); }\n", + ) + .unwrap(); + let path = dir.0.to_str().unwrap(); + let (stdout, stderr, ok) = run(&dir.0, &["init", path]); + assert!(ok, "init failed: {stdout} {stderr}"); + + let (stdout, stderr, ok) = run(&dir.0, &["sync", path]); + assert!(ok, "sync failed: {stdout} {stderr}"); + assert!( + !stdout.contains("pending references"), + "a healthy sync sweeps nothing: {stdout}" + ); + + // Simulate an interrupted resolution pass: the resolved call edge is gone, + // its reference is parked again, and the incomplete-resolution marker is set. + { + let mut store = Store::open(&dir.0.join(".codegraph").join("codegraph.db")).unwrap(); + let caller = store + .nodes_by_file_path("src/app.ts") + .unwrap() + .into_iter() + .find(|node| node.name == "run") + .expect("run node"); + store.delete_resolved_edges_from_file("src/app.ts").unwrap(); + store + .insert_unresolved_refs(&[ + UnresolvedRef { + id: None, + from_node_id: caller.id.clone(), + reference_name: "add".to_string(), + reference_kind: EdgeKind::Calls, + line: 2, + col: 0, + candidates: None, + file_path: "src/app.ts".to_string(), + language: Language::TypeScript, + is_function_ref: false, + reference_subkind: None, + }, + UnresolvedRef { + id: None, + from_node_id: caller.id, + reference_name: "missingEverywhere".to_string(), + reference_kind: EdgeKind::Calls, + line: 2, + col: 0, + candidates: None, + file_path: "src/app.ts".to_string(), + language: Language::TypeScript, + is_function_ref: false, + reference_subkind: None, + }, + ]) + .unwrap(); + store.set_resolution_incomplete().unwrap(); + } + + let (stdout, stderr, ok) = run(&dir.0, &["sync", path]); + assert!(ok, "sync failed: {stdout} {stderr}"); + assert!( + stdout.contains("Resolved 1 pending references (1 unresolved)"), + "stdout={stdout}" + ); +} diff --git a/crates/codegraph-watch/src/sync.rs b/crates/codegraph-watch/src/sync.rs index c20ab82a..780b9b13 100644 --- a/crates/codegraph-watch/src/sync.rs +++ b/crates/codegraph-watch/src/sync.rs @@ -102,6 +102,12 @@ pub struct SyncOutcome { /// this remains populated when another writer (for example startup catch-up) /// indexed the same event first and this sync consequently skips it unchanged. pub trigger_paths: Vec, + /// References the orphan sweep attempted, resolved and left unresolved + /// when it ran (upstream #1360), so a sync that healed an interrupted index + /// does not read as a no-op. Zero on a healthy index. + pub pending_refs_processed: usize, + pub pending_refs_resolved: usize, + pub pending_refs_unresolved: usize, } /// Read-only difference between the current project scope and the persisted @@ -556,7 +562,10 @@ fn sync_paths_with_store( // healthy index (marker absent) skips this entirely, so an ordinary sync is // byte-for-byte unchanged. if store.is_resolution_incomplete()? { - sweep_orphaned_refs(project_root, store, scope.options.max_file_size)?; + let stats = sweep_orphaned_refs(project_root, store, scope.options.max_file_size)?; + outcome.pending_refs_processed = stats.total; + outcome.pending_refs_resolved = stats.resolved; + outcome.pending_refs_unresolved = stats.unresolved; } outcome.duration_ms = started.elapsed().as_millis(); @@ -697,7 +706,11 @@ fn merge_dependent_site( /// batched pass re-arms and clears the marker itself, so on success the index is /// no longer flagged partial. Framework per-file extract is re-run first so any /// framework refs an interrupted run never re-injected are present for the sweep. -fn sweep_orphaned_refs(project_root: &Path, store: &mut Store, max_file_size: u64) -> Result<()> { +fn sweep_orphaned_refs( + project_root: &Path, + store: &mut Store, + max_file_size: u64, +) -> Result { let mut resolver = ReferenceResolver::new(project_root.to_string_lossy()).with_max_file_size(max_file_size); { @@ -711,9 +724,9 @@ fn sweep_orphaned_refs(project_root: &Path, store: &mut Store, max_file_size: u6 // duplicate rows (`unresolved_refs` has no UNIQUE constraint), which // accumulate across interrupt→heal cycles and diverge from `index --force`. // Re-resolving the existing rows alone heals the interrupted pass byte-equal. - resolver.resolve_and_persist_batched(store, ORPHAN_SWEEP_BATCH_ROWS)?; + let result = resolver.resolve_and_persist_batched(store, ORPHAN_SWEEP_BATCH_ROWS)?; resolver.run_post_extract(store)?; - Ok(()) + Ok(result.stats) } /// Batch size for the #1187 orphan sweep, matching the CLI full-index batch @@ -1564,6 +1577,16 @@ pub(crate) mod tests { // When: a bare no-change sync runs (no path is dirty on disk). let outcome = sync_changed_paths(dir.path(), &db, Vec::::new()).unwrap(); assert_eq!(outcome.files_reindexed, 0, "no file changed"); + // The sweep's work is reported, so a recovering sync is not a no-op + // (upstream #1360). + assert_eq!( + ( + outcome.pending_refs_processed, + outcome.pending_refs_resolved, + outcome.pending_refs_unresolved + ), + (1, 1, 0) + ); // Then: the orphaned ref is swept into a Calls edge and the marker clears. let store = Store::open(&db).unwrap(); @@ -1607,6 +1630,7 @@ pub(crate) mod tests { // Then: no marker was set, nothing was swept, and the retained // unresolvable rows are untouched (#1240 retry state preserved). assert_eq!(outcome.files_reindexed, 0); + assert_eq!(outcome.pending_refs_processed, 0, "no sweep ran"); assert!(!before.0, "a healthy index must not carry the marker"); let store = Store::open(&db).unwrap(); assert!(!store.is_resolution_incomplete().unwrap()); From e99c9499390fb72dcabbde36f68b0502768cd2f1 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 10:14:57 +0800 Subject: [PATCH 08/19] fix(mcp): a database file without a published index is not a root MCP root discovery counted any .codegraph/codegraph.db as an index, so a stray, empty or schema-less file in an ancestor captured the upward walk for every project below it and the sub-project scan never ran. Require the published namespace state as well, the authority the CLI's ancestor walk already uses; it reads only the state slots and never opens SQLite (upstream #1895). Test fixtures now publish their namespace. --- crates/codegraph-mcp/src/rmcp_handler.rs | 14 ++-- crates/codegraph-mcp/src/roots.rs | 100 ++++++++++++++++++++--- crates/codegraph-mcp/src/server.rs | 12 +-- 3 files changed, 95 insertions(+), 31 deletions(-) diff --git a/crates/codegraph-mcp/src/rmcp_handler.rs b/crates/codegraph-mcp/src/rmcp_handler.rs index c01387d3..7ff303c6 100644 --- a/crates/codegraph-mcp/src/rmcp_handler.rs +++ b/crates/codegraph-mcp/src/rmcp_handler.rs @@ -998,22 +998,18 @@ mod handler_tests { TempDir { path } } - /// Write a placeholder (non-SQLite) db file so `db_exists_for(p)` is true — - /// resolution treats the dir as indexed, but a real engine open fails. + /// Write a placeholder (non-SQLite) db file under a published namespace so + /// `db_exists_for(p)` is true — resolution treats the dir as indexed, but a + /// real engine open fails. fn placeholder_indexed(tag: &str) -> TempDir { let dir = unique_dir(tag); - let db = db_path_for(&dir.path).expect("default project resolves"); - std::fs::create_dir_all(db.parent().unwrap()).unwrap(); - std::fs::write(&db, b"not a real sqlite db").unwrap(); + crate::roots::write_index_fixture(&dir.path, b"not a real sqlite db"); dir } fn placeholder_indexed_child(workspace: &TempDir, name: &str) -> PathBuf { let child = workspace.path.join(name); - std::fs::create_dir_all(&child).unwrap(); - let db = db_path_for(&child).expect("child project resolves"); - std::fs::create_dir_all(db.parent().unwrap()).unwrap(); - std::fs::write(&db, b"not a real sqlite db").unwrap(); + crate::roots::write_index_fixture(&child, b"not a real sqlite db"); child } diff --git a/crates/codegraph-mcp/src/roots.rs b/crates/codegraph-mcp/src/roots.rs index 32d2c634..c5e19e4a 100644 --- a/crates/codegraph-mcp/src/roots.rs +++ b/crates/codegraph-mcp/src/roots.rs @@ -391,8 +391,16 @@ fn classify_resolve( ) -> RootStatus { use codegraph_core::IndexPathsError; match resolved { + // An index is a published namespace, not merely a file: a stray, empty or + // schema-less `codegraph.db` with no state record (an interrupted copy, a + // stray touch) must not capture the upward walk for every project below + // it (upstream #1895). The CLI's ancestor walk uses the same authority, + // which reads only the state slots and never opens SQLite. Ok(paths) => { - if paths.current_db().is_file() { + if paths.current_db().is_file() + && codegraph_store::Store::extraction_status(&paths) + != codegraph_store::ExtractionStatus::Missing + { RootStatus::Indexed } else { RootStatus::Absent @@ -696,6 +704,34 @@ fn percent_decode(input: &str) -> String { String::from_utf8_lossy(&out).into_owned() } +/// Test fixture: an index in the eyes of root discovery under `paths` — the +/// namespace created and published as Current, which is what makes a directory +/// an index (#1895), around a database file holding `db_bytes`. +#[cfg(test)] +pub(crate) fn write_index_fixture_at(paths: &codegraph_core::IndexPaths, db_bytes: &[u8]) { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let lease = codegraph_store::IndexLease::create_exclusive(paths, deadline, || false) + .expect("fixture namespace lease"); + codegraph_store::publish_index_state(paths, &lease, codegraph_store::StatePhase::Building) + .expect("publish fixture building state"); + std::fs::create_dir_all(paths.current_db().parent().unwrap()).unwrap(); + std::fs::write(paths.current_db(), db_bytes).unwrap(); + codegraph_store::publish_index_state(paths, &lease, codegraph_store::StatePhase::Current) + .expect("publish fixture current state"); +} + +/// [`write_index_fixture_at`] for `project` under the current `CODEGRAPH_DIR`. +#[cfg(test)] +pub(crate) fn write_index_fixture(project: &Path, db_bytes: &[u8]) { + std::fs::create_dir_all(project).unwrap(); + let paths = codegraph_core::IndexPaths::resolve( + project, + std::env::var("CODEGRAPH_DIR").ok().as_deref(), + ) + .expect("fixture project resolves"); + write_index_fixture_at(&paths, db_bytes); +} + #[cfg(test)] mod tests { use super::*; @@ -723,12 +759,7 @@ mod tests { let seq = SEQ.fetch_add(1, Ordering::Relaxed); let path = std::env::temp_dir().join(format!("cg-mcp-roots-{tag}-{}-{seq}", std::process::id())); - // The project dir must exist before `db_path_for` (which resolves the - // physical identity) can succeed. - std::fs::create_dir_all(&path).unwrap(); - let db = db_path_for(&path).expect("default project resolves"); - std::fs::create_dir_all(db.parent().unwrap()).unwrap(); - std::fs::write(&db, b"placeholder").unwrap(); + write_index_fixture(&path, b"placeholder"); TempProject { path } } @@ -1140,12 +1171,30 @@ mod tests { )); std::fs::create_dir_all(&dir).unwrap(); let paths = codegraph_core::IndexPaths::resolve(&dir, None).expect("resolve default"); + write_index_fixture_at(&paths, b"placeholder"); + assert!(matches!( + classify_resolve(codegraph_core::IndexPaths::resolve(&dir, None)), + RootStatus::Indexed + )); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A database file with no published namespace is not an index (#1895). + #[test] + fn classify_resolve_unpublished_database_file_is_absent() { + let dir = std::env::temp_dir().join(format!( + "cg-roots-classify-unpublished-{}-{}", + std::process::id(), + SEQ.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&dir).unwrap(); + let paths = codegraph_core::IndexPaths::resolve(&dir, None).expect("resolve default"); let db = paths.current_db(); std::fs::create_dir_all(db.parent().unwrap()).unwrap(); std::fs::write(&db, b"placeholder").unwrap(); assert!(matches!( classify_resolve(codegraph_core::IndexPaths::resolve(&dir, None)), - RootStatus::Indexed + RootStatus::Absent )); let _ = std::fs::remove_dir_all(&dir); } @@ -1298,10 +1347,37 @@ mod tests { } fn mark_indexed(path: &Path) { - std::fs::create_dir_all(path).unwrap(); - let db = db_path_for(path).expect("indexed fixture path resolves"); - std::fs::create_dir_all(db.parent().unwrap()).unwrap(); - std::fs::write(db, b"placeholder").unwrap(); + write_index_fixture(path, b"placeholder"); + } + + /// A stray `codegraph.db` with no published namespace — an interrupted + /// copy, a stray touch — at a workspace root must not capture discovery: + /// the sub-project scan still finds the real index below it (#1895). + #[test] + fn a_stray_database_file_does_not_capture_root_discovery() { + let workspace = unindexed_dir("stray-db"); + std::fs::create_dir(workspace.path().join(".git")).unwrap(); + let stray = db_path_for(workspace.path()).expect("workspace resolves"); + std::fs::create_dir_all(stray.parent().unwrap()).unwrap(); + std::fs::write(&stray, b"").unwrap(); + let child = workspace.path().join("service-a"); + mark_indexed(&child); + + assert!(!db_exists_for(workspace.path())); + let resolved = resolve_server_root(workspace.path(), true); + assert_eq!(resolved.root.as_deref(), Some(child.as_path())); + assert!(resolved.via_subproject_scan); + + let nested = child.join("src"); + std::fs::create_dir_all(&nested).unwrap(); + assert_eq!( + resolve_server_root(&nested, false).root.as_deref(), + Some(child.as_path()) + ); + assert!(matches!( + resolve_project_arg(workspace.path().to_str(), None, None), + ProjectArg::NotIndexed + )); } #[test] diff --git a/crates/codegraph-mcp/src/server.rs b/crates/codegraph-mcp/src/server.rs index 87a86269..52998871 100644 --- a/crates/codegraph-mcp/src/server.rs +++ b/crates/codegraph-mcp/src/server.rs @@ -748,12 +748,7 @@ mod tests { let seq = SEQ.fetch_add(1, Ordering::Relaxed); let path = std::env::temp_dir().join(format!("cg-mcp-init-{tag}-{}-{seq}", std::process::id())); - // The project dir must exist before `db_path_for` (which resolves the - // physical identity) can succeed. - std::fs::create_dir_all(&path).unwrap(); - let db = db_path_for(&path).expect("default project resolves"); - std::fs::create_dir_all(db.parent().unwrap()).unwrap(); - std::fs::write(&db, b"placeholder").unwrap(); + crate::roots::write_index_fixture(&path, b"placeholder"); TempProject { path } } @@ -769,10 +764,7 @@ mod tests { fn indexed_child(workspace: &TempProject, name: &str) -> PathBuf { let child = workspace.path().join(name); - std::fs::create_dir_all(&child).unwrap(); - let db = db_path_for(&child).expect("child project resolves"); - std::fs::create_dir_all(db.parent().unwrap()).unwrap(); - std::fs::write(db, b"placeholder").unwrap(); + crate::roots::write_index_fixture(&child, b"placeholder"); child } From d7d43b8c88eee9ea08bba433639a62ea083a2259 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 10:24:07 +0800 Subject: [PATCH 09/19] fix(wsl): keep WSL off Windows' index on /mnt drives and explain the I/O error When Windows-native and WSL CodeGraph open the same .codegraph index under /mnt//, SQLite's locking and -shm memory do not hold across the 9p/DrvFs bridge and WSL fails with a bare disk I/O error. With CODEGRAPH_DIR unset on a WSL Windows drive, the index root is now chosen per project: an existing .codegraph-wsl/ wins, an existing .codegraph/codegraph.db is kept, and a fresh index goes in .codegraph-wsl/, which init reports. A SQLite I/O error on a shared default index carries the CODEGRAPH_DIR instruction in the CLI and in MCP open failures. Every other host is unchanged (upstream #995). --- crates/codegraph-cli/src/main.rs | 19 +++ crates/codegraph-core/src/index_paths.rs | 59 +++++++++- crates/codegraph-core/src/lib.rs | 1 + crates/codegraph-core/src/wsl.rs | 55 +++++++++ crates/codegraph-mcp/src/rmcp_handler.rs | 6 +- crates/codegraph-mcp/src/server.rs | 6 +- crates/codegraph-store/src/lib.rs | 2 + .../codegraph-store/src/wsl_shared_index.rs | 109 ++++++++++++++++++ crates/codegraph-watch/src/policy.rs | 29 +---- docs/cli.md | 33 +++--- 10 files changed, 274 insertions(+), 45 deletions(-) create mode 100644 crates/codegraph-core/src/wsl.rs create mode 100644 crates/codegraph-store/src/wsl_shared_index.rs diff --git a/crates/codegraph-cli/src/main.rs b/crates/codegraph-cli/src/main.rs index 04b29781..65ca7478 100644 --- a/crates/codegraph-cli/src/main.rs +++ b/crates/codegraph-cli/src/main.rs @@ -209,6 +209,9 @@ fn cli_main() { if let Some(guidance) = index_removal_holder_guidance(&err) { eprint!("{guidance}"); } + if let Some(guidance) = codegraph_store::wsl_shared_index_guidance(err.as_ref()) { + eprint!("{guidance}"); + } std::process::exit(1); } } @@ -1438,6 +1441,22 @@ fn cmd_init( "init", )?; println!("Initialized in {}", project.display()); + // A fresh index on a Windows drive under WSL gets its own directory (#995); + // it is not the documented name, so say where it went and why. + if std::env::var_os("CODEGRAPH_DIR").is_none() + && let Ok(paths) = index_paths(&project) + && paths + .current_root() + .file_name() + .is_some_and(|name| name != codegraph_core::index_paths::DEFAULT_CURRENT_DIR) + { + println!( + "The index is in {}/: this project is on a Windows drive, so WSL keeps its own index \ + rather than share {}/ with CodeGraph on Windows. Set CODEGRAPH_DIR to choose the name yourself.", + codegraph_core::index_paths::WSL_CURRENT_DIR, + codegraph_core::index_paths::DEFAULT_CURRENT_DIR + ); + } print_index_result(&result); installer::run_install_local_targets(project, target) } diff --git a/crates/codegraph-core/src/index_paths.rs b/crates/codegraph-core/src/index_paths.rs index daa143fb..8dd63b43 100644 --- a/crates/codegraph-core/src/index_paths.rs +++ b/crates/codegraph-core/src/index_paths.rs @@ -34,6 +34,10 @@ use thiserror::Error; /// from [`IndexPaths::resolve`]. pub const DEFAULT_CURRENT_DIR: &str = ".codegraph"; +/// The index-root name WSL gives a fresh project on a Windows drive, so it +/// never shares one index with CodeGraph on Windows (upstream #995). +pub const WSL_CURRENT_DIR: &str = ".codegraph-wsl"; + /// Version byte of the binary payload hashed into the project identity. Bump /// only with a deliberate, reviewed identity-format change. const IDENTITY_PAYLOAD_VERSION: u8 = 1; @@ -117,7 +121,10 @@ impl IndexPaths { let project_identity = physical_identity(&project)?; - let directory_name = configured_directory_name(codegraph_dir)?; + let directory_name = match codegraph_dir { + None => default_directory_name(&project, crate::wsl::is_wsl_windows_drive(&project)), + Some(_) => configured_directory_name(codegraph_dir)?, + }; let candidate = lexical_normalize(&project.join(directory_name)); reject_project_alias(&project, &candidate, directory_name)?; let current_root = physical_normalize(&project, &candidate)?; @@ -259,6 +266,32 @@ fn lexical_normalize(path: &Path) -> PathBuf { out } +/// The index-root name for `project` with no `CODEGRAPH_DIR` set. Off a WSL +/// Windows drive it is always [`DEFAULT_CURRENT_DIR`]. On one, Windows-native +/// CodeGraph opens `.codegraph` in the same tree and SQLite's locking does not +/// hold across the 9p/DrvFs bridge, so sharing one index fails with "disk I/O +/// error" (upstream #995): +/// 1. `.codegraph-wsl/` exists: it, even once Windows builds a `.codegraph`; +/// 2. `.codegraph/codegraph.db` exists: `.codegraph`, so an index built before +/// this default is kept rather than silently rebuilt elsewhere; +/// 3. neither: `.codegraph-wsl`, so a fresh WSL index never shares. +fn default_directory_name(project: &Path, wsl_windows_drive: bool) -> &'static str { + if !wsl_windows_drive { + return DEFAULT_CURRENT_DIR; + } + if project.join(WSL_CURRENT_DIR).is_dir() { + return WSL_CURRENT_DIR; + } + if project + .join(DEFAULT_CURRENT_DIR) + .join("codegraph.db") + .exists() + { + return DEFAULT_CURRENT_DIR; + } + WSL_CURRENT_DIR +} + fn configured_directory_name(codegraph_dir: Option<&str>) -> Result<&str, IndexPathsError> { let Some(raw_value) = codegraph_dir else { return Ok(DEFAULT_CURRENT_DIR); @@ -692,6 +725,30 @@ mod tests { let _ = std::fs::remove_dir_all(&cache); } + #[test] + fn wsl_windows_drive_projects_default_to_their_own_index_root() { + let fresh = temp_dir("wsl-fresh"); + assert_eq!(default_directory_name(&fresh, false), DEFAULT_CURRENT_DIR); + assert_eq!(default_directory_name(&fresh, true), WSL_CURRENT_DIR); + + let legacy = temp_dir("wsl-legacy"); + std::fs::create_dir_all(legacy.join(DEFAULT_CURRENT_DIR)).unwrap(); + assert_eq!( + default_directory_name(&legacy, true), + WSL_CURRENT_DIR, + "an index root without a database is not an index to keep" + ); + std::fs::write(legacy.join(DEFAULT_CURRENT_DIR).join("codegraph.db"), b"").unwrap(); + assert_eq!(default_directory_name(&legacy, true), DEFAULT_CURRENT_DIR); + + std::fs::create_dir_all(legacy.join(WSL_CURRENT_DIR)).unwrap(); + assert_eq!( + default_directory_name(&legacy, true), + WSL_CURRENT_DIR, + "WSL keeps its own index once it has one" + ); + } + #[test] fn configured_directory_name_stays_local_to_each_project() { let project_a = temp_dir("collide-a"); diff --git a/crates/codegraph-core/src/lib.rs b/crates/codegraph-core/src/lib.rs index d4beed05..bfcb7dda 100644 --- a/crates/codegraph-core/src/lib.rs +++ b/crates/codegraph-core/src/lib.rs @@ -11,6 +11,7 @@ pub mod node_id; pub mod source_file; pub mod traits; pub mod types; +pub mod wsl; pub use errors::{CodeGraphError, Result}; pub use index_paths::{IndexPaths, IndexPathsError}; diff --git a/crates/codegraph-core/src/wsl.rs b/crates/codegraph-core/src/wsl.rs new file mode 100644 index 00000000..5fa182fa --- /dev/null +++ b/crates/codegraph-core/src/wsl.rs @@ -0,0 +1,55 @@ +//! WSL detection, shared by the index-root default (upstream #995) and the +//! watcher's `/mnt/` opt-out. + +use std::path::Path; + +/// Whether this process runs under WSL: Linux, and `WSL_DISTRO_NAME` / +/// `WSL_INTEROP` set or `/proc/version` naming Microsoft or WSL. +pub fn is_wsl() -> bool { + if !cfg!(target_os = "linux") { + return false; + } + if std::env::var_os("WSL_DISTRO_NAME").is_some() || std::env::var_os("WSL_INTEROP").is_some() { + return true; + } + std::fs::read_to_string("/proc/version") + .map(|version| { + let version = version.to_ascii_lowercase(); + version.contains("microsoft") || version.contains("wsl") + }) + .unwrap_or(false) +} + +/// Whether `path` is a Windows drive as WSL mounts it (`/mnt/c`, +/// `/mnt/d/project`). A pure path check. +pub fn is_windows_drive_mount(path: &Path) -> bool { + let normalized = path.to_string_lossy().replace('\\', "/"); + let mut parts = normalized.split('/'); + matches!( + (parts.next(), parts.next(), parts.next()), + (Some(""), Some("mnt"), Some(drive)) + if drive.len() == 1 && drive.as_bytes()[0].is_ascii_alphabetic() + ) +} + +/// Whether `path` is on a Windows drive under WSL, where Windows-native +/// CodeGraph opens the same tree. Off a `/mnt/` path this never reads +/// the environment or `/proc/version`. +pub fn is_wsl_windows_drive(path: &Path) -> bool { + is_windows_drive_mount(path) && is_wsl() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn windows_drive_mounts_are_single_letter_mnt_paths() { + assert!(is_windows_drive_mount(Path::new("/mnt/c"))); + assert!(is_windows_drive_mount(Path::new("/mnt/D/project"))); + assert!(!is_windows_drive_mount(Path::new("/mnt/abc/project"))); + assert!(!is_windows_drive_mount(Path::new("/home/user/project"))); + assert!(!is_windows_drive_mount(Path::new("/mnt"))); + assert!(!is_wsl_windows_drive(Path::new("/home/user/project"))); + } +} diff --git a/crates/codegraph-mcp/src/rmcp_handler.rs b/crates/codegraph-mcp/src/rmcp_handler.rs index 7ff303c6..c70eceae 100644 --- a/crates/codegraph-mcp/src/rmcp_handler.rs +++ b/crates/codegraph-mcp/src/rmcp_handler.rs @@ -414,8 +414,10 @@ fn execute_owned(project_path: &Path, tool_name: &str, args: &Value) -> ToolResu Ok(engine) => engine, Err(e) => { return ToolResult::error(format!( - "Failed to open project at {}: {e}", - project_path.display() + "Failed to open project at {}: {e}{}", + project_path.display(), + codegraph_store::wsl_shared_index_guidance(e.as_ref()) + .map_or(String::new(), |guidance| format!("\n{guidance}")) )); } }; diff --git a/crates/codegraph-mcp/src/server.rs b/crates/codegraph-mcp/src/server.rs index 52998871..33d9446c 100644 --- a/crates/codegraph-mcp/src/server.rs +++ b/crates/codegraph-mcp/src/server.rs @@ -579,8 +579,10 @@ impl McpServer { Err(e) => { return Dispatch::Reply( serde_json::to_value(ToolResult::error(format!( - "Failed to open project at {}: {e}", - project_path.display() + "Failed to open project at {}: {e}{}", + project_path.display(), + codegraph_store::wsl_shared_index_guidance(e.as_ref()) + .map_or(String::new(), |guidance| format!("\n{guidance}")) ))) .expect("ToolResult serializes"), ); diff --git a/crates/codegraph-store/src/lib.rs b/crates/codegraph-store/src/lib.rs index cc97cef4..255dfcdd 100644 --- a/crates/codegraph-store/src/lib.rs +++ b/crates/codegraph-store/src/lib.rs @@ -10,6 +10,7 @@ pub mod schema; #[cfg(feature = "test-hooks")] pub mod test_support; pub mod uninit; +pub mod wsl_shared_index; pub use connection::{ ExtractionStampIssue, Store, StoreError, StoreStatusOpen, StoreWriteAuthorization, @@ -33,3 +34,4 @@ pub use rebuild::{ resume_full_rebuild, }; pub use uninit::{UninitError, UninitOutcome, uninit_index, uninit_index_with_drain}; +pub use wsl_shared_index::wsl_shared_index_guidance; diff --git a/crates/codegraph-store/src/wsl_shared_index.rs b/crates/codegraph-store/src/wsl_shared_index.rs new file mode 100644 index 00000000..801f12b5 --- /dev/null +++ b/crates/codegraph-store/src/wsl_shared_index.rs @@ -0,0 +1,109 @@ +//! Windows and WSL sharing one index on a Windows drive (upstream #995). +//! +//! A repo under `/mnt//` can be used from both sides: Windows-native +//! CodeGraph and WSL CodeGraph then open the same `.codegraph/codegraph.db` +//! across the 9p/DrvFs bridge, where SQLite's file locking and its `-shm` +//! shared memory do not hold between the two systems, and WSL's connection +//! fails with a bare "disk I/O error". A fresh WSL index there already gets its +//! own `.codegraph-wsl` (`IndexPaths::resolve`); an index kept in `.codegraph` +//! that fails this way gets the instruction below instead of an unexplained +//! I/O failure. + +use std::error::Error; +use std::path::Path; + +use codegraph_core::index_paths::{DEFAULT_CURRENT_DIR, WSL_CURRENT_DIR}; + +use crate::StoreError; + +/// Guidance for `error` when a SQLite I/O failure hit the default-named index +/// of a project on a Windows drive under WSL; `None` otherwise. Only the +/// default `.codegraph` counts: it is the directory Windows CodeGraph opens +/// too, and an I/O failure on any other root has some other cause. +pub fn wsl_shared_index_guidance(error: &(dyn Error + 'static)) -> Option { + guidance_for(error, codegraph_core::wsl::is_wsl_windows_drive) +} + +fn guidance_for( + error: &(dyn Error + 'static), + on_wsl_windows_drive: impl Fn(&Path) -> bool, +) -> Option { + let mut cause = Some(error); + let db = std::iter::from_fn(|| { + let current = cause?; + cause = current.source(); + Some(current) + }) + .find_map(|error| io_failure_database(error.downcast_ref::()?))?; + let data_dir = db.parent()?; + if data_dir.file_name()? != DEFAULT_CURRENT_DIR || !on_wsl_windows_drive(db) { + return None; + } + Some(format!( + "This project is on a Windows drive, where SQLite's file locking doesn't work across \ + the Windows/WSL boundary, so this usually means CodeGraph on Windows is using the same \ + index at {}. Windows and WSL can't share one index on a Windows drive; give WSL its own:\n\ + \x20 1. Set CODEGRAPH_DIR={WSL_CURRENT_DIR} in the WSL environment (your shell profile, \ + or the env of the MCP server your agent starts).\n\ + \x20 2. Run \"codegraph init\" in WSL to build that index. Windows keeps using \ + {DEFAULT_CURRENT_DIR}.\n", + data_dir.display() + )) +} + +/// The database a SQLite I/O failure was reported against, if `error` is one. +fn io_failure_database(error: &StoreError) -> Option<&Path> { + let (path, source) = match error { + StoreError::Open { path, source } + | StoreError::Configure { path, source } + | StoreError::Migrate { path, source } + | StoreError::ReadExtractionStamp { path, source } + | StoreError::Close { path, source } + | StoreError::CheckpointWal { path, source } => (path, source), + _ => return None, + }; + is_sqlite_io_error(source).then_some(path.as_path()) +} + +/// SQLite reports an I/O failure as `SQLITE_IOERR`, whose extended codes keep +/// the primary code in their low byte. +fn is_sqlite_io_error(error: &rusqlite::Error) -> bool { + matches!( + error, + rusqlite::Error::SqliteFailure(failure, _) if failure.extended_code & 0xff == 10 + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + + fn io_error(path: &str) -> StoreError { + StoreError::Configure { + path: PathBuf::from(path), + // SQLITE_IOERR_SHMMAP: the -shm mapping the bridge cannot share. + source: rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(5130), None), + } + } + + #[test] + fn an_io_error_on_a_shared_default_index_explains_the_wsl_fix() { + let error = io_error("/mnt/c/repo/.codegraph/codegraph.db"); + let guidance = guidance_for(&error, |_| true).expect("guidance"); + assert!( + guidance.contains("CODEGRAPH_DIR=.codegraph-wsl"), + "{guidance}" + ); + assert!(guidance.contains("/mnt/c/repo/.codegraph"), "{guidance}"); + + assert_eq!(guidance_for(&error, |_| false), None, "not WSL"); + let own_root = io_error("/mnt/c/repo/.codegraph-wsl/codegraph.db"); + assert_eq!(guidance_for(&own_root, |_| true), None, "WSL's own index"); + let not_io = StoreError::Configure { + path: PathBuf::from("/mnt/c/repo/.codegraph/codegraph.db"), + source: rusqlite::Error::SqliteFailure(rusqlite::ffi::Error::new(5), None), + }; + assert_eq!(guidance_for(¬_io, |_| true), None, "SQLITE_BUSY"); + } +} diff --git a/crates/codegraph-watch/src/policy.rs b/crates/codegraph-watch/src/policy.rs index 69bd59f0..21c38406 100644 --- a/crates/codegraph-watch/src/policy.rs +++ b/crates/codegraph-watch/src/policy.rs @@ -1,4 +1,3 @@ -use std::fs; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -301,7 +300,7 @@ pub fn watch_disabled_reason(project_root: impl AsRef, no_watch: bool) -> if std::env::var("CODEGRAPH_FORCE_WATCH").as_deref() == Ok("1") { return None; } - if detect_wsl() && is_windows_drive_mount(project_root.as_ref()) { + if codegraph_core::wsl::is_wsl_windows_drive(project_root.as_ref()) { return Some( "project is on a WSL2 /mnt/ drive, where recursive fs.watch is too slow to be reliable" .to_string(), @@ -463,34 +462,12 @@ fn rule_matches(pattern: &str, relative: &str, is_dir: bool) -> bool { relative == pattern || relative.ends_with(&format!("/{pattern}")) } -fn detect_wsl() -> bool { - if !cfg!(target_os = "linux") { - return false; - } - if std::env::var_os("WSL_DISTRO_NAME").is_some() || std::env::var_os("WSL_INTEROP").is_some() { - return true; - } - fs::read_to_string("/proc/version") - .map(|version| { - let version = version.to_ascii_lowercase(); - version.contains("microsoft") || version.contains("wsl") - }) - .unwrap_or(false) -} - -fn is_windows_drive_mount(path: &Path) -> bool { - let normalized = normalize_path(path); - let mut parts = normalized.split('/'); - matches!( - (parts.next(), parts.next(), parts.next()), - (Some(""), Some("mnt"), Some(drive)) if drive.len() == 1 && drive.as_bytes()[0].is_ascii_alphabetic() - ) -} - #[cfg(test)] mod tests { use super::*; use crate::test_env::{EnvGuard, env_guard}; + use codegraph_core::wsl::is_windows_drive_mount; + use std::fs; #[test] fn watch_disabled_when_root_is_home() { diff --git a/docs/cli.md b/docs/cli.md index 22cd0c5f..393fceaf 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1028,7 +1028,12 @@ daemon mode when several agents or editor windows need the same live index. The daemon watches the project for file changes and re-indexes automatically. Changes are debounced before the re-index triggers. On WSL2, watching files under `/mnt/` is automatically disabled because recursive `fs.watch` is too slow on -those paths; the reason is surfaced in the log. +those paths; the reason is surfaced in the log. On such a Windows drive WSL also +keeps its own index: SQLite's locking does not hold across the Windows/WSL +bridge, so with `CODEGRAPH_DIR` unset an existing `.codegraph-wsl/` is used, then +an existing `.codegraph/codegraph.db` is kept, and otherwise a fresh index goes +in `.codegraph-wsl/` (`init` says so). A disk I/O error on a shared +`.codegraph/` index explains how to give WSL its own. The watcher registers per-directory watches only on non-ignored directories, pruning `node_modules`, `.venv`, `__pycache__`, `target`, `dist`, `.godot`, @@ -1078,19 +1083,19 @@ Three escape hatches: ### Environment variable reference -| Variable | Default | Clamp range | Meaning | -| ---------------------------------- | ------------ | ------------------- | ------------------------------------------------------------------------------------------------------------------ | -| `CODEGRAPH_NO_DAEMON` | — | — | Force foreground Direct mode; one indexed-project writer only, enforced by `writer.pid` | -| `CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS` | `300000` | 1000–3600000 | Exit after this long with no connected clients | -| `CODEGRAPH_DAEMON_MAX_IDLE_MS` | `1800000` | 1000–3600000 | Hard cap on total daemon lifetime when idle | -| `CODEGRAPH_DAEMON_CLIENT_SWEEP_MS` | `30000` | 50–600000 | How often the daemon sweeps for dead clients | -| `CODEGRAPH_WATCH_DEBOUNCE_MS` | `2000` | 100–60000 | File-change debounce window before a re-index triggers | -| `CODEGRAPH_NO_WATCH` | — | — | Disable the live file watcher (equivalent to `serve --no-watch`) | -| `CODEGRAPH_FORCE_WATCH` | — | — | Override WSL2 `/mnt/` auto-disable; does not override `NO_WATCH` | -| `CODEGRAPH_NO_WAL_DEFER` | — | `1` enables opt-out | Keep SQLite's default WAL autocheckpoint interval during bulk indexing | -| `CODEGRAPH_WAL_VALVE_MB` | `256` | >0; invalid→default | Shared MB threshold for the active WAL valve, resetting `journal_size_limit`, and `status` WAL warning | -| `CODEGRAPH_MCP_REGISTRY_DIR` | — | — | Override the stdio MCP registry directory read by `mcp list` | -| `CODEGRAPH_DIR` | `.codegraph` | — | Select one non-empty project-local directory name; absolute paths, separators, `.`, `..`, and aliases are rejected | +| Variable | Default | Clamp range | Meaning | +| ---------------------------------- | ------------ | ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `CODEGRAPH_NO_DAEMON` | — | — | Force foreground Direct mode; one indexed-project writer only, enforced by `writer.pid` | +| `CODEGRAPH_DAEMON_IDLE_TIMEOUT_MS` | `300000` | 1000–3600000 | Exit after this long with no connected clients | +| `CODEGRAPH_DAEMON_MAX_IDLE_MS` | `1800000` | 1000–3600000 | Hard cap on total daemon lifetime when idle | +| `CODEGRAPH_DAEMON_CLIENT_SWEEP_MS` | `30000` | 50–600000 | How often the daemon sweeps for dead clients | +| `CODEGRAPH_WATCH_DEBOUNCE_MS` | `2000` | 100–60000 | File-change debounce window before a re-index triggers | +| `CODEGRAPH_NO_WATCH` | — | — | Disable the live file watcher (equivalent to `serve --no-watch`) | +| `CODEGRAPH_FORCE_WATCH` | — | — | Override WSL2 `/mnt/` auto-disable; does not override `NO_WATCH` | +| `CODEGRAPH_NO_WAL_DEFER` | — | `1` enables opt-out | Keep SQLite's default WAL autocheckpoint interval during bulk indexing | +| `CODEGRAPH_WAL_VALVE_MB` | `256` | >0; invalid→default | Shared MB threshold for the active WAL valve, resetting `journal_size_limit`, and `status` WAL warning | +| `CODEGRAPH_MCP_REGISTRY_DIR` | — | — | Override the stdio MCP registry directory read by `mcp list` | +| `CODEGRAPH_DIR` | `.codegraph` | — | Select one non-empty project-local directory name; absolute paths, separators, `.`, `..`, and aliases are rejected. Unset on a WSL Windows drive (`/mnt//`), the default is `.codegraph-wsl` unless `.codegraph/codegraph.db` already exists, so WSL never shares Windows CodeGraph's index | Timeout/debounce values outside their clamp range are silently clamped to the nearest bound. `CODEGRAPH_WAL_VALVE_MB` instead falls back to `256` when it is From 1851a2b5042df4f6f1f3f1440e33979d8309ee8b Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 10:27:12 +0800 Subject: [PATCH 10/19] fix(daemon): retry a Windows sharing violation when replacing daemon.pid A transient Windows sharing violation (an antivirus scan or indexer holding daemon.pid open) aborted the lock publish, and the post-bind socket rewrite truncated the lock in place, so a reader could see a partial record. Both now rename a complete temp record over the lock, re-check that the lock still holds this daemon's record before every attempt, and retry only Windows sharing errors with a 25 ms to 400 ms backoff over about 1.6 s (upstream #2053, #1773). --- crates/codegraph-daemon/src/lock.rs | 196 +++++++++++++++++++++++++++- 1 file changed, 190 insertions(+), 6 deletions(-) diff --git a/crates/codegraph-daemon/src/lock.rs b/crates/codegraph-daemon/src/lock.rs index f9740283..23f82f8d 100644 --- a/crates/codegraph-daemon/src/lock.rs +++ b/crates/codegraph-daemon/src/lock.rs @@ -14,6 +14,13 @@ use crate::process::is_process_alive; const EMPTY_RETRY_DELAY: Duration = Duration::from_millis(20); +/// Waits between retries of a pid-file replace on Windows, doubling to a cap: +/// eight attempts wait about 1.6s in all, well inside a launcher's connect +/// window (upstream #2053, #1773). Windows refuses to replace a file another +/// process holds open without sharing deletion — an antivirus scan, an indexer +/// — for a moment, and for longer on a loaded machine. +const LOCK_REPLACE_RETRY_DELAYS_MS: [u64; 7] = [25, 50, 100, 200, 400, 400, 400]; + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct DaemonLockInfo { @@ -85,9 +92,17 @@ pub fn try_acquire_daemon_lock(project_root: &Path) -> Result { .create_new(true) .open(&pid_path) { - Ok(_placeholder) => { - fs::rename(&tmp, &pid_path) - .with_context(|| format!("publishing daemon lock {}", pid_path.display()))?; + Ok(placeholder) => { + drop(placeholder); + replace_owned_lock( + &pid_path, + &tmp, + "", + is_transient_sharing_violation, + |from, to| fs::rename(from, to), + thread::sleep, + ) + .with_context(|| format!("publishing daemon lock {}", pid_path.display()))?; true } Err(err) if err.kind() == ErrorKind::AlreadyExists => { @@ -118,9 +133,61 @@ pub fn rewrite_lock_socket_path(pid_path: &Path, socket_path: &Path) -> Result<( let mut info = decode_lock_info(&raw) .with_context(|| format!("decoding daemon lock {}", pid_path.display()))?; info.socket_path = socket_path.to_path_buf(); - fs::write(pid_path, encode_lock_info(&info)?) - .with_context(|| format!("rewriting daemon lock {}", pid_path.display()))?; - Ok(()) + // A complete temp record renamed over the lock, so a reader never sees a + // truncated one, and only while the lock still holds the record read above. + let tmp = pid_path.with_extension(format!("pid.{}.bound", process::id())); + fs::write(&tmp, encode_lock_info(&info)?) + .with_context(|| format!("writing temp daemon lock {}", tmp.display()))?; + replace_owned_lock( + pid_path, + &tmp, + &raw, + is_transient_sharing_violation, + |from, to| fs::rename(from, to), + thread::sleep, + ) + .with_context(|| format!("rewriting daemon lock {}", pid_path.display())) +} + +/// Replace `pid_path` with the fully written `tmp` while `pid_path` still holds +/// `expected`. A transient failure (`is_transient`) is retried per +/// [`LOCK_REPLACE_RETRY_DELAYS_MS`], re-checking ownership before every +/// attempt, so a retry never replaces a record whose owner changed meanwhile; +/// any other failure aborts at once. `tmp` is removed unless it was renamed. +fn replace_owned_lock( + pid_path: &Path, + tmp: &Path, + expected: &str, + is_transient: impl Fn(&std::io::Error) -> bool, + mut rename: impl FnMut(&Path, &Path) -> std::io::Result<()>, + mut wait: impl FnMut(Duration), +) -> Result<()> { + let outcome = (|| { + let mut delays = LOCK_REPLACE_RETRY_DELAYS_MS.iter(); + loop { + let current = fs::read_to_string(pid_path).unwrap_or_default(); + if current != expected { + anyhow::bail!("lost daemon lock ownership of {}", pid_path.display()); + } + match rename(tmp, pid_path) { + Ok(()) => return Ok(()), + Err(error) if is_transient(&error) => match delays.next() { + Some(delay) => wait(Duration::from_millis(*delay)), + None => return Err(error.into()), + }, + Err(error) => return Err(error.into()), + } + } + })(); + let _ = fs::remove_file(tmp); + outcome +} + +/// A Windows sharing failure another process's open handle causes: +/// `ERROR_ACCESS_DENIED`, `ERROR_SHARING_VIOLATION` or `ERROR_LOCK_VIOLATION`. +/// Never transient elsewhere. +fn is_transient_sharing_violation(error: &std::io::Error) -> bool { + cfg!(windows) && matches!(error.raw_os_error(), Some(5 | 32 | 33)) } pub fn clear_stale_daemon_lock(pid_path: &Path, expected_dead_pid: Option) -> bool { @@ -412,6 +479,123 @@ fn now_millis() -> u128 { mod tests { use super::*; + fn sharing_violation() -> std::io::Error { + std::io::Error::from_raw_os_error(32) + } + + /// The replace a lock publish ends with retries a transient sharing + /// violation, re-checking ownership first, and never touches a record + /// whose owner changed (upstream #2053, #1773). + #[test] + fn lock_replace_retries_transient_failures_while_the_record_is_ours() { + let base = temp_base("replace-retry"); + let pid_path = base.join("daemon.pid"); + let tmp = base.join("daemon.pid.tmp"); + fs::create_dir_all(&base).unwrap(); + fs::write(&pid_path, "ours").unwrap(); + fs::write(&tmp, "next").unwrap(); + + let mut failures = 3; + let mut waits = Vec::new(); + replace_owned_lock( + &pid_path, + &tmp, + "ours", + |_| true, + |from, to| { + if failures > 0 { + failures -= 1; + return Err(sharing_violation()); + } + fs::rename(from, to) + }, + |delay| waits.push(delay.as_millis()), + ) + .expect("transient failures are retried"); + assert_eq!(fs::read_to_string(&pid_path).unwrap(), "next"); + assert!(!tmp.exists()); + assert_eq!(waits, vec![25, 50, 100]); + + // A failure that never clears gives up after the whole budget. + fs::write(&tmp, "later").unwrap(); + let mut attempts = 0; + let mut waits = Vec::new(); + let result = replace_owned_lock( + &pid_path, + &tmp, + "next", + |_| true, + |_, _| { + attempts += 1; + Err(sharing_violation()) + }, + |delay| waits.push(delay.as_millis()), + ); + assert!(result.is_err()); + assert_eq!(attempts, 8); + assert_eq!(waits, vec![25, 50, 100, 200, 400, 400, 400]); + assert!(!tmp.exists(), "the temp record is cleaned up"); + assert_eq!(fs::read_to_string(&pid_path).unwrap(), "next"); + let _ = fs::remove_dir_all(&base); + } + + #[test] + fn lock_replace_stops_on_lost_ownership_and_on_non_transient_errors() { + let base = temp_base("replace-stop"); + let pid_path = base.join("daemon.pid"); + let tmp = base.join("daemon.pid.tmp"); + fs::create_dir_all(&base).unwrap(); + fs::write(&pid_path, "ours").unwrap(); + fs::write(&tmp, "next").unwrap(); + + let mut attempts = 0; + let result = replace_owned_lock( + &pid_path, + &tmp, + "ours", + |_| true, + |_, to| { + attempts += 1; + // Another daemon takes the lock while this one waits. + fs::write(to, "theirs").unwrap(); + Err(sharing_violation()) + }, + |_| {}, + ); + assert!(result.is_err()); + assert_eq!(attempts, 1, "a changed owner stops the retry"); + assert_eq!(fs::read_to_string(&pid_path).unwrap(), "theirs"); + + fs::write(&tmp, "next").unwrap(); + let mut attempts = 0; + let result = replace_owned_lock( + &pid_path, + &tmp, + "theirs", + |_| false, + |_, _| { + attempts += 1; + Err(std::io::Error::other("disk full")) + }, + |_| {}, + ); + assert!(result.is_err()); + assert_eq!(attempts, 1, "a non-transient error is not retried"); + assert!(!tmp.exists()); + let _ = fs::remove_dir_all(&base); + } + + #[test] + fn only_windows_sharing_errors_are_transient() { + assert_eq!( + is_transient_sharing_violation(&sharing_violation()), + cfg!(windows) + ); + assert!(!is_transient_sharing_violation(&std::io::Error::other( + "disk full" + ))); + } + #[test] fn legacy_plain_pid_decodes() { let info = decode_lock_info("1234\n").expect("pid decodes"); From a5c29a65c52dba7ac396ddb006ecffd4e1ea302e Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 10:29:04 +0800 Subject: [PATCH 11/19] fix(install): point Windows shell users at the PowerShell installer Under Git Bash, MSYS2 or Cygwin, install.sh stopped at an unsupported OS. Recognise those shells, print the exact PowerShell command and exit non-zero before downloading anything; Linux and Darwin are unchanged (upstream #1294). --- scripts/install.sh | 7 +++ scripts/tests/install-windows-shell.test.sh | 55 +++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 scripts/tests/install-windows-shell.test.sh diff --git a/scripts/install.sh b/scripts/install.sh index 3d76bf12..7cc7adae 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -35,6 +35,13 @@ fi case "$(uname -s)" in Linux) os_part="unknown-linux-musl" ;; Darwin) os_part="apple-darwin" ;; +MINGW* | MSYS* | CYGWIN*) + # A Windows POSIX shell (Git Bash, MSYS2, Cygwin): the Windows build is + # installed by the PowerShell installer. + info "codegraph: on Windows, open PowerShell and run:" + info " irm https://raw.githubusercontent.com/${REPO}/main/scripts/install.ps1 | iex" + exit 1 + ;; *) err "unsupported OS: $(uname -s) (supported: Linux, Darwin)" ;; esac case "$(uname -m)" in diff --git a/scripts/tests/install-windows-shell.test.sh b/scripts/tests/install-windows-shell.test.sh new file mode 100644 index 00000000..e86ba9f9 --- /dev/null +++ b/scripts/tests/install-windows-shell.test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# install.sh under a Windows POSIX shell (Git Bash, MSYS2, Cygwin) points at +# the PowerShell installer and exits before downloading anything (upstream +# #1294); Linux and Darwin are unchanged. +set -euo pipefail +root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)" +installer="$root/scripts/install.sh" +work=$(mktemp -d); trap 'rm -rf "$work"' EXIT +pass=0; fail=0 + +bin="$work/bin"; mkdir -p "$bin" +for tool in sh tar gzip sed head mktemp cut awk tr rm mkdir mv chmod cat sha256sum; do + src=$(command -v "$tool") && ln -s "$src" "$bin/$tool" +done +cat >"$bin/uname" <<'SH' +#!/bin/sh +case "$1" in -s) echo "$CG_UNAME_S";; -m) echo x86_64;; *) echo "$CG_UNAME_S";; esac +SH +cat >"$bin/curl" <<'SH' +#!/bin/sh +echo "$*" >>"$CG_DOWNLOADS" +exit 22 +SH +chmod +x "$bin/uname" "$bin/curl" + +run() { + rm -f "$work/downloads" + set +e + env -i PATH="$bin" HOME="$work/home" CG_UNAME_S="$1" CG_DOWNLOADS="$work/downloads" \ + /bin/sh "$installer" >"$work/out" 2>"$work/err" + rc=$?; set -e +} +check() { + if [ "$2" -eq 1 ]; then echo "PASS: $1"; pass=$((pass+1)); + else echo "FAIL: $1 (exit $rc)" >&2; cat "$work/err" >&2; fail=$((fail+1)); fi +} + +for shell_os in MINGW64_NT-10.0-26100 MSYS_NT-10.0-26100 CYGWIN_NT-10.0-26100; do + run "$shell_os" + ok=1 + [ "$rc" -ne 0 ] || ok=0 + grep -Fq 'irm https://raw.githubusercontent.com/sunerpy/codegraph-rust/main/scripts/install.ps1 | iex' "$work/err" || ok=0 + [ ! -e "$work/downloads" ] || ok=0 + check "$shell_os" "$ok" +done + +# Linux still gets past the OS check to the (failing, fake) download. +run Linux +ok=1 +[ -e "$work/downloads" ] || ok=0 +! grep -q 'install.ps1' "$work/err" || ok=0 +check Linux "$ok" + +echo "windows-shell harness: $pass passed, $fail failed" +[ "$fail" -eq 0 ] From 6d383941f3fdede46081a265229dcc4f4989ea16 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 10:32:13 +0800 Subject: [PATCH 12/19] fix(installer): keep detection read-only for configs it cannot parse Every target's detect() reads its agent's config, and install --refresh detects every target to skip the ones codegraph was never installed into, so an unparseable config of an unmanaged agent got a .backup from the read alone. Reading is now pure; the backup is made where a config is replaced (write_json_file) and before a write path skips one it cannot parse (upstream #1870). --- crates/codegraph-cli/src/installer/shared.rs | 87 +++++++++++++++++--- 1 file changed, 74 insertions(+), 13 deletions(-) diff --git a/crates/codegraph-cli/src/installer/shared.rs b/crates/codegraph-cli/src/installer/shared.rs index a9c046bb..e6615ef2 100644 --- a/crates/codegraph-cli/src/installer/shared.rs +++ b/crates/codegraph-cli/src/installer/shared.rs @@ -356,33 +356,60 @@ pub fn parse_json_object(text: &str) -> Option> { } } -/// Read an agent config file, distinguishing missing / parsed / unparseable. +/// Read an agent config file on a WRITE path, distinguishing missing / parsed / +/// unparseable. /// /// Tolerates JSONC (comments + trailing commas). A present-but-unparseable file /// is backed up to `.backup` and reported as [`ConfigRead::Unparseable`] /// WITHOUT being modified, so the caller can skip writing instead of clobbering /// the user's config. pub fn read_config_file(path: &Path) -> ConfigRead { + let read = read_config(path); + if matches!(read, ConfigRead::Unparseable) { + let _ = fs::copy(path, path.with_extension("backup")); + } + read +} + +/// Read a JSON/JSONC file into a map, `{}` when missing or unparseable. +/// +/// Read-only and silent: every target's `detect()` reads its agent's config +/// on every run, including agents codegraph was never installed into, and +/// `install --refresh` skips the unconfigured ones on that answer, so a +/// `.backup` left by a read would touch a config codegraph has no business +/// touching (upstream #1870). Callers on the WRITE path must NOT use this (it +/// cannot signal the unparseable case); use [`read_config_file`] and abort on +/// [`ConfigRead::Unparseable`] there. +pub fn read_json_file(path: &Path) -> Map { + match read_config(path) { + ConfigRead::Parsed(map) => map, + ConfigRead::Missing | ConfigRead::Unparseable => Map::new(), + } +} + +fn read_config(path: &Path) -> ConfigRead { let Ok(text) = fs::read_to_string(path) else { return ConfigRead::Missing; }; match parse_json_object(&text) { Some(map) => ConfigRead::Parsed(map), - None => { - let _ = fs::copy(path, path.with_extension("backup")); - ConfigRead::Unparseable - } + None => ConfigRead::Unparseable, } } -/// Read a JSON/JSONC file into a map. Backward-compatible helper that maps -/// [`ConfigRead::Missing`] to `{}`. Callers on the WRITE path must NOT use this -/// (it cannot signal the unparseable case); use [`read_config_file`] and abort -/// on [`ConfigRead::Unparseable`] there. -pub fn read_json_file(path: &Path) -> Map { - match read_config_file(path) { - ConfigRead::Parsed(map) => map, - ConfigRead::Missing | ConfigRead::Unparseable => Map::new(), +/// Copy a config about to be replaced to `.backup` when it does not +/// parse: a read handed the caller `{}` for it, so the write is a replacement, +/// not an edit. No-op when the file is absent or parses. +fn backup_unparseable_config(path: &Path) { + let Ok(bytes) = fs::read(path) else { + return; + }; + if std::str::from_utf8(&bytes) + .ok() + .and_then(parse_json_object) + .is_none() + { + let _ = fs::copy(path, path.with_extension("backup")); } } @@ -406,7 +433,11 @@ pub fn atomic_write_file(path: &Path, content: &str) -> std::io::Result<()> { /// Atomic JSON write with a trailing newline. Ports `writeJsonFile` /// (shared.ts:99) — `JSON.stringify(data, null, 2) + '\n'`. +/// +/// The one place a target replaces a JSON config, so it is also where an +/// unparseable one is preserved (upstream #1870). pub fn write_json_file(path: &Path, data: &Map) -> std::io::Result<()> { + backup_unparseable_config(path); let mut content = to_upstream_json(&Value::Object(data.clone())); content.push('\n'); atomic_write_file(path, &content) @@ -1757,6 +1788,36 @@ name = "important-user-tool" let _ = fs::remove_dir_all(&dir); } + /// Reading a config is read-only (upstream #1870): every target's + /// `detect()` reads its agent's config on every run, and `install + /// --refresh` skips unconfigured agents on that answer, so a read must not + /// leave a `.backup` beside a config codegraph never touches. A backup is + /// made where a config is about to be replaced, or before a write path + /// skips one it cannot parse. + #[test] + fn reading_an_unparseable_config_leaves_no_backup() { + let config = tmp_path("mcp_config.json"); + let backup = config.with_extension("backup"); + fs::write(&config, "{ broken").unwrap(); + + assert!(read_json_file(&config).is_empty()); + assert!(!backup.exists(), "a read is not a write"); + + assert!(matches!(read_config_file(&config), ConfigRead::Unparseable)); + assert_eq!(fs::read_to_string(&backup).unwrap(), "{ broken"); + fs::remove_file(&backup).unwrap(); + + let mut data = Map::new(); + data.insert("k".to_string(), Value::from(1)); + write_json_file(&config, &data).unwrap(); + assert_eq!(fs::read_to_string(&backup).unwrap(), "{ broken"); + + fs::remove_file(&backup).unwrap(); + write_json_file(&config, &data).unwrap(); + assert!(!backup.exists(), "a parseable config needs no backup"); + let _ = fs::remove_dir_all(config.parent().unwrap()); + } + fn tmp_path(name: &str) -> std::path::PathBuf { let dir = std::env::temp_dir().join(format!( "cg-jsonc-{}-{}-{}", From baae0cb347f08565cb29bae7d5d4cc135e0eb1b2 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 10:36:07 +0800 Subject: [PATCH 13/19] fix(watch): keep a failed sync's batch and owed full reconcile The event loop took the pending paths and the owed full sync before an attempt and dropped them when it failed, so a directory removal whose full sync failed never reconciled and its files kept their nodes. Keep both and retry after the debounce doubled per consecutive failure, capped at 30 s (upstream #1964). An index-lease timeout behind a long foreground index is now lock contention, backed off in place, instead of a sync failure counted toward disabling auto-sync. --- crates/codegraph-watch/src/watcher.rs | 135 +++++++++++++++++++++++++- 1 file changed, 130 insertions(+), 5 deletions(-) diff --git a/crates/codegraph-watch/src/watcher.rs b/crates/codegraph-watch/src/watcher.rs index 51fe0c2e..63f07b5c 100644 --- a/crates/codegraph-watch/src/watcher.rs +++ b/crates/codegraph-watch/src/watcher.rs @@ -1158,8 +1158,8 @@ fn event_loop(ctx: EventLoopCtx) { } Some(LoopMessage::Stop) => break, None => { - let paths = pending.keys().cloned().collect::>(); - pending.clear(); + let batch = std::mem::take(&mut pending); + let paths = batch.keys().cloned().collect::>(); deadline = None; let full_sync = std::mem::take(&mut full_sync_pending); let attempt = if full_sync { @@ -1183,6 +1183,16 @@ fn event_loop(ctx: EventLoopCtx) { if let Some(cb) = &on_sync_error { cb(reason); } + // The failed batch is still unindexed: keep its paths + // and any full reconcile it owed (a directory removal + // adds no pending file), and retry after a backoff + // rather than drop them (upstream #1964). + pending = batch; + full_sync_pending |= full_sync; + deadline = Some( + Instant::now() + + sync_retry_delay(runtime_scope.debounce, consecutive_sync_errors), + ); } SyncAttempt::Degraded(_) => {} } @@ -1339,11 +1349,32 @@ fn run_with_backoff( } } -/// A sync error is "lock contention" iff its chain mentions a busy/locked DB, -/// which is the only error worth retrying with backoff. +/// A sync error is "lock contention" iff another writer holds the index: its +/// chain mentions a busy/locked DB, or the bounded permanent-index-lock +/// acquisition timed out behind a long foreground `index`. Only contention is +/// retried in place with backoff; a timed-out lease counted as a sync failure +/// would drop the batch and push auto-sync toward being disabled. fn is_lock_contention(err: &anyhow::Error) -> bool { + use codegraph_store::{IndexLeaseError, StoreError}; + let lease_timed_out = err.chain().any(|cause| { + matches!( + cause.downcast_ref::(), + Some(IndexLeaseError::TimedOut { .. }) + ) || matches!( + cause.downcast_ref::(), + Some(StoreError::Lease(IndexLeaseError::TimedOut { .. })) + ) + }); let text = format!("{err:#}").to_ascii_lowercase(); - text.contains("locked") || text.contains("busy") + lease_timed_out || text.contains("locked") || text.contains("busy") +} + +/// How long the event loop waits before retrying a batch whose sync failed: +/// the debounce doubled per consecutive failure, capped at [`MAX_BACKOFF`] +/// (upstream `debounceMs * 2 ** (n - 1)`). +fn sync_retry_delay(debounce: Duration, consecutive_errors: u32) -> Duration { + let doublings = consecutive_errors.saturating_sub(1).min(16); + debounce.saturating_mul(1_u32 << doublings).min(MAX_BACKOFF) } fn snapshot(pending: &BTreeMap) -> Vec { @@ -2587,6 +2618,100 @@ mod tests { ); } + /// A failed sync leaves its batch unindexed. A directory removal owes a + /// full reconcile and adds no pending file, so dropping the batch on the + /// failure lost the reconcile and the removed files kept their nodes + /// (upstream #1964): both the paths and the owed full sync are kept and + /// retried after a backoff. + #[test] + fn a_failed_sync_keeps_its_batch_and_owed_full_reconcile_and_retries() { + let _env = crate::test_env::env_guard(); + let dir = crate::sync::tests::TestDir::new("watch-retry-owed-full"); + let full_calls = Arc::new(AtomicUsize::new(0)); + let full_counter = Arc::clone(&full_calls); + let full_sync_fn: FullSyncFn = Arc::new(move || { + if full_counter.fetch_add(1, AtomicOrdering::SeqCst) == 0 { + return Err(anyhow::anyhow!("disk full while removing nodes")); + } + Ok(SyncOutcome { + files_removed: 2, + ..Default::default() + }) + }); + let incremental_paths = Arc::new(Mutex::new(Vec::>::new())); + let recorded = Arc::clone(&incremental_paths); + let sync_fn: SyncFn = Arc::new(move |paths| { + let mut calls = recorded.lock().unwrap(); + calls.push(paths); + if calls.len() == 1 { + return Err(anyhow::anyhow!("parse error while re-extracting")); + } + Ok(SyncOutcome::default()) + }); + let (outcome_tx, outcome_rx) = mpsc::channel(); + let watcher = ProjectWatcher::start( + dir.path(), + WatchOptions { + debounce: Duration::from_millis(20), + inert_for_tests: true, + sync_fn: Some(sync_fn), + full_sync_fn: Some(full_sync_fn), + on_sync_complete: Some(Arc::new(move |outcome| { + outcome_tx.send(outcome).unwrap(); + })), + ..WatchOptions::default() + }, + ) + .unwrap() + .unwrap(); + + watcher.ingest_removed_dir_for_tests("src/feature"); + watcher.flush_for_tests(); + let outcome = outcome_rx + .recv_timeout(Duration::from_secs(5)) + .expect("the owed full reconcile is retried"); + assert_eq!(outcome.files_removed, 2); + assert_eq!(full_calls.load(AtomicOrdering::SeqCst), 2); + assert!(incremental_paths.lock().unwrap().is_empty()); + + watcher.ingest_event_for_tests("src/app.ts"); + watcher.flush_for_tests(); + outcome_rx + .recv_timeout(Duration::from_secs(5)) + .expect("the failed batch is retried"); + assert_eq!( + *incremental_paths.lock().unwrap(), + vec![ + vec!["src/app.ts".to_string()], + vec!["src/app.ts".to_string()] + ] + ); + assert!(watcher.pending_files().is_empty()); + watcher.stop(); + } + + /// A long foreground `index` holds the permanent index lock past a sync's + /// bounded acquisition: that is contention to back off from, not a sync + /// failure counted toward disabling auto-sync. + #[test] + fn an_index_lease_timeout_is_lock_contention() { + let timed_out = codegraph_store::IndexLeaseError::TimedOut { + path: std::path::PathBuf::from("/p/.codegraph/codegraph.lock"), + }; + assert!(is_lock_contention(&anyhow::Error::new( + codegraph_store::StoreError::Lease(timed_out) + ))); + assert!(is_lock_contention( + &anyhow::Error::new(codegraph_store::IndexLeaseError::TimedOut { + path: std::path::PathBuf::from("/p/.codegraph/codegraph.lock"), + }) + .context("opening the sync writer") + )); + assert!(!is_lock_contention(&anyhow::anyhow!( + "schema version mismatch" + ))); + } + #[test] fn event_loop_disables_auto_sync_after_persistent_errors() { let _env = crate::test_env::env_guard(); From bce672f58eb756e2f2e293b24dedc4ea37477242 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 10:50:11 +0800 Subject: [PATCH 14/19] fix(watch): recover auto-sync after lock contention and report its health Lock contention past the budget stopped the watcher for good, so after one long foreground index the daemon answered from a frozen index with no notice. The watcher now pauses as RECOVERING instead: it keeps collecting changes and retries a full reconcile every 30 s until one commits. Persistent sync failures and exhausted watch resources still stop it as DISABLED. MCP answers carry a RECOVERING or DISABLED whole-index banner, status reports the state, and while auto-sync is not healthy search, callers, callees and impact name any indexed file that changed since its last sync instead of answering from the frozen graph (upstream #876, #1959). --- crates/codegraph-mcp/Cargo.toml | 2 +- crates/codegraph-mcp/src/engine.rs | 179 +++++++++++++- crates/codegraph-watch/Cargo.toml | 5 + crates/codegraph-watch/src/lib.rs | 5 +- crates/codegraph-watch/src/watcher.rs | 323 ++++++++++++++++++++++++-- docs/mcp.md | 20 ++ 6 files changed, 514 insertions(+), 20 deletions(-) diff --git a/crates/codegraph-mcp/Cargo.toml b/crates/codegraph-mcp/Cargo.toml index dd487e4d..be8548ea 100644 --- a/crates/codegraph-mcp/Cargo.toml +++ b/crates/codegraph-mcp/Cargo.toml @@ -54,7 +54,7 @@ tokio-util = { version = "0.7" } # dependency of an integration test binary, so a Cargo feature (auto-enabled for # this crate's own dev/test builds via the dev-dependency self-reference below) # is the only reliable gate. NEVER enabled in the shipped binary. -test-hooks = ["codegraph-store/test-hooks"] +test-hooks = ["codegraph-store/test-hooks", "codegraph-watch/test-hooks"] [dev-dependencies] # Self-reference with `test-hooks` so the seams above are compiled into the lib diff --git a/crates/codegraph-mcp/src/engine.rs b/crates/codegraph-mcp/src/engine.rs index 2bd03138..4cd2af18 100644 --- a/crates/codegraph-mcp/src/engine.rs +++ b/crates/codegraph-mcp/src/engine.rs @@ -7,7 +7,7 @@ //! file:line for its API call + output template. use std::cell::RefCell; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -126,6 +126,10 @@ pub struct CodeGraphEngine { source_probes: RefCell>, /// Request-local set of files whose current bytes actually reached the result. source_citations: RefCell>, + /// Request-local set of indexed files a graph answer (search, callers, + /// callees, impact) names as locations. Checked for drift before the answer + /// is served while auto-sync is not healthy (upstream #1959). + answer_files: RefCell>, } impl CodeGraphEngine { @@ -152,6 +156,7 @@ impl CodeGraphEngine { deprioritize, source_probes: RefCell::new(HashMap::new()), source_citations: RefCell::new(HashSet::new()), + answer_files: RefCell::new(BTreeSet::new()), }) } @@ -336,6 +341,8 @@ impl CodeGraphEngine { pub fn execute(&self, tool_name: &str, args: &Value) -> ToolResult { self.source_probes.borrow_mut().clear(); self.source_citations.borrow_mut().clear(); + self.answer_files.borrow_mut().clear(); + let health = codegraph_watch::watch_health(&self.project_root); let result = match tool_name { "codegraph_search" => self.handle_search(args), "codegraph_callers" => self.handle_callers(args), @@ -350,11 +357,51 @@ impl CodeGraphEngine { other => return ToolResult::error(format!("Unknown tool: {other}")), }; match result { - Ok(tr) => self.with_staleness_banner(tr), + Ok(tr) => { + let tr = match &health { + Some(health) if *health != codegraph_watch::WatchHealth::Healthy => { + self.refuse_drifted_answer(tr) + } + _ => tr, + }; + with_health_banner(self.with_staleness_banner(tr), health.as_ref()) + } Err(e) => ToolResult::error(format!("Tool execution failed: {e}")), } } + fn mark_answer_files<'a>(&self, nodes: impl IntoIterator) { + self.answer_files + .borrow_mut() + .extend(nodes.into_iter().map(|node| node.file_path.clone())); + } + + /// While auto-sync is paused or stopped the graph is frozen: a graph answer + /// that names a file changed (or gone) since its last sync would present + /// stale edges as current, so the drifted files are named instead of the + /// answer (upstream #1959). `codegraph_node` and explore keep their own + /// per-file drift gates, which serve or omit current bytes. + fn refuse_drifted_answer(&self, result: ToolResult) -> ToolResult { + if result.is_error == Some(true) { + return result; + } + let files = std::mem::take(&mut *self.answer_files.borrow_mut()); + let stale = files + .into_iter() + .filter(|file| self.project_source(file).is_possibly_drifted()) + .collect::>(); + if stale.is_empty() { + return result; + } + let mut lines = vec![ + "⚠️ CodeGraph cannot answer from this index:".to_string(), + "These files changed or became unavailable after their last sync:".to_string(), + ]; + lines.extend(stale.iter().map(|file| format!("- {file}"))); + lines.push("Retry after a successful codegraph sync, or narrow the query.".to_string()); + ToolResult::text(lines.join("\n")) + } + /// `codegraph_check` — additive cycle-detection tool (not in the upstream pin). /// Renders `find_circular_dependencies` /// (`upstream graph/queries.ts:225-263`) as file chains. @@ -419,6 +466,7 @@ impl CodeGraphEngine { ))); } let nodes: Vec = results.into_iter().map(|r| r.node).collect(); + self.mark_answer_files(&nodes); Ok(ToolResult::text(truncate_output(&format_search_results( &nodes, )))) @@ -479,9 +527,11 @@ impl CodeGraphEngine { } } } + self.mark_answer_files(&related); Ok(related) }; let selected = grouped.groups.iter().flatten().cloned().collect::>(); + self.mark_answer_files(&selected); let godot = match dir { CallDir::Callers => self.godot_honesty(&selected)?, CallDir::Callees => GodotHonesty::default(), @@ -634,9 +684,11 @@ impl CodeGraphEngine { } } } + self.mark_answer_files(nodes.values()); Ok((order, nodes)) }; let selected = grouped.groups.iter().flatten().cloned().collect::>(); + self.mark_answer_files(&selected); let godot = self.godot_honesty(&selected)?; if grouped.groups.len() == 1 { @@ -2009,6 +2061,18 @@ impl CodeGraphEngine { )); lines.push("**Backend:** node:sqlite (Node built-in) — full WAL + FTS5".to_string()); lines.push("**Journal mode:** wal (concurrent reads safe)".to_string()); + match codegraph_watch::watch_health(&self.project_root) { + Some(codegraph_watch::WatchHealth::Healthy) => { + lines.push("**Auto-sync:** watching".to_string()); + } + Some(codegraph_watch::WatchHealth::Recovering { reason }) => lines.push(format!( + "**Auto-sync:** RECOVERING — a full catch-up has not committed yet ({reason})" + )), + Some(codegraph_watch::WatchHealth::Disabled { reason }) => { + lines.push(format!("**Auto-sync:** DISABLED ({reason})")); + } + None => {} + } let by_kind = self.store.node_counts_by_kind()?; if !by_kind.is_empty() { @@ -3276,6 +3340,37 @@ impl Cluster { // === Free-function renderers (1:1 with upstream helpers) ==================== /// `formatSearchResults` (`tools.ts:3324-3338`). +/// Prepend the whole-index notice a paused or stopped watcher owes every +/// answer (upstream #876, #1959): the per-file drift banner cannot cover edits +/// the index never heard about while auto-sync was off. +fn with_health_banner( + mut result: ToolResult, + health: Option<&codegraph_watch::WatchHealth>, +) -> ToolResult { + if result.is_error == Some(true) { + return result; + } + let banner = match health { + Some(codegraph_watch::WatchHealth::Recovering { .. }) => { + "⚠️ CodeGraph auto-sync is RECOVERING — another process held the index lock, so \ + syncing paused; changes are still being collected, but the full index catch-up \ + has not completed. Read files directly to confirm current content before relying \ + on these results." + .to_string() + } + Some(codegraph_watch::WatchHealth::Disabled { reason }) => format!( + "⚠️ CodeGraph auto-sync is DISABLED — live file watching stopped, so the index is \ + frozen and any file edited since then is stale here. Read files directly to confirm \ + current content before relying on it.\n Reason: {reason}" + ), + Some(codegraph_watch::WatchHealth::Healthy) | None => return result, + }; + if let Some(first) = result.content.first_mut() { + first.text = format!("{banner}\n\n{}", first.text); + } + result +} + fn format_search_results(nodes: &[Node]) -> String { let mut lines = vec![ format!("## Search Results ({} found)", nodes.len()), @@ -4303,6 +4398,7 @@ mod tests { deprioritize: Arc::new(DeprioritizeMatcher::default()), source_probes: RefCell::new(HashMap::new()), source_citations: RefCell::new(HashSet::new()), + answer_files: RefCell::new(BTreeSet::new()), } } @@ -5459,6 +5555,85 @@ mod tests { ); } + /// While auto-sync is paused or stopped the graph is frozen, so a graph + /// answer that names a drifted file is refused, the file is named instead, + /// and every answer carries the whole-index notice (upstream #876, #1959). + #[test] + fn graph_answers_naming_a_drifted_file_are_refused_while_auto_sync_is_off() { + let mut engine = test_engine(); + put_indexed_source(&engine, "src/a.rs", "fn alpha() {}\n", Language::Rust, 1); + put_indexed_source(&engine, "src/b.rs", "fn beta() {}\n", Language::Rust, 1); + put_nodes( + &mut engine, + &[ + node_lang( + "alpha", + "alpha", + "src/a.rs", + 1, + 1, + NodeKind::Function, + Language::Rust, + ), + node_lang( + "beta", + "beta", + "src/b.rs", + 1, + 1, + NodeKind::Function, + Language::Rust, + ), + ], + ); + write_src(&engine, "src/a.rs", "fn alpha() { changed(); }\n"); + let search = |engine: &CodeGraphEngine, query: &str| { + text_of(&engine.execute("codegraph_search", &serde_json::json!({ "query": query }))) + }; + + // No watcher in this process: per-file gates only, answers as usual. + assert!(search(&engine, "alpha").contains("alpha")); + + let guard = codegraph_watch::register_watch_health_for_tests( + &engine.project_root, + codegraph_watch::WatchHealth::Recovering { + reason: "lock held".to_string(), + }, + ); + let refused = search(&engine, "alpha"); + assert!( + refused.starts_with("⚠️ CodeGraph auto-sync is RECOVERING"), + "{refused}" + ); + assert!( + refused.contains("cannot answer from this index"), + "{refused}" + ); + assert!(refused.contains("- src/a.rs"), "{refused}"); + let fresh = search(&engine, "beta"); + assert!( + fresh.starts_with("⚠️ CodeGraph auto-sync is RECOVERING"), + "{fresh}" + ); + assert!(!fresh.contains("cannot answer"), "{fresh}"); + assert!(fresh.contains("beta"), "{fresh}"); + drop(guard); + + let _guard = codegraph_watch::register_watch_health_for_tests( + &engine.project_root, + codegraph_watch::WatchHealth::Disabled { + reason: "inotify exhausted".to_string(), + }, + ); + let disabled = search(&engine, "alpha"); + assert!( + disabled.starts_with("⚠️ CodeGraph auto-sync is DISABLED"), + "{disabled}" + ); + assert!(disabled.contains("Reason: inotify exhausted"), "{disabled}"); + assert!(disabled.contains("- src/a.rs"), "{disabled}"); + } + /// Indexing records a file over the limit by its size stamp, so the probe /// compares it by that stamp, unread: a same-size rewrite is no drift /// (upstream #1910), as it is no change to sync. diff --git a/crates/codegraph-watch/Cargo.toml b/crates/codegraph-watch/Cargo.toml index f52b5205..a3496769 100644 --- a/crates/codegraph-watch/Cargo.toml +++ b/crates/codegraph-watch/Cargo.toml @@ -13,6 +13,11 @@ rust-version.workspace = true description = "Debounced file watcher and incremental sync for CodeGraph-rs." readme = "../../README.md" +[features] +# Test seams for dependents' test builds (e.g. publishing a watcher health +# without a running watcher). NEVER enabled in the shipped binary. +test-hooks = [] + [dependencies] serde = { workspace = true } anyhow = { workspace = true } diff --git a/crates/codegraph-watch/src/lib.rs b/crates/codegraph-watch/src/lib.rs index 2ac7b8bd..97080c2d 100644 --- a/crates/codegraph-watch/src/lib.rs +++ b/crates/codegraph-watch/src/lib.rs @@ -17,8 +17,11 @@ pub use sync::{ sync_project_once, sync_project_once_cancellable, sync_project_once_with_progress, }; pub use watcher::{ - PendingFile, ProjectWatcher, WatchOptions, start_serve_watcher, watch_options_for_project, + LOCK_RECOVERY_INTERVAL, PendingFile, ProjectWatcher, WatchHealth, WatchOptions, + start_serve_watcher, watch_health, watch_options_for_project, }; +#[cfg(feature = "test-hooks")] +pub use watcher::{WatchHealthGuard, register_watch_health_for_tests}; pub use worktree::{ WorktreeIndexMismatch, detect_worktree_index_mismatch, git_worktree_root, worktree_mismatch_notice, worktree_mismatch_warning, diff --git a/crates/codegraph-watch/src/watcher.rs b/crates/codegraph-watch/src/watcher.rs index 63f07b5c..fe67c6d4 100644 --- a/crates/codegraph-watch/src/watcher.rs +++ b/crates/codegraph-watch/src/watcher.rs @@ -1,10 +1,10 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fs; use std::io; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc::{self, RecvTimeoutError, Sender}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, OnceLock}; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -477,24 +477,148 @@ pub fn next_backoff(prev: Duration) -> Duration { #[derive(Default)] struct DegradedState { degraded: AtomicBool, + /// Set while lock contention paused auto-sync: the watcher still collects + /// changes and retries a full reconcile, which clears the whole state. + recovering: AtomicBool, reason: Mutex>, } impl DegradedState { + /// Auto-sync stopped for good (until a restart). fn mark(&self, reason: String) { if let Ok(mut guard) = self.reason.lock() { *guard = Some(reason); } + self.recovering.store(false, Ordering::SeqCst); self.degraded.store(true, Ordering::SeqCst); } + /// Auto-sync paused by lock contention, recovering on its own. + fn mark_recovering(&self, reason: String) { + if let Ok(mut guard) = self.reason.lock() { + *guard = Some(reason); + } + self.recovering.store(true, Ordering::SeqCst); + self.degraded.store(true, Ordering::SeqCst); + } + + /// A committed full reconcile ends a recovery; a stopped watcher stays stopped. + fn recover(&self) { + if self.recovering.swap(false, Ordering::SeqCst) { + if let Ok(mut guard) = self.reason.lock() { + *guard = None; + } + self.degraded.store(false, Ordering::SeqCst); + } + } + fn is_degraded(&self) -> bool { self.degraded.load(Ordering::SeqCst) } + fn is_recovering(&self) -> bool { + self.recovering.load(Ordering::SeqCst) + } + fn reason(&self) -> Option { self.reason.lock().ok().and_then(|guard| guard.clone()) } + + fn health(&self) -> WatchHealth { + let reason = self.reason().unwrap_or_default(); + if !self.is_degraded() { + WatchHealth::Healthy + } else if self.is_recovering() { + WatchHealth::Recovering { reason } + } else { + WatchHealth::Disabled { reason } + } + } +} + +/// Auto-sync health of a project's live watcher in this process, for the hosts +/// that answer from its index (upstream #876, #1959). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WatchHealth { + /// Watching, and every change is synced or queued for a retry. + Healthy, + /// Another writer held the index past the contention budget. Changes are + /// still collected and a full reconcile is retried, but until one commits + /// the index may be stale. + Recovering { reason: String }, + /// Watching stopped (watch resources exhausted, or sync failing + /// persistently): the index is frozen until `codegraph sync` and a restart. + Disabled { reason: String }, +} + +type HealthRegistry = Mutex>>; + +fn health_registry() -> &'static HealthRegistry { + static REGISTRY: OnceLock = OnceLock::new(); + REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// The auto-sync health of the live watcher this process runs for +/// `project_root`, or `None` when it runs none. +pub fn watch_health(project_root: &Path) -> Option { + let key = project_root + .canonicalize() + .unwrap_or_else(|_| project_root.to_path_buf()); + let registry = health_registry().lock().ok()?; + registry + .get(&key) + .and_then(std::sync::Weak::upgrade) + .map(|state| state.health()) +} + +/// Test seam: publish `health` for `project_root` as a watcher there would, +/// for as long as the returned guard lives. +#[cfg(feature = "test-hooks")] +pub fn register_watch_health_for_tests( + project_root: &Path, + health: WatchHealth, +) -> WatchHealthGuard { + let key = project_root + .canonicalize() + .unwrap_or_else(|_| project_root.to_path_buf()); + let state = Arc::new(DegradedState::default()); + match health { + WatchHealth::Healthy => {} + WatchHealth::Recovering { reason } => state.mark_recovering(reason), + WatchHealth::Disabled { reason } => state.mark(reason), + } + register_health(&key, &state); + WatchHealthGuard { key, state } +} + +/// Keeps a [`register_watch_health_for_tests`] registration alive. +#[cfg(feature = "test-hooks")] +pub struct WatchHealthGuard { + key: PathBuf, + state: Arc, +} + +#[cfg(feature = "test-hooks")] +impl Drop for WatchHealthGuard { + fn drop(&mut self) { + unregister_health(&self.key, &self.state); + } +} + +fn register_health(project_root: &Path, state: &Arc) { + if let Ok(mut registry) = health_registry().lock() { + registry.insert(project_root.to_path_buf(), Arc::downgrade(state)); + } +} + +fn unregister_health(project_root: &Path, state: &Arc) { + if let Ok(mut registry) = health_registry().lock() + && registry + .get(project_root) + .is_some_and(|registered| std::ptr::eq(registered.as_ptr(), Arc::as_ptr(state))) + { + registry.remove(project_root); + } } #[derive(Clone)] @@ -528,8 +652,19 @@ pub struct WatchOptions { /// shutdown can refuse queued lease loops and interrupt a running one /// (frozen plan lines 598-601). cancel: SyncCancellation, + /// Cumulative backoff a sync spends waiting out another writer before + /// auto-sync pauses as RECOVERING. Defaults to [`MAX_BACKOFF`]. + pub lock_contention_budget: Duration, + /// How often a RECOVERING watcher retries its full reconcile. Defaults to + /// [`LOCK_RECOVERY_INTERVAL`]. + pub lock_recovery_interval: Duration, } +/// How often a watcher paused by lock contention retries its full reconcile, +/// so a long-lived writer is not polled in a loop (upstream #1959's re-arm +/// cooldown). +pub const LOCK_RECOVERY_INTERVAL: Duration = Duration::from_secs(30); + impl Default for WatchOptions { fn default() -> Self { let indexing = codegraph_core::config::IndexingConfig::default(); @@ -551,6 +686,8 @@ impl Default for WatchOptions { sync_fn: None, full_sync_fn: None, cancel: SyncCancellation::new(), + lock_contention_budget: MAX_BACKOFF, + lock_recovery_interval: LOCK_RECOVERY_INTERVAL, } } } @@ -597,6 +734,8 @@ pub struct ProjectWatcher { thread: Option>, watcher: SharedWatcher, degraded: Arc, + /// The canonical project root its health is registered under. + health_key: PathBuf, cancel: SyncCancellation, /// Set by the event-loop thread as its LAST action. Lets a caller observe /// completion without joining, so a bounded shutdown never blocks on a @@ -669,6 +808,9 @@ impl ProjectWatcher { }); let (tx, rx) = mpsc::channel(); let degraded = Arc::new(DegradedState::default()); + let health_key = project_root.clone(); + let lock_contention_budget = options.lock_contention_budget; + let lock_recovery_interval = options.lock_recovery_interval; // Capture directory identity before registering the backend. Windows emits // `RemoveKind::Any` after deletion, so this pre-event snapshot is the only // deterministic way to distinguish a known directory from an extensionless @@ -781,15 +923,19 @@ impl ProjectWatcher { degraded: loop_degraded, watcher: loop_watcher, known_dirs, + lock_contention_budget, + lock_recovery_interval, }); loop_finished.store(true, Ordering::SeqCst); }); + register_health(&health_key, °raded); Ok(Some(Self { tx, thread: Some(thread), watcher, degraded, + health_key, cancel, finished, })) @@ -847,6 +993,12 @@ impl ProjectWatcher { self.degraded.reason() } + /// Auto-sync health; [`WatchHealth::Recovering`] while lock contention + /// paused it and a full reconcile has not committed yet. + pub fn health(&self) -> WatchHealth { + self.degraded.health() + } + pub fn ingest_event_for_tests(&self, relative: impl Into) { let _ = self.tx.send(LoopMessage::Event(WatchEventBatch::paths(vec![ relative.into(), @@ -893,6 +1045,7 @@ impl ProjectWatcher { // Signal first (never join without cancelling: the event-loop thread may be // inside a bounded lease acquisition), then join. self.begin_shutdown(); + unregister_health(&self.health_key, &self.degraded); if let Some(thread) = self.thread.take() { let _ = thread.join(); } @@ -991,6 +1144,8 @@ struct EventLoopCtx { degraded: Arc, watcher: SharedWatcher, known_dirs: BTreeSet, + lock_contention_budget: Duration, + lock_recovery_interval: Duration, } fn event_loop(ctx: EventLoopCtx) { @@ -1008,6 +1163,8 @@ fn event_loop(ctx: EventLoopCtx) { degraded, watcher, mut known_dirs, + lock_contention_budget, + lock_recovery_interval, } = ctx; let mut pending = BTreeMap::::new(); let mut deadline = None::; @@ -1134,7 +1291,9 @@ fn event_loop(ctx: EventLoopCtx) { }); } } - if !pending.is_empty() { + // While lock contention paused auto-sync, the recovery retry keeps + // its own cadence; a burst of edits must not poll the other writer. + if !pending.is_empty() && !degraded.is_recovering() { // Resetting the timer on every event ports the upstream exactly-once // burst semantics (`upstream sync/watcher.ts:529-540`). deadline = Some(Instant::now() + runtime_scope.debounce); @@ -1158,18 +1317,27 @@ fn event_loop(ctx: EventLoopCtx) { } Some(LoopMessage::Stop) => break, None => { - let batch = std::mem::take(&mut pending); - let paths = batch.keys().cloned().collect::>(); + // Taken for the attempt; a failed or paused sync puts it back. + let mut batch = Some(std::mem::take(&mut pending)); + let paths = batch + .as_ref() + .map(|batch| batch.keys().cloned().collect::>()) + .unwrap_or_default(); deadline = None; let full_sync = std::mem::take(&mut full_sync_pending); let attempt = if full_sync { - run_full_sync_with_backoff(&full_sync_fn) + run_full_sync_with_backoff(&full_sync_fn, lock_contention_budget) } else { - run_sync_with_backoff(&sync_fn, paths.clone()) + run_sync_with_backoff(&sync_fn, paths.clone(), lock_contention_budget) }; let decision = classify_persistent_failure(&attempt, &mut consecutive_sync_errors); match attempt { SyncAttempt::Done(outcome) => { + // A committed full reconcile is what ends a recovery: the + // edits missed while another writer held the index are in. + if full_sync { + degraded.recover(); + } // Preserve the event batch independently of actual DB // mutations. Startup catch-up can win the writer lease and // index the same file first; the watcher still handled this @@ -1187,7 +1355,7 @@ fn event_loop(ctx: EventLoopCtx) { // and any full reconcile it owed (a directory removal // adds no pending file), and retry after a backoff // rather than drop them (upstream #1964). - pending = batch; + pending = batch.take().unwrap_or_default(); full_sync_pending |= full_sync; deadline = Some( Instant::now() @@ -1196,10 +1364,28 @@ fn event_loop(ctx: EventLoopCtx) { } SyncAttempt::Degraded(_) => {} } - if let PersistentFailure::Degrade(reason) | PersistentFailure::Disable(reason) = - decision - { - if !degraded.is_degraded() { + if let PersistentFailure::Degrade(reason) = decision { + // Another writer held the index past the contention budget. + // Keep watching and collecting changes, report RECOVERING, and + // retry a FULL reconcile on a slow cadence until one commits + // (upstream #1959). + if !degraded.is_recovering() + && let Some(cb) = &on_sync_error + { + cb(format!( + "auto-sync paused: {reason}; retrying a full reconcile every {}s", + lock_recovery_interval.as_secs() + )); + } + degraded.mark_recovering(reason); + pending = batch.take().unwrap_or_default(); + full_sync_pending = true; + deadline = Some(Instant::now() + lock_recovery_interval); + continue; + } + if let PersistentFailure::Disable(reason) = decision { + // A recovery that keeps failing is a stop like any other. + if !degraded.is_degraded() || degraded.is_recovering() { degraded.mark(reason.clone()); if let Some(cb) = &on_degraded { cb(reason); @@ -1297,14 +1483,14 @@ fn classify_persistent_failure( /// Run `sync_fn`, retrying on write-lock contention with bounded exponential /// backoff capped at [`MAX_BACKOFF`]. Once the cumulative sleep budget is spent /// the watcher degrades; any non-contention error is surfaced as a sync error. -fn run_sync_with_backoff(sync_fn: &SyncFn, paths: Vec) -> SyncAttempt { - run_sync_with_backoff_inner(sync_fn, paths, MAX_BACKOFF, thread::sleep) +fn run_sync_with_backoff(sync_fn: &SyncFn, paths: Vec, budget: Duration) -> SyncAttempt { + run_sync_with_backoff_inner(sync_fn, paths, budget, thread::sleep) } /// Same bounded-backoff contract as [`run_sync_with_backoff`], for the /// whole-project sync a removed directory escalates to. -fn run_full_sync_with_backoff(full_sync_fn: &FullSyncFn) -> SyncAttempt { - run_with_backoff(MAX_BACKOFF, thread::sleep, || full_sync_fn()) +fn run_full_sync_with_backoff(full_sync_fn: &FullSyncFn, budget: Duration) -> SyncAttempt { + run_with_backoff(budget, thread::sleep, || full_sync_fn()) } /// Inner retry loop with an injectable budget and sleeper so the cap can be @@ -2690,6 +2876,111 @@ mod tests { watcher.stop(); } + /// Lock contention past the budget pauses auto-sync as RECOVERING rather + /// than stopping it: changes are still collected, a FULL reconcile is + /// retried on a slow cadence, and the one that commits restores health + /// (upstream #1959). + #[test] + fn lock_contention_pauses_auto_sync_until_a_full_reconcile_commits() { + let _env = crate::test_env::env_guard(); + let dir = crate::sync::tests::TestDir::new("watch-recover-lock"); + let held = Arc::new(AtomicBool::new(true)); + let full_calls = Arc::new(AtomicUsize::new(0)); + let full_held = Arc::clone(&held); + let full_counter = Arc::clone(&full_calls); + let full_sync_fn: FullSyncFn = Arc::new(move || { + full_counter.fetch_add(1, AtomicOrdering::SeqCst); + if full_held.load(AtomicOrdering::SeqCst) { + return Err(anyhow::anyhow!("database is locked")); + } + Ok(SyncOutcome { + files_reindexed: 2, + ..Default::default() + }) + }); + let held_for_sync = Arc::clone(&held); + let sync_fn: SyncFn = Arc::new(move |_paths| { + if held_for_sync.load(AtomicOrdering::SeqCst) { + return Err(anyhow::anyhow!("database is locked")); + } + Ok(SyncOutcome::default()) + }); + let notices = Arc::new(Mutex::new(Vec::::new())); + let recorded = Arc::clone(¬ices); + let stops = Arc::new(AtomicUsize::new(0)); + let stop_counter = Arc::clone(&stops); + let (outcome_tx, outcome_rx) = mpsc::channel(); + let watcher = ProjectWatcher::start( + dir.path(), + WatchOptions { + debounce: Duration::from_millis(20), + inert_for_tests: true, + sync_fn: Some(sync_fn), + full_sync_fn: Some(full_sync_fn), + lock_contention_budget: Duration::from_millis(1), + lock_recovery_interval: Duration::from_millis(50), + on_sync_error: Some(Arc::new(move |msg| recorded.lock().unwrap().push(msg))), + on_degraded: Some(Arc::new(move |_| { + stop_counter.fetch_add(1, AtomicOrdering::SeqCst); + })), + on_sync_complete: Some(Arc::new(move |outcome| { + outcome_tx.send(outcome).unwrap(); + })), + ..WatchOptions::default() + }, + ) + .unwrap() + .unwrap(); + assert_eq!(watch_health(dir.path()), Some(WatchHealth::Healthy)); + + watcher.ingest_event_for_tests("src/app.ts"); + watcher.flush_for_tests(); + let mut recovering = false; + for _ in 0..80 { + if matches!(watcher.health(), WatchHealth::Recovering { .. }) { + recovering = true; + break; + } + std::thread::sleep(Duration::from_millis(25)); + } + assert!(recovering, "contention must pause auto-sync as recovering"); + assert!(matches!( + watch_health(dir.path()), + Some(WatchHealth::Recovering { .. }) + )); + assert!(watcher.is_degraded()); + assert_eq!( + stops.load(AtomicOrdering::SeqCst), + 0, + "recovering is not a stop" + ); + assert!( + notices.lock().unwrap()[0].contains("auto-sync paused"), + "{:?}", + notices.lock().unwrap() + ); + watcher.ingest_event_for_tests("src/other.ts"); + + held.store(false, AtomicOrdering::SeqCst); + let outcome = outcome_rx + .recv_timeout(Duration::from_secs(5)) + .expect("the full reconcile commits once the writer is gone"); + assert_eq!(outcome.files_reindexed, 2); + assert_eq!(watcher.health(), WatchHealth::Healthy); + assert_eq!(watch_health(dir.path()), Some(WatchHealth::Healthy)); + assert!( + full_calls.load(AtomicOrdering::SeqCst) >= 1, + "the recovery ran the full reconcile" + ); + assert!(watcher.pending_files().is_empty()); + watcher.stop(); + assert_eq!( + watch_health(dir.path()), + None, + "a stopped watcher is unregistered" + ); + } + /// A long foreground `index` holds the permanent index lock past a sync's /// bounded acquisition: that is contention to back off from, not a sync /// failure counted toward disabling auto-sync. diff --git a/docs/mcp.md b/docs/mcp.md index 70476e31..9683731a 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -696,3 +696,23 @@ gets **no** banner, which is what makes "trust everything not listed" a real guarantee rather than an assumption — the agent instructions had described this banner before anything produced it. Run `codegraph sync`, or wait for the watcher, if you see it on a hot codebase; ordinary drift does not require a full rebuild. + +### Auto-sync health + +When the server runs the project's live watcher in-process, every answer also +reports its health, since edits the index never heard about cannot reach the +per-file banner above: + +- **RECOVERING** — another process held the index past the sync's contention + budget (a long foreground `index`). Watching continues and changes are still + collected; a full reconcile is retried every 30 s, and until one commits each + response starts with `⚠️ CodeGraph auto-sync is RECOVERING …`. +- **DISABLED** — watching stopped (watch resources exhausted, or syncs failing + persistently). Responses start with `⚠️ CodeGraph auto-sync is DISABLED …` and + the reason; run `codegraph sync` and restart the server. + +In either state `codegraph_search`, `codegraph_callers`, `codegraph_callees` +and `codegraph_impact` check every indexed file they would name against disk, +and when one changed or disappeared since its last sync they name those files +instead of answering from the frozen graph. `codegraph_status` reports the +state as `Auto-sync:`. From c0e392197432c29ced0ff734f295110e6f28f653 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 10:53:12 +0800 Subject: [PATCH 15/19] fix(prompt-hook): require corroboration for everyday como and wie Portuguese/Spanish como and German wie are everyday words, so treating them as structural keywords fired the HIGH tier on ordinary prompts. Drop them from the unconditional list; another structural keyword, a verified code token or indexed prose segments still decide the tier (upstream #1654). --- crates/codegraph-cli/src/structural_gate.rs | 15 ++++---- crates/codegraph-cli/tests/prompt_hook.rs | 41 +++++++++++++++++++++ 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/crates/codegraph-cli/src/structural_gate.rs b/crates/codegraph-cli/src/structural_gate.rs index e884c3f5..da9cd4e7 100644 --- a/crates/codegraph-cli/src/structural_gate.rs +++ b/crates/codegraph-cli/src/structural_gate.rs @@ -34,8 +34,10 @@ use regex::Regex; /// Structural keywords matched as EXACT words (boundary on both sides): short /// or ambiguous tokens where prefix matching would false-positive ("flow" in /// "flower", "path" in "pathological"). Grouped by language; a term appears once -/// even when several languages share it ("como" is Portuguese for how AND -/// unaccented-typed Spanish "cómo"). Verbatim from `directory.ts:255-310`. +/// even when several languages share it. Ambiguous everyday words like PT/ES +/// "como" and DE "wie" are excluded: the hook instead requires another strong +/// keyword, a verified code token, or indexed prose segments (upstream #1654). +/// Verbatim from `directory.ts:255-310`. const STRUCTURAL_WORDS: &[&str] = &[ // English — the pre-#1126 list minus what moved to STRUCTURAL_STEMS: the // bare-stem entries never matched their own derived forms (`\barchitect\b` @@ -55,7 +57,7 @@ const STRUCTURAL_WORDS: &[&str] = &[ "flux", "chemins?", "casse", - // Spanish (cómo/como=how, dónde/donde=where, flujo=flow, ruta/camino=path, + // Spanish (cómo=how, dónde/donde=where, flujo=flow, ruta/camino=path, // rompe=breaks, llaman / quién llama = call(s) — bare "llama" is excluded: // it's also the animal/model name in English prompts) "cómo", @@ -68,16 +70,13 @@ const STRUCTURAL_WORDS: &[&str] = &[ "llaman", "quién llama", "quien llama", - // Portuguese (como=how — also covers unaccented Spanish; onde=where, - // fluxo=flow, caminho=path) - "como", + // Portuguese (onde=where, fluxo=flow, caminho=path) "onde", "fluxos?", "caminhos?", - // German (wie=how, wo/woher/wohin=where, Pfad=path, Fluss/Ablauf=flow, + // German (wo/woher/wohin=where, Pfad=path, Fluss/Ablauf=flow, // bricht/kaputt=breaks, ruft=calls, hängt=depends — "hängt … von X ab" // splits the separable verb "abhängen", so the "abhäng" stem can't catch it) - "wie", "wo", "woher", "wohin", diff --git a/crates/codegraph-cli/tests/prompt_hook.rs b/crates/codegraph-cli/tests/prompt_hook.rs index c3367fb6..5943bd0e 100644 --- a/crates/codegraph-cli/tests/prompt_hook.rs +++ b/crates/codegraph-cli/tests/prompt_hook.rs @@ -292,6 +292,47 @@ fn gate_no_false_fire_on_ordinary_words() { } } +/// PT/ES "como" and DE "wie" are everyday words; alone they are not structural +/// evidence. Another structural keyword still fires (upstream #1654). +#[test] +fn everyday_como_and_wie_need_corroboration() { + let (_dir, project) = index_mini("como-wie"); + let hook = |q: &str| { + let (out, err, ok) = cli(&[ + "prompt-hook", + "--path", + project.to_str().unwrap(), + "--query", + q, + ]); + assert!(ok, "failed for {q:?}: stdout={out} stderr={err}"); + out + }; + for q in [ + "eu como pizza toda sexta", + "faz como a gente combinou ontem", + "commita isso como fix, nao como feat", + "hazlo como ayer", + "mach es wie gestern", + ] { + let out = hook(q); + assert!( + out.trim().is_empty(), + "everyday {q:?} must stay silent, got:\n{out}" + ); + } + for q in [ + "como funciona a máquina de estados?", + "wie funktioniert die Zustandsmaschine?", + ] { + let out = hook(q); + assert!( + out.contains(" Date: Thu, 1 Oct 2026 10:55:07 +0800 Subject: [PATCH 16/19] fix(explore): flag dynamic imports built from a template or concatenation The JS dynamic-import form skipped any argument starting with a quote or backtick, so import(`./locales/${lang}.js`) and import('./x/' + y) read as literal imports and explore never noted the runtime boundary. A call is now dropped only when its argument is exactly one complete string literal, with escape-aware template scanning (upstream #1967). --- .../codegraph-mcp/src/dynamic_boundaries.rs | 99 ++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/crates/codegraph-mcp/src/dynamic_boundaries.rs b/crates/codegraph-mcp/src/dynamic_boundaries.rs index ed5647cf..38af9216 100644 --- a/crates/codegraph-mcp/src/dynamic_boundaries.rs +++ b/crates/codegraph-mcp/src/dynamic_boundaries.rs @@ -577,6 +577,54 @@ struct FormSpec { re: Regex, key_from: Option Option>, key_window: Option, + /// Final say on a match, for forms whose argument decides it. + accept: Option, +} + +/// Given the stripped and original bytes and the offset just past a match, +/// `false` drops the match. +type AcceptMatch = fn(&[u8], &[u8], usize) -> bool; + +/// Whether the call argument starting at `start` is anything but ONE complete +/// string literal (upstream #1967): `import('./a')` or a backtick string with +/// no substitution is an ordinary import, while a template with a `${}` +/// substitution, a concatenation or a bare expression picks the module at +/// runtime. String contents are blank in `stripped` (quotes kept), so the +/// substitution check reads `original` at the same offsets. +fn is_runtime_import_argument(stripped: &[u8], original: &[u8], start: usize) -> bool { + let skip_space = |mut at: usize| { + while stripped.get(at).is_some_and(u8::is_ascii_whitespace) { + at += 1; + } + at + }; + let open = skip_space(start); + let quote = match stripped.get(open) { + Some(b')') => return false, // `import()` — nothing to resolve + Some("e @ (b'"' | b'\'' | b'`')) => quote, + _ => return true, + }; + let Some(close) = stripped[open + 1..] + .iter() + .position(|&byte| byte == quote) + .map(|offset| open + 1 + offset) + else { + return false; + }; + if quote == b'`' { + let mut at = open + 1; + while at < close { + match original.get(at) { + // An escape pair: `\${` is literal text, but `\\${` interpolates. + Some(b'\\') => at += 2, + Some(b'$') if original.get(at + 1) == Some(&b'{') => return true, + _ => at += 1, + } + } + } + // `)` ends the call and `,` starts import()'s options argument; anything + // else (`+`, `.concat(`, …) builds the specifier at runtime. + !matches!(stripped.get(skip_space(close + 1)), Some(b')' | b',')) } fn key_computed_call(orig: &str) -> Option { @@ -638,15 +686,16 @@ fn forms() -> &'static [FormSpec] { re: Regex::new(r"[\w$)\]]\s*\[([^\[\]\n]{1,80})\]\s*\(").expect("computed-call"), key_from: Some(key_computed_call), key_window: None, + accept: None, }, FormSpec { form: "dynamic-import", label: "dynamic import", langs: Some(JS_FAMILY), - re: Regex::new(r#"\b(?:import|require)\s*\(\s*(?:[^\s'"`)])"#) - .expect("dynamic-import-js"), + re: Regex::new(r"\b(?:import|require)\s*\(").expect("dynamic-import-js"), key_from: None, key_window: None, + accept: Some(is_runtime_import_argument), }, FormSpec { form: "dynamic-import", @@ -656,6 +705,7 @@ fn forms() -> &'static [FormSpec] { .expect("dynamic-import-py"), key_from: None, key_window: None, + accept: None, }, FormSpec { form: "ruby-send", @@ -665,6 +715,7 @@ fn forms() -> &'static [FormSpec] { .expect("ruby-send"), key_from: Some(key_ruby_send), key_window: None, + accept: None, }, FormSpec { form: "php-dynamic", @@ -676,6 +727,7 @@ fn forms() -> &'static [FormSpec] { .expect("php-dynamic"), key_from: Some(key_single_literal), key_window: Some(80), + accept: None, }, FormSpec { form: "reflection", @@ -687,6 +739,7 @@ fn forms() -> &'static [FormSpec] { .expect("reflection"), key_from: Some(key_single_literal), key_window: Some(80), + accept: None, }, FormSpec { form: "proxy-reflect", @@ -696,6 +749,7 @@ fn forms() -> &'static [FormSpec] { .expect("proxy-reflect"), key_from: None, key_window: None, + accept: None, }, FormSpec { form: "typed-bus", @@ -707,6 +761,7 @@ fn forms() -> &'static [FormSpec] { .expect("typed-bus"), key_from: Some(key_typed_bus), key_window: None, + accept: None, }, FormSpec { form: "var-key-dispatch", @@ -718,6 +773,7 @@ fn forms() -> &'static [FormSpec] { .expect("var-key-dispatch"), key_from: None, key_window: None, + accept: None, }, FormSpec { form: "selector", @@ -727,6 +783,7 @@ fn forms() -> &'static [FormSpec] { .expect("selector"), key_from: Some(key_selector), key_window: None, + accept: None, }, ] }) @@ -1008,6 +1065,11 @@ pub fn scan_dynamic_dispatch( continue; } for m in spec.re.find_iter(&stripped_str) { + if let Some(accept) = spec.accept + && !accept(&stripped, original, m.end()) + { + continue; + } let m_index = m.start(); let mut slice_end = m.end(); if let Some(window) = spec.key_window { @@ -1057,6 +1119,39 @@ pub fn scan_dynamic_dispatch( mod tests { use super::*; + /// A JS dynamic import is a runtime boundary unless its argument is exactly + /// one complete string literal (upstream #1967). + #[test] + fn js_dynamic_import_flags_templates_concatenations_and_expressions() { + let flagged = |body: &str| { + scan_dynamic_dispatch(body, "typescript", 1) + .iter() + .any(|m| m.form == "dynamic-import") + }; + for runtime in [ + "const m = await import(`./locales/${lang}.js`);", + "const m = await import('./x/' + y);", + "const m = await import(\"./a\".concat(b));", + "const m = await import(name);", + "const m = require('./plugins/' + id);", + "const m = await import(`\\\\${dir}/a.js`);", + ] { + assert!(flagged(runtime), "{runtime}"); + } + for literal in [ + "const m = await import('./a');", + "const m = await import(\"./a\");", + "const m = await import(`./a.js`);", + "const m = await import('./data.json', { with: { type: 'json' } });", + "const m = await import(`\\${literal}.js`);", + "const m = await import( './spaced' );", + "const m = require('./a');", + "import();", + ] { + assert!(!flagged(literal), "{literal}"); + } + } + fn one(body: &str, lang: &str, start: i64) -> BoundaryMatch { let mut m = scan_dynamic_dispatch(body, lang, start); assert_eq!(m.len(), 1, "expected exactly one match for {body:?}"); From 1fe6c5cd940845da4e9c57c3735adadb8b3e79b8 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 10:58:02 +0800 Subject: [PATCH 17/19] perf(resolve): answer each file-existence probe once per pass Import resolution probes every extension of a specifier from every file that imports it, and each probe queried the store and stat'd the disk. Memoize the answer for the pass in both contexts, bounded at 200,000 entries as upstream's fileExistsMemo is; neither the borrowed store nor the tree changes within a pass. --- crates/codegraph-resolve/src/context.rs | 41 ++++++++++++------- .../codegraph-resolve/src/snapshot_context.rs | 11 ++++- .../tests/resolution_file_reads.rs | 28 +++++++++++++ 3 files changed, 64 insertions(+), 16 deletions(-) diff --git a/crates/codegraph-resolve/src/context.rs b/crates/codegraph-resolve/src/context.rs index 06e4c809..8aa550d6 100644 --- a/crates/codegraph-resolve/src/context.rs +++ b/crates/codegraph-resolve/src/context.rs @@ -51,8 +51,15 @@ struct Caches { workspace_packages: Option>, all_files: Option>>, files_by_basename: Option>>>, + /// Existence answers for the pass: import resolution probes every extension + /// of a specifier from every importing file, and neither the borrowed store + /// nor the tree changes within a pass (upstream `fileExistsMemo`). + file_exists: HashMap, } +/// Bound on memoized existence answers; the memo restarts past it, as upstream. +pub(crate) const FILE_EXISTS_MEMO_LIMIT: usize = 200_000; + impl Caches { fn new() -> Self { let limit = DEFAULT_CACHE_LIMIT; @@ -73,6 +80,7 @@ impl Caches { workspace_packages: None, all_files: None, files_by_basename: None, + file_exists: HashMap::new(), } } } @@ -110,6 +118,7 @@ impl<'a> StoreResolutionContext<'a> { c.qualified_name_cache.clear(); c.all_files = None; c.files_by_basename = None; + c.file_exists.clear(); } fn cached_all_files(&self) -> Arc> { @@ -240,24 +249,28 @@ impl ResolutionContext for StoreResolutionContext<'_> { } fn file_exists(&self, file_path: &str) -> bool { + if let Some(&exists) = self.caches.borrow().file_exists.get(file_path) { + return exists; + } // Known-file fast path then filesystem fallback (index.ts:358-374). // The store is the index of known files. - if self.store.file_by_path(file_path).ok().flatten().is_some() { - return true; - } let normalized = file_path.replace('\\', "/"); - if normalized != file_path - && self - .store - .file_by_path(&normalized) - .ok() - .flatten() - .is_some() - { - return true; + let exists = self.store.file_by_path(file_path).ok().flatten().is_some() + || (normalized != file_path + && self + .store + .file_by_path(&normalized) + .ok() + .flatten() + .is_some()) + || pathutil::lexical_path_within_root(&self.project_root, file_path) + .is_some_and(|full_path| full_path.exists()); + let mut c = self.caches.borrow_mut(); + if c.file_exists.len() >= FILE_EXISTS_MEMO_LIMIT { + c.file_exists.clear(); } - pathutil::lexical_path_within_root(&self.project_root, file_path) - .is_some_and(|full_path| full_path.exists()) + c.file_exists.insert(file_path.to_string(), exists); + exists } fn read_file(&self, file_path: &str) -> Option { diff --git a/crates/codegraph-resolve/src/snapshot_context.rs b/crates/codegraph-resolve/src/snapshot_context.rs index bf297c3b..53ab2e08 100644 --- a/crates/codegraph-resolve/src/snapshot_context.rs +++ b/crates/codegraph-resolve/src/snapshot_context.rs @@ -174,6 +174,10 @@ struct SnapshotCaches { source_facts: BoundedFileMemo>>, import_mappings: BoundedFileMemo>>, re_exports: BoundedFileMemo>>, + /// Filesystem existence probes for paths the snapshot does not know: the + /// same candidates are probed from every importing file, and the tree does + /// not change within a pass (upstream `fileExistsMemo`). + fs_exists: BoundedFileMemo, #[cfg(test)] counters: SnapshotCacheCounters, } @@ -188,6 +192,7 @@ impl SnapshotCaches { source_facts: BoundedFileMemo::new(content_limit), import_mappings: BoundedFileMemo::new(DEFAULT_CACHE_LIMIT), re_exports: BoundedFileMemo::new(DEFAULT_CACHE_LIMIT), + fs_exists: BoundedFileMemo::new(crate::context::FILE_EXISTS_MEMO_LIMIT), #[cfg(test)] counters: SnapshotCacheCounters::default(), } @@ -499,8 +504,10 @@ impl ResolutionContext for SnapshotResolutionContext { if normalized != file_path && self.snapshot.known_file_paths.contains(&normalized) { return true; } - pathutil::lexical_path_within_root(&self.snapshot.project_root, file_path) - .is_some_and(|full_path| full_path.exists()) + self.caches.fs_exists.get_or_init(file_path, || { + pathutil::lexical_path_within_root(&self.snapshot.project_root, file_path) + .is_some_and(|full_path| full_path.exists()) + }) } fn read_file(&self, file_path: &str) -> Option { diff --git a/crates/codegraph-resolve/tests/resolution_file_reads.rs b/crates/codegraph-resolve/tests/resolution_file_reads.rs index 9b064684..8dc853e8 100644 --- a/crates/codegraph-resolve/tests/resolution_file_reads.rs +++ b/crates/codegraph-resolve/tests/resolution_file_reads.rs @@ -253,3 +253,31 @@ fn framework_extraction_skips_a_file_too_large_to_extract() { .collect::>(); assert_eq!(sources, vec!["src/app.ts"]); } + +/// Import resolution probes the same candidate paths (every extension of a +/// specifier) from every file that imports it, and the tree does not change +/// within a pass, so a pass answers each existence probe once (upstream +/// v1.6.1's `fileExistsMemo`); the next pass probes afresh. +#[test] +fn existence_probes_are_answered_once_per_pass() { + let project = Project::new("exists-memo"); + project.write("present.ts", "export const a = 1;\n"); + let (store_context, snapshot) = contexts(&project); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert!(context.file_exists("present.ts")); + assert!(!context.file_exists("later.ts")); + } + + project.write("later.ts", "export const b = 2;\n"); + std::fs::remove_file(project.root.join("present.ts")).expect("remove"); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert!(context.file_exists("present.ts"), "memoized for the pass"); + assert!(!context.file_exists("later.ts"), "memoized for the pass"); + } + + let (store_context, snapshot) = contexts(&project); + for context in [&store_context as &dyn ResolutionContext, &snapshot] { + assert!(!context.file_exists("present.ts")); + assert!(context.file_exists("later.ts")); + } +} From 07ecb2c3d4fa7d37dbe2cd6bb731ad9c81b91d02 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 11:05:05 +0800 Subject: [PATCH 18/19] docs(upstream): record where the PR-5 v1.6.1 rows landed --- docs/upstream-sync/V1_6_1_AUDIT_2026-09-30.md | 100 +++++++++--------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/docs/upstream-sync/V1_6_1_AUDIT_2026-09-30.md b/docs/upstream-sync/V1_6_1_AUDIT_2026-09-30.md index fa994fb8..798d9318 100644 --- a/docs/upstream-sync/V1_6_1_AUDIT_2026-09-30.md +++ b/docs/upstream-sync/V1_6_1_AUDIT_2026-09-30.md @@ -20,38 +20,38 @@ documented divergence). ## Resolution and extraction -| Issue / PR | Upstream | Behavior | Disposition | State | -| --------------------------------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| #1862 (a) | `4297b8e` | an identifier-rooted TS/JS chain of three or more segments keeps its qualified call site; only a framework resolver may bind it | CORRECT | landed in #277 | -| #1862 (b) | `4297b8e` | typed method lookup matches within a language family (`.tsx` reaches `.ts`, `.js` reaches `.jsx`) | PORT | landed in #275 | -| #1862 (c), #647 | `4297b8e`, `68eaf0d` | Zustand `get()`, `useStore.getState()`, destructured and selector bindings resolve inside one identified store; requires the store-initializer action extraction the port never had | PORT | landed in #277 | -| #1987 | `b65e05b` | `this.#field.m()` resolves on the private field's type; JS built-in method names need receiver evidence | PORT | landed in #277 | -| #1986 | `9649ce1` | code references crossing code families need bridge evidence; TS/JS receiver wrappers peel and untyped expression receivers emit nothing | PORT | landed in #277 | -| #1857 | `53fe53d` | a receiver-less Go call never binds a method | CORRECT | landed in #275 | -| #1882 | `8f80819` | Rust `self.m()` with same-named owners in several modules pins the caller's file | CORRECT | landed in #275 | -| #1885 / #1840 | `d0996a2` | awaited receiver inference: block scope, reassignment, call tail, imported or reachable factory, `Promise` | CORRECT | landed in #275 | -| #1824 | `e27d6da` | a Scala object is never a supertype target but owns methods and supertypes | CORRECT | landed in #275 | -| #2040 | `9be3f1a` | an unknown member of a named Python import stays unresolved | PORT | landed in #277 | -| #2055 | `4960720` | an inheritance ref that lands on a TS value moves to the same-file same-named type | PORT | landed in #277 | -| #1932 | `17d09bf` | object-literal members that alias a function follow the binding lexically (last own property, spread/computed invalidation, shadowing) | CORRECT | landed in #277 | -| #1747 | `9fb2c25` | a curried-wrapper function that is an object member is named by its key | CORRECT | landed in #277 | -| #1905 | `bd993b1` | Python module/class/function body docstrings (PEP 257), joined after a preceding comment | PORT | landed in #277 | -| #1373 | `9b5c8e4` | single-argument C/C++ function macros recover the defined function name with local `#define` evidence | PORT | landed in #277 | -| #1820 | `5eaa6fe` | Python attribute and Go selector method values keep their receiver and resolve through type/import scope | CORRECT | receiver-qualified JS/TS/Python values landed in #275, the Python named-import module fallback in #277, Go selector values and Python receiver scoping in #278 | -| #1838 / #1839 / #2069 / #2070 | `fa25883`, `15a5d1d` | translation-unit-ordered macro visibility; `extern`, array and `initializer_list` constructor rules; macro-name calls bind only functions | CORRECT | constructor/macro basics landed in #275; translation-unit macro visibility, the newer constructor rules, #2069 and #2070 planned | -| #1974 | `52df4d2` | depth-limited traversal re-expands a node reached again at a shallower depth | PORT | landed in #278 | -| #1349 | `86708bf` | a later same-line, same-name declaration gets a column-suffixed node id | PORT, touches the node-id invariant | landed in #278 (approved by the owner on 2026-10-01) | -| #2072 | `fef3776` | performance: memoised gates and caches, graph output unchanged | partial PORT | `this.` declaration memo landed in #277; per-file source facts in #275; the `file_exists` miss cache is planned | -| #1719 | (09-15 row) | sealed-module markers read after blanking | CORRECT | landed in #275 | -| #1642 | `4ebd0d6` | `src/*/{java,kotlin,scala}/**/build/` is source, not build output | PORT | planned | -| #1553 | `276df3a` | resolver file reads stat first and skip non-regular or oversized files | PORT | planned | -| #1910 | `04ec5f5` | bounded extraction reads and MPEG-TS sniffing for `.ts` | PORT | planned | -| #1209 / #1222 | `9e4b8e7` | MyBatis closing tags tolerate whitespace | PORT (refid half ALREADY-HAVE) | planned | -| #1906 | `c505d0d` | Liquid `{% liquid %}` blocks | PORT | planned | -| #1360 | `6184610` | sync reports recovered orphan references | PORT | planned | -| #1829 / #1878 | `4871114` | git-stamped fast path for pending status | KEEP-RUST (full inventory); fast path DEFER pending measurement | — | -| #1833 / #1849, #1830, #1784, #1823, #1519, #1350, #1348, #1359, #1361 | various | already implemented by the port | ALREADY-HAVE | — | -| #1827, #1773, #1864, #1973 / #1543, #1355 / #1988 / #2038 | various | Spring DI heuristics, Node `indexAll` spin, dominant-file SQL, dead-code, callback synthesis storage | N/A | — | +| Issue / PR | Upstream | Behavior | Disposition | State | +| --------------------------------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| #1862 (a) | `4297b8e` | an identifier-rooted TS/JS chain of three or more segments keeps its qualified call site; only a framework resolver may bind it | CORRECT | landed in #277 | +| #1862 (b) | `4297b8e` | typed method lookup matches within a language family (`.tsx` reaches `.ts`, `.js` reaches `.jsx`) | PORT | landed in #275 | +| #1862 (c), #647 | `4297b8e`, `68eaf0d` | Zustand `get()`, `useStore.getState()`, destructured and selector bindings resolve inside one identified store; requires the store-initializer action extraction the port never had | PORT | landed in #277 | +| #1987 | `b65e05b` | `this.#field.m()` resolves on the private field's type; JS built-in method names need receiver evidence | PORT | landed in #277 | +| #1986 | `9649ce1` | code references crossing code families need bridge evidence; TS/JS receiver wrappers peel and untyped expression receivers emit nothing | PORT | landed in #277 | +| #1857 | `53fe53d` | a receiver-less Go call never binds a method | CORRECT | landed in #275 | +| #1882 | `8f80819` | Rust `self.m()` with same-named owners in several modules pins the caller's file | CORRECT | landed in #275 | +| #1885 / #1840 | `d0996a2` | awaited receiver inference: block scope, reassignment, call tail, imported or reachable factory, `Promise` | CORRECT | landed in #275 | +| #1824 | `e27d6da` | a Scala object is never a supertype target but owns methods and supertypes | CORRECT | landed in #275 | +| #2040 | `9be3f1a` | an unknown member of a named Python import stays unresolved | PORT | landed in #277 | +| #2055 | `4960720` | an inheritance ref that lands on a TS value moves to the same-file same-named type | PORT | landed in #277 | +| #1932 | `17d09bf` | object-literal members that alias a function follow the binding lexically (last own property, spread/computed invalidation, shadowing) | CORRECT | landed in #277 | +| #1747 | `9fb2c25` | a curried-wrapper function that is an object member is named by its key | CORRECT | landed in #277 | +| #1905 | `bd993b1` | Python module/class/function body docstrings (PEP 257), joined after a preceding comment | PORT | landed in #277 | +| #1373 | `9b5c8e4` | single-argument C/C++ function macros recover the defined function name with local `#define` evidence | PORT | landed in #277 | +| #1820 | `5eaa6fe` | Python attribute and Go selector method values keep their receiver and resolve through type/import scope | CORRECT | receiver-qualified JS/TS/Python values landed in #275, the Python named-import module fallback in #277, Go selector values and Python receiver scoping in #278 | +| #1838 / #1839 / #2069 / #2070 | `fa25883`, `15a5d1d` | translation-unit-ordered macro visibility; `extern`, array and `initializer_list` constructor rules; macro-name calls bind only functions | CORRECT | constructor/macro basics landed in #275; translation-unit macro visibility, the newer constructor rules, #2069 and #2070 planned | +| #1974 | `52df4d2` | depth-limited traversal re-expands a node reached again at a shallower depth | PORT | landed in #278 | +| #1349 | `86708bf` | a later same-line, same-name declaration gets a column-suffixed node id | PORT, touches the node-id invariant | landed in #278 (approved by the owner on 2026-10-01) | +| #2072 | `fef3776` | performance: memoised gates and caches, graph output unchanged | partial PORT | `this.` declaration memo landed in #277; per-file source facts in #275; the `file_exists` memo in #279 | +| #1719 | (09-15 row) | sealed-module markers read after blanking | CORRECT | landed in #275 | +| #1642 | `4ebd0d6` | `src/*/{java,kotlin,scala}/**/build/` is source, not build output | PORT | landed in #279, with the root `.gitignore` read by git's rules in scan and watch (both matchers diverged from git before) | +| #1553 | `276df3a` | resolver file reads stat first and skip non-regular or oversized files | PORT | landed in #279 | +| #1910 | `04ec5f5` | bounded extraction reads and MPEG-TS sniffing for `.ts` | PORT | landed in #279, with lossy UTF-8 decoding: a non-UTF-8 source file failed the whole index before (a Rust-only defect) | +| #1209 / #1222 | `9e4b8e7` | MyBatis closing tags tolerate whitespace | PORT (refid half ALREADY-HAVE), CORRECT for #1182 | landed in #279 with the rest of #1182 (comments, quote styles, namespace-less ``, iBatis verbs, `databaseId`); KEEP-RUST: a self-closing `` is never a root | +| #1906 | `c505d0d` | Liquid `{% liquid %}` blocks | PORT | landed in #279 | +| #1360 | `6184610` | sync reports recovered orphan references | PORT | landed in #279 | +| #1829 / #1878 | `4871114` | git-stamped fast path for pending status | KEEP-RUST (full inventory); fast path DEFER pending measurement | — | +| #1833 / #1849, #1830, #1784, #1823, #1519, #1350, #1348, #1359, #1361 | various | already implemented by the port | ALREADY-HAVE | — | +| #1827, #1773, #1864, #1973 / #1543, #1355 / #1988 / #2038 | various | Spring DI heuristics, Node `indexAll` spin, dominant-file SQL, dead-code, callback synthesis storage | N/A | — | Findings outside the upstream range, recorded so the next audit starts from them: callers/callees never followed `instantiates` edges (upstream #774/#804, @@ -79,28 +79,28 @@ KEEP-RUST divergences taken in this PR: ## MCP, CLI, installer, lifecycle and explore -| Issue / PR | Upstream | Behavior | Disposition | State | -| ------------------------------------------------------------------------------------ | ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | -| #1832 | `cba44c9` | only a prompt that is entirely a task-notification envelope is skipped | CORRECT | landed in #275 | -| #1967 | `94ae04d` | dynamic-import boundaries: template or concatenated specifiers are runtime imports | PORT | planned | -| #1964 | `a9c8808` | a failed sync keeps the pending paths and owed full scan and retries | PORT | planned | -| #1959 | `2268a09` | restart a degraded watcher, RECOVERING/DISABLED banners, refuse drifted answers while degraded | PORT | planned | -| #1654 | `4fd3e6d` | `como`/`wie` leave the unconditional structural keyword list | PORT | planned | -| #1870 | `98718cf` | installer `detect()` is read-only; backups only when a config is rewritten | PORT | planned | -| #1839 (CLI) | `fa25883` | callers/callees JSON carries each node's `relationships` | PORT | landed in #278, with callers/callees following `instantiates` edges (upstream #774/#804, never ported before) | -| #1895 | `ee97a57` | MCP root discovery ignores a `codegraph.db` without a schema | PORT (CLI ALREADY-HAVE) | planned | -| #995 | `c13bd08` | WSL `/mnt/` defaults to `.codegraph-wsl` and maps SQLITE_IOERR to guidance | PORT | planned | -| #2053 / #1773 | `0a9f3de`, `c405ac5` | bounded retry when Windows sharing violations block `daemon.pid` replacement | PORT | planned | -| #1294 | `d3671fa` | `install.sh` under MINGW/MSYS/CYGWIN points at the PowerShell installer | PORT | planned | -| #1904, PR #2062, PR #2063, PR #2068, PR #2071, PR #2077 | `9ad6ee9`, `adad840`, `47840a6`, `724b5de`, `290e03f`, `003305f` | explore: empty-result lexical diagnostics, cluster budget order, line anchors as exact targets, single-file budget, same-basename pins, honest completion notes | PORT | planned | -| #1834, #1831, #1830, #1968, #1372, #1361, #1835, #1963 / #1356, #1325, #1902 | various | already implemented by the port | ALREADY-HAVE | — | -| #770 | `c897daf` | watch symlinked directories | DEFER until the scan's symlink policy is settled (the scan does not follow symlinked directories) | — | -| telemetry, npm/fnm, Node timers, `FILE_NOTIFY_CHANGE_LAST_ACCESS`, test-only commits | various | no Rust analogue | N/A | — | +| Issue / PR | Upstream | Behavior | Disposition | State | +| ------------------------------------------------------------------------------------ | ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| #1832 | `cba44c9` | only a prompt that is entirely a task-notification envelope is skipped | CORRECT | landed in #275 | +| #1967 | `94ae04d` | dynamic-import boundaries: template or concatenated specifiers are runtime imports | PORT | landed in #279 | +| #1964 | `a9c8808` | a failed sync keeps the pending paths and owed full scan and retries | PORT | landed in #279, with the index-lease timeout classified as contention | +| #1959 | `2268a09` | restart a degraded watcher, RECOVERING/DISABLED banners, refuse drifted answers while degraded | PORT | landed in #279 as an in-loop recovery (the watcher keeps collecting events and retries a full reconcile every 30 s), with the #876 banner never ported before | +| #1654 | `4fd3e6d` | `como`/`wie` leave the unconditional structural keyword list | PORT | landed in #279 | +| #1870 | `98718cf` | installer `detect()` is read-only; backups only when a config is rewritten | PORT | landed in #279 | +| #1839 (CLI) | `fa25883` | callers/callees JSON carries each node's `relationships` | PORT | landed in #278, with callers/callees following `instantiates` edges (upstream #774/#804, never ported before) | +| #1895 | `ee97a57` | MCP root discovery ignores a `codegraph.db` without a schema | PORT (CLI ALREADY-HAVE) | landed in #279 | +| #995 | `c13bd08` | WSL `/mnt/` defaults to `.codegraph-wsl` and maps SQLITE_IOERR to guidance | PORT | landed in #279 | +| #2053 / #1773 | `0a9f3de`, `c405ac5` | bounded retry when Windows sharing violations block `daemon.pid` replacement | PORT | landed in #279 | +| #1294 | `d3671fa` | `install.sh` under MINGW/MSYS/CYGWIN points at the PowerShell installer | PORT | landed in #279 | +| #1904, PR #2062, PR #2063, PR #2068, PR #2071, PR #2077 | `9ad6ee9`, `adad840`, `47840a6`, `724b5de`, `290e03f`, `003305f` | explore: empty-result lexical diagnostics, cluster budget order, line anchors as exact targets, single-file budget, same-basename pins, honest completion notes | PORT | planned | +| #1834, #1831, #1830, #1968, #1372, #1361, #1835, #1963 / #1356, #1325, #1902 | various | already implemented by the port | ALREADY-HAVE | — | +| #770 | `c897daf` | watch symlinked directories | DEFER until the scan's symlink policy is settled (the scan does not follow symlinked directories) | — | +| telemetry, npm/fnm, Node timers, `FILE_NOTIFY_CHANGE_LAST_ACCESS`, test-only commits | various | no Rust analogue | N/A | — | A Rust-only defect found by this pass: the watcher treats only `locked`/`busy` errors as lock contention, but an index-lease timeout reports `timed out acquiring permanent index lock`, so a long foreground `index` makes the watcher -drop the batch and count a failure. It is planned with #1964. +drop the batch and count a failure. It landed with #1964 in #279. ## UI family (re-triage) From fd5fc42b29a8055e3449caad9dc21f3b84f74087 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 11:16:02 +0800 Subject: [PATCH 19/19] test(resolve): keep the FIFO-only Path import out of non-unix builds --- crates/codegraph-resolve/tests/resolution_file_reads.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/codegraph-resolve/tests/resolution_file_reads.rs b/crates/codegraph-resolve/tests/resolution_file_reads.rs index 8dc853e8..9c68ef0d 100644 --- a/crates/codegraph-resolve/tests/resolution_file_reads.rs +++ b/crates/codegraph-resolve/tests/resolution_file_reads.rs @@ -2,7 +2,7 @@ //! a regular file no larger than the extraction size limit, checked before any //! byte is read, with the rejection cached for the pass like any other read. -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; use codegraph_core::config::DEFAULT_MAX_FILE_SIZE; @@ -203,7 +203,7 @@ fn a_fifo_is_rejected_without_blocking() { let db = project.root.join("index.db"); let (sender, receiver) = std::sync::mpsc::channel(); std::thread::spawn(move || { - let store = Store::open(Path::new(&db)).expect("open store"); + let store = Store::open(&db).expect("open store"); let store_read = StoreResolutionContext::new(&store, root.clone()).read_file("pipe.ts"); let snapshot_read = SnapshotResolutionContext::from_store(&store, root) .expect("snapshot")