From ea0d0691207223068a713adbe10b56f46faf8a48 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 17:42:08 +0800 Subject: [PATCH 01/10] feat(status): answer pending changes from git's candidates when git sees every change (#1878) --- Cargo.lock | 1 + crates/codegraph-cli/src/main.rs | 6 + crates/codegraph-extract/src/engine.rs | 283 +++++-- crates/codegraph-extract/src/ext_config.rs | 8 + .../tests/scan_membership.rs | 167 +++++ crates/codegraph-watch/Cargo.toml | 5 + crates/codegraph-watch/src/git_pending.rs | 580 +++++++++++++++ crates/codegraph-watch/src/lib.rs | 9 +- crates/codegraph-watch/src/migrate.rs | 10 + crates/codegraph-watch/src/sync.rs | 62 ++ .../tests/git_pending_fast_path.rs | 695 ++++++++++++++++++ docs/cli.md | 21 + docs/mcp.md | 2 + 13 files changed, 1778 insertions(+), 71 deletions(-) create mode 100644 crates/codegraph-extract/tests/scan_membership.rs create mode 100644 crates/codegraph-watch/src/git_pending.rs create mode 100644 crates/codegraph-watch/tests/git_pending_fast_path.rs diff --git a/Cargo.lock b/Cargo.lock index d4a03f1f..804996b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -651,6 +651,7 @@ dependencies = [ "codegraph-extract", "codegraph-resolve", "codegraph-store", + "codegraph-watch", "notify", "rusqlite", "serde", diff --git a/crates/codegraph-cli/src/main.rs b/crates/codegraph-cli/src/main.rs index e355e9d3..b0985f38 100644 --- a/crates/codegraph-cli/src/main.rs +++ b/crates/codegraph-cli/src/main.rs @@ -5272,6 +5272,9 @@ fn index_project_inner( eprintln!("Scanning files…"); } let scan_started = std::time::Instant::now(); + // Capture git before the scan, so the pending-status record this build + // writes is race-safe (#1878). + let git_capture = codegraph_watch::GitIndexCapture::begin(project, &options); let scan = codegraph_extract::engine::scan_project_with_stats(project, &options)?; let files_skipped_unsupported = scan .unsupported_by_extension @@ -5674,6 +5677,9 @@ fn index_project_inner( ); store.set_project_metadata("indexed_with_version", VERSION)?; codegraph_watch::record_followed_links(&store, &followed_links)?; + if let Some(capture) = git_capture { + capture.record(&store, project, &options, !followed_links.is_empty())?; + } let after = store.counts()?; // Explicit fallible finalization: pragma restore -> checkpoint + compaction -> // extraction stamp -> stamp checkpoint -> close the final connection -> diff --git a/crates/codegraph-extract/src/engine.rs b/crates/codegraph-extract/src/engine.rs index ae0316c1..f7d3885d 100644 --- a/crates/codegraph-extract/src/engine.rs +++ b/crates/codegraph-extract/src/engine.rs @@ -492,6 +492,96 @@ pub enum LinkKind { Dir, } +/// How the project scan treats one root-relative path. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Membership { + /// [`scan_project`] yields the file, or descends into the directory. + Admitted, + /// The scan never yields it — including a path that does not exist. + Rejected, + /// The path runs through a symlink, which only a full scan can resolve, + /// or a component could not be inspected. + Unknown, +} + +/// Whether [`scan_project`] would yield `relative` as a source file or, with +/// `dir`, descend into it as a directory, judged component by component by the +/// same per-entry rules the walk applies. +pub fn scan_membership( + root: &Path, + options: &ExtractOptions, + relative: &str, + dir: bool, +) -> Membership { + let components = relative + .split('/') + .filter(|component| !component.is_empty()) + .collect::>(); + if components.is_empty() || components.iter().any(|c| *c == "." || *c == "..") { + return Membership::Rejected; + } + let ignored_dirs = options + .ignore_dirs + .iter() + .map(String::as_str) + .collect::>(); + let gitignore = RootGitignore::load(root); + let pattern_sets: Vec<&[String]> = vec![&options.ignore_paths, &options.exclude]; + let include = IncludeSet::new(&options.include, &options.exclude); + let reserved_roots = codegraph_core::IndexPaths::reserved_index_roots( + root, + std::env::var("CODEGRAPH_DIR").ok().as_deref(), + ); + let rules = ScanRules { + root, + ignored_dirs: &ignored_dirs, + reserved_roots: &reserved_roots, + pattern_sets: &pattern_sets, + gitignore: &gitignore, + include: &include, + overrides: &options.extensions, + }; + let mut parent = root.to_path_buf(); + let mut above = GitignoreAncestry::default(); + for (index, name) in components.iter().enumerate() { + let path = parent.join(name); + let partial = components[..=index].join("/"); + let metadata = match fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Membership::Rejected; + } + Err(_) => return Membership::Unknown, + }; + if metadata.file_type().is_symlink() { + return Membership::Unknown; + } + let is_dir = metadata.is_dir(); + if rules.prunes_by_name(&parent, &path, name, &partial, || is_dir) { + return Membership::Rejected; + } + let last = index + 1 == components.len(); + if last && !dir { + if !metadata.is_file() { + return Membership::Rejected; + } + return match rules.decide(&partial, false, above) { + EntryDecision::File { extractable: true } => Membership::Admitted, + _ => Membership::Rejected, + }; + } + if !is_dir { + return Membership::Rejected; + } + match rules.decide(&partial, true, above) { + EntryDecision::Descend(next) => above = next, + _ => return Membership::Rejected, + } + parent = path; + } + Membership::Admitted +} + pub fn scan_project(root: &Path, options: &ExtractOptions) -> Result> { Ok(scan_project_with_stats(root, options)?.files) } @@ -518,13 +608,15 @@ pub fn scan_project_with_stats(root: &Path, options: &ExtractOptions) -> Result< // No symlink may lead into the project's `.git` or a reserved index root. let blocked = std::iter::once(root.join(".git")).chain(reserved_roots.iter().cloned()); let mut walk = ScanWalk { - root, - ignored_dirs: &ignored_dirs, - reserved_roots: &reserved_roots, - pattern_sets: &pattern_sets, - gitignore: &gitignore, - include: &include, - overrides: &options.extensions, + rules: ScanRules { + root, + ignored_dirs: &ignored_dirs, + reserved_roots: &reserved_roots, + pattern_sets: &pattern_sets, + gitignore: &gitignore, + include: &include, + overrides: &options.extensions, + }, links: LinkWalk::new(root, blocked), files: Vec::new(), unsupported_by_extension: BTreeMap::new(), @@ -551,8 +643,9 @@ pub fn scan_project_with_stats(root: &Path, options: &ExtractOptions) -> Result< Ok(walk.finish()) } -/// One scan's fixed inputs and accumulated output. -struct ScanWalk<'a> { +/// The scan's per-entry rules, shared by the walk and [`scan_membership`] so +/// that single-path membership cannot drift from what the walk yields. +struct ScanRules<'a> { root: &'a Path, ignored_dirs: &'a HashSet<&'a str>, reserved_roots: &'a std::collections::BTreeSet, @@ -560,6 +653,78 @@ struct ScanWalk<'a> { gitignore: &'a RootGitignore, include: &'a IncludeSet<'a>, overrides: &'a ExtensionOverrides, +} + +/// The scan's verdict on one entry whose kind is known. +enum EntryDecision { + Skip, + /// Descend into the directory with this ancestry. + Descend(GitignoreAncestry), + /// A file the ignore model admits; it is indexed iff `extractable`, and + /// otherwise counted as an unsupported extension. + File { + extractable: bool, + }, +} + +impl ScanRules<'_> { + /// Whether the entry `name` at `path` inside `dir` is pruned before its + /// kind is read: `.git` directly under the root, an exact reserved index + /// root at any depth, or an `ignore_dirs` name — except a `build` that is a + /// JVM package segment under a conventional source root (#1642). + fn prunes_by_name( + &self, + dir: &Path, + path: &Path, + name: &str, + relative: &str, + resolves_to_dir: impl FnOnce() -> bool, + ) -> bool { + // Skip `.git` (a direct child of the scan root) and any directory whose + // FULL path is a resolved reserved index root — matched at any depth, so + // a nested configured root like `/cache/index` is pruned exactly, + // while a same-basename user directory elsewhere is not. + let is_reserved_root_here = + (dir == self.root && name == ".git") || self.reserved_roots.contains(path); + // `build` is also a legal JVM package segment: keep it under a + // conventional source root while still pruning build output (#1642). + let jvm_package = name == "build" + && codegraph_core::config::is_jvm_source_build_dir(relative) + && resolves_to_dir(); + is_reserved_root_here || (self.ignored_dirs.contains(name) && !jvm_package) + } + + fn decide(&self, relative: &str, is_dir: bool, above: GitignoreAncestry) -> EntryDecision { + let own = self.gitignore.matched(relative, is_dir); + let ignored = is_path_ignored(relative, self.pattern_sets, above.decide(own)); + if is_dir { + // A model-ignored dir is normally pruned before descent, so a FILE + // include under a gitignored ancestor would never be reached. + // Descend anyway when this dir is an ancestor of (or matches) an + // include pattern; files inside are still pruned unless force-included. + if ignored && !self.include.wants_descend(relative) { + EntryDecision::Skip + } else { + EntryDecision::Descend(above.child(own)) + } + } else if !ignored || self.include.forces(relative) { + // Post-model include decision: a model-ignored file is force-included + // iff it matches `include` and is NOT overridden by an explicit + // `exclude` (checked inside `IncludeSet::forces`). Built-in dir skips + // are already handled structurally above, so include can never + // resurface node_modules/dist/.git/etc. + EntryDecision::File { + extractable: is_extractable_source_path(relative, self.overrides), + } + } else { + EntryDecision::Skip + } + } +} + +/// One scan's fixed inputs and accumulated output. +struct ScanWalk<'a> { + rules: ScanRules<'a>, links: LinkWalk, files: Vec, unsupported_by_extension: BTreeMap, @@ -586,20 +751,10 @@ impl ScanWalk<'_> { let path = entry.path(); let file_name = entry.file_name(); let name = file_name.to_string_lossy(); - // Skip `.git` (a direct child of the scan root) and any directory whose - // FULL path is a resolved reserved index root — matched at any depth, so - // a nested configured root like `/cache/index` is pruned exactly, - // while a same-basename user directory elsewhere is not. - let is_reserved_root_here = - (dir == self.root && name == ".git") || self.reserved_roots.contains(&path); - let relative = normalize_path(path.strip_prefix(self.root).unwrap_or(&path)); - // `build` is also a legal JVM package segment: keep it under a - // conventional source root while still pruning build output (#1642). - let jvm_package = name == "build" - && codegraph_core::config::is_jvm_source_build_dir(&relative) - && resolves_to_dir(&entry, &path); - if is_reserved_root_here || (self.ignored_dirs.contains(name.as_ref()) && !jvm_package) - { + let relative = normalize_path(path.strip_prefix(self.rules.root).unwrap_or(&path)); + if self.rules.prunes_by_name(dir, &path, &name, &relative, || { + resolves_to_dir(&entry, &path) + }) { continue; } let file_type = entry.file_type()?; @@ -619,54 +774,44 @@ impl ScanWalk<'_> { } else { continue; }; - let own = self.gitignore.matched(&relative, is_dir); - let ignored = is_path_ignored(&relative, self.pattern_sets, above.decide(own)); - if is_dir { - // A model-ignored dir is normally pruned before descent, so a FILE - // include under a gitignored ancestor would never be reached. - // Descend anyway when this dir is an ancestor of (or matches) an - // include pattern; files inside are still pruned unless force-included. - if ignored && !self.include.wants_descend(&relative) { - continue; - } - if is_link { - self.links.queue(hops + 1, relative, path, above.child(own)); - continue; - } - let canonical = canonical_dir.join(&file_name); - if !self.links.enter(canonical.clone(), &relative) { - continue; - } - if hops > 0 { - self.linked_dirs.push(relative); - } - self.scan_dir(&path, &canonical, above.child(own), hops)?; - } else if !ignored || self.include.forces(&relative) { - // Post-model include decision: a model-ignored file is force-included - // iff it matches `include` and is NOT overridden by an explicit - // `exclude` (checked inside `IncludeSet::forces`). Built-in dir skips - // are already handled structurally above, so include can never - // resurface node_modules/dist/.git/etc. - let target = if is_link { - match self.links.file_target(&path) { - Some(target) => Some(target), - None => continue, + match self.rules.decide(&relative, is_dir, above) { + EntryDecision::Skip => {} + EntryDecision::Descend(next) => { + if is_link { + self.links.queue(hops + 1, relative, path, next); + continue; + } + let canonical = canonical_dir.join(&file_name); + if !self.links.enter(canonical.clone(), &relative) { + continue; } - } else { - None - }; - if is_extractable_source_path(&relative, self.overrides) { - if let Some(target) = target { - self.followed.push(FollowedLink { - relative: relative.clone(), - canonical: target.clone(), - kind: LinkKind::File, - }); - self.file_links.push((relative.clone(), target)); + if hops > 0 { + self.linked_dirs.push(relative); + } + self.scan_dir(&path, &canonical, next, hops)?; + } + EntryDecision::File { extractable } => { + let target = if is_link { + match self.links.file_target(&path) { + Some(target) => Some(target), + None => continue, + } + } else { + None + }; + if extractable { + if let Some(target) = target { + self.followed.push(FollowedLink { + relative: relative.clone(), + canonical: target.clone(), + kind: LinkKind::File, + }); + self.file_links.push((relative.clone(), target)); + } + self.files.push(relative); + } else if let Some(extension) = unsupported_extension(&relative) { + *self.unsupported_by_extension.entry(extension).or_default() += 1; } - self.files.push(relative); - } else if let Some(extension) = unsupported_extension(&relative) { - *self.unsupported_by_extension.entry(extension).or_default() += 1; } } } diff --git a/crates/codegraph-extract/src/ext_config.rs b/crates/codegraph-extract/src/ext_config.rs index 2bf6132c..8a58fb63 100644 --- a/crates/codegraph-extract/src/ext_config.rs +++ b/crates/codegraph-extract/src/ext_config.rs @@ -44,6 +44,14 @@ impl ExtensionOverrides { Arc::new(Self::default()) } + /// Every override, in extension order: a stable identity for the scope a + /// scan ran under. + pub fn entries(&self) -> impl Iterator + '_ { + self.map + .iter() + .map(|(extension, language)| (extension.as_str(), *language)) + } + /// Load the overrides declared by ONE project's current index root. /// /// Reads exactly `paths.extension_config()`. A missing file yields empty diff --git a/crates/codegraph-extract/tests/scan_membership.rs b/crates/codegraph-extract/tests/scan_membership.rs new file mode 100644 index 00000000..270d97b8 --- /dev/null +++ b/crates/codegraph-extract/tests/scan_membership.rs @@ -0,0 +1,167 @@ +//! `scan_membership` judges one path exactly as the scan walk does: for every +//! file on disk, it is `Admitted` iff `scan_project` yields that file. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::SystemTime; + +use codegraph_extract::ExtractOptions; +use codegraph_extract::engine::{Membership, scan_membership, scan_project}; + +fn sandbox(tag: &str) -> PathBuf { + static COUNTER: AtomicU32 = AtomicU32::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + let nanos = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let dir = std::env::temp_dir().join(format!( + "cg_scan_membership_{tag}_{}_{nanos}_{n}", + std::process::id() + )); + fs::create_dir_all(&dir).expect("create sandbox"); + dir +} + +fn touch(root: &Path, relative: &str) { + let path = root.join(relative); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, "export const value = 1;\n").unwrap(); +} + +/// Every regular file under `root`, root-relative. +fn files_on_disk(root: &Path) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in fs::read_dir(&dir).unwrap() { + let entry = entry.unwrap(); + let kind = entry.file_type().unwrap(); + if kind.is_dir() { + stack.push(entry.path()); + } else if kind.is_file() { + let relative = entry.path().strip_prefix(root).unwrap().to_path_buf(); + out.push(relative.to_string_lossy().replace('\\', "/")); + } + } + } + out.sort(); + out +} + +#[test] +fn membership_matches_the_walk_for_every_file_on_disk() { + let root = sandbox("parity"); + fs::write(root.join(".gitignore"), "gen/\n!gen/keep.ts\nlogs/*.ts\n").unwrap(); + for file in [ + "src/app.ts", + "src/util.ts", + "gen/out.ts", + "gen/keep.ts", + "logs/run.ts", + "node_modules/pkg/index.ts", + "build/out.ts", + "src/main/java/com/build/Builder.java", + ".cache/cached.ts", + ".codegraph/stray.ts", + ".codegraph-sources/kept.ts", + "src/.codegraph/nested.ts", + ".git/hooks/hook.ts", + "docs/notes.txt", + "assets/blob.bin", + ] { + touch(&root, file); + } + + for options in [ + ExtractOptions::default(), + ExtractOptions { + include: vec!["gen/keep.ts".to_string()], + ..ExtractOptions::default() + }, + ExtractOptions { + exclude: vec!["src/util.ts".to_string(), "logs/".to_string()], + ..ExtractOptions::default() + }, + ] { + let scanned = scan_project(&root, &options).expect("scan"); + for file in files_on_disk(&root) { + let membership = scan_membership(&root, &options, &file, false); + assert_ne!(membership, Membership::Unknown, "{file}: no symlinks here"); + assert_eq!( + membership == Membership::Admitted, + scanned.contains(&file), + "{file}: membership {membership:?} vs scan {scanned:?} (include {:?}, exclude {:?})", + options.include, + options.exclude + ); + } + } + fs::remove_dir_all(&root).ok(); +} + +#[test] +fn directories_missing_paths_and_symlinks() { + let root = sandbox("dirs"); + fs::write(root.join(".gitignore"), "gen/\n").unwrap(); + touch(&root, "src/app.ts"); + touch(&root, "gen/keep.ts"); + touch(&root, "node_modules/pkg/index.ts"); + let default = ExtractOptions::default(); + assert_eq!( + scan_membership(&root, &default, "src", true), + Membership::Admitted + ); + assert_eq!( + scan_membership(&root, &default, "node_modules", true), + Membership::Rejected + ); + assert_eq!( + scan_membership(&root, &default, "gen", true), + Membership::Rejected + ); + let include = ExtractOptions { + include: vec!["gen/keep.ts".to_string()], + ..ExtractOptions::default() + }; + assert_eq!( + scan_membership(&root, &include, "gen", true), + Membership::Admitted, + "an include below an ignored directory makes the walk descend" + ); + assert_eq!( + scan_membership(&root, &default, "src/missing.ts", false), + Membership::Rejected + ); + assert_eq!( + scan_membership(&root, &default, "../escape.ts", false), + Membership::Rejected + ); + + let outside = sandbox("dirs_outside"); + touch(&outside, "o.ts"); + #[cfg(unix)] + let made = std::os::unix::fs::symlink(&outside, root.join("linked")); + #[cfg(windows)] + let made = std::os::windows::fs::symlink_dir(&outside, root.join("linked")); + match made { + Ok(()) => { + assert_eq!( + scan_membership(&root, &default, "linked/o.ts", false), + Membership::Unknown, + "a path through a symlink is the full scan's to resolve" + ); + assert_eq!( + scan_membership(&root, &default, "linked", true), + Membership::Unknown + ); + } + Err(error) if std::env::var_os("CI").is_some() => { + panic!("CI must be able to create symlinks: {error}") + } + Err(_) => {} + } + fs::remove_dir_all(&root).ok(); + fs::remove_dir_all(&outside).ok(); +} diff --git a/crates/codegraph-watch/Cargo.toml b/crates/codegraph-watch/Cargo.toml index 8d88a344..02351cc9 100644 --- a/crates/codegraph-watch/Cargo.toml +++ b/crates/codegraph-watch/Cargo.toml @@ -29,6 +29,11 @@ codegraph-extract = { path = "../codegraph-extract" } codegraph-resolve = { path = "../codegraph-resolve" } codegraph-store = { path = "../codegraph-store" } +[dev-dependencies] +# Self-reference with `test-hooks`, so integration tests compile the writer +# seams in (the pattern `codegraph-mcp` uses); the shipped binary never does. +codegraph-watch = { path = ".", features = ["test-hooks"] } + [target.'cfg(windows)'.dev-dependencies] windows-sys = { version = "0.52", features = [ "Win32_Foundation", diff --git a/crates/codegraph-watch/src/git_pending.rs b/crates/codegraph-watch/src/git_pending.rs new file mode 100644 index 00000000..afaa1de2 --- /dev/null +++ b/crates/codegraph-watch/src/git_pending.rs @@ -0,0 +1,580 @@ +//! Git-stamped fast path for pending status (upstream #1829 / #1878). +//! +//! `status` reports the work the next whole-tree sync would do. The reference +//! answer is the full inventory: scan, stat every file, hash on a stat +//! mismatch. That costs a walk of the whole tree, so a full build or full sync +//! also records, under [`GIT_PENDING_KEY`], the commit it started at and every +//! path that may differ from that commit: the paths git reported dirty around +//! the build, rows whose stat moved while it ran, and paths a fresh scan and +//! the database disagree on. Incremental syncs add the paths they are handed. +//! +//! The fast path then asks git for the candidates — what changed between the +//! recorded commit and HEAD, plus what `git status` reports now — and +//! classifies only those, plus the recorded paths, exactly as the full +//! inventory would. Git supplies candidates, never the verdict. It declines, +//! leaving the full inventory to answer, whenever git might not see a change: +//! +//! - there is no record, or its scope fingerprint differs from now; +//! - the index was built through symlinks, or the repository has submodules; +//! - git is missing, slow (10 s), loud (64 MiB), or has no HEAD; +//! - the recorded commit is no longer in the repository; +//! - an untracked nested repository, a directory or symlink candidate, an +//! `assume-unchanged` or `skip-worktree` entry, or an ignored path the scan +//! would keep. + +use std::collections::BTreeSet; +use std::fs; +use std::io::Read as _; +use std::path::Path; +use std::process::Stdio; +use std::time::{Duration, Instant}; + +use anyhow::Result; +use codegraph_core::node_id::hash_content; +use codegraph_core::{IndexPaths, source_file::is_source_file, source_file::read_source_file}; +use codegraph_extract::ExtractOptions; +use codegraph_extract::engine::{Membership, scan_membership, scan_project}; +use codegraph_store::{CURRENT_EXTRACTION_VERSION, Store}; +use serde::{Deserialize, Serialize}; + +use crate::sync::{PendingChanges, modified_millis}; + +/// `project_metadata` key holding the git record (one JSON value, so it can +/// never be read half-written). +pub const GIT_PENDING_KEY: &str = "git_pending_state"; + +const RECORD_VERSION: u32 = 1; +const GIT_TIMEOUT: Duration = Duration::from_secs(10); +const GIT_MAX_STDOUT: u64 = 64 * 1024 * 1024; + +/// Which computation answered a pending-changes query. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PendingSource { + GitFastPath, + FullInventory, +} + +#[derive(Debug, Serialize, Deserialize)] +struct Record { + v: u32, + commit: String, + /// Project-relative paths that are always re-classified. + dirty: Vec, + scope: String, + links: bool, + submodules: bool, +} + +/// A stable identity of everything that decides scan membership. A record +/// made under another scope (config, extension overrides, root `.gitignore`, +/// index root, extraction rules, binary) proves nothing about this one. +pub fn scope_fingerprint(root: &Path, options: &ExtractOptions) -> String { + let gitignore = fs::read(root.join(".gitignore")).unwrap_or_default(); + let reserved = + IndexPaths::reserved_index_roots(root, std::env::var("CODEGRAPH_DIR").ok().as_deref()); + let identity = serde_json::json!({ + "ignore_dirs": options.ignore_dirs, + "ignore_paths": options.ignore_paths, + "include": options.include, + "exclude": options.exclude, + "max_file_size": options.max_file_size, + "extensions": options + .extensions + .entries() + .map(|(extension, language)| (extension.to_string(), format!("{language:?}"))) + .collect::>(), + "gitignore": hash_content(&String::from_utf8_lossy(&gitignore)), + "reserved": reserved + .iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect::>(), + "extraction_version": CURRENT_EXTRACTION_VERSION, + "version": env!("CARGO_PKG_VERSION"), + }); + hash_content(&identity.to_string()) +} + +/// Run `git --no-optional-locks ` in `root`, with the ambient +/// repository-selection environment stripped. `None` on any failure, a +/// timeout, or output over the cap. +fn git(root: &Path, args: &[&str]) -> Option> { + let mut command = std::process::Command::new(test_hooks::git_program()); + command.current_dir(root); + for var in crate::git::GIT_REPO_SELECTION_ENV { + command.env_remove(var); + } + command + .arg("--no-optional-locks") + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()); + #[cfg(windows)] + { + use std::os::windows::process::CommandExt as _; + const CREATE_NO_WINDOW: u32 = 0x0800_0000; + command.creation_flags(CREATE_NO_WINDOW); + } + let mut child = command.spawn().ok()?; + let stdout = child.stdout.take()?; + let reader = std::thread::spawn(move || { + let mut out = Vec::new(); + stdout + .take(GIT_MAX_STDOUT + 1) + .read_to_end(&mut out) + .ok() + .map(|_| out) + }); + let deadline = Instant::now() + test_hooks::git_timeout(GIT_TIMEOUT); + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break status, + Ok(None) if Instant::now() < deadline => std::thread::sleep(Duration::from_millis(5)), + _ => { + let _ = child.kill(); + let _ = child.wait(); + return None; + } + } + }; + let out = reader.join().ok()??; + (status.success() && out.len() as u64 <= GIT_MAX_STDOUT).then_some(out) +} + +/// HEAD and the project's prefix inside its work tree. +struct Repo { + head: String, + /// The project root relative to the work tree, with a trailing `/`, or + /// empty at the top level. + prefix: String, + toplevel: String, +} + +impl Repo { + fn read(root: &Path) -> Option { + let out = git( + root, + &["rev-parse", "HEAD", "--show-prefix", "--show-toplevel"], + )?; + let text = String::from_utf8(out).ok()?; + let mut lines = text.lines(); + let head = lines.next()?.trim().to_string(); + let prefix = lines.next()?.to_string(); + let toplevel = lines.next()?.to_string(); + (!head.is_empty()).then_some(Self { + head, + prefix, + toplevel, + }) + } + + /// A repo-root-relative git path as a project-relative one; `None` when it + /// lies outside the project. + fn rebase(&self, path: &str) -> Option { + path.strip_prefix(self.prefix.as_str()) + .filter(|rest| !rest.is_empty()) + .map(str::to_string) + } +} + +fn nul_fields(out: &[u8]) -> impl Iterator + '_ { + out.split(|byte| *byte == 0) + .filter(|field| !field.is_empty()) + .map(|field| String::from_utf8_lossy(field).into_owned()) +} + +/// Project-relative paths `git status` reports. An untracked directory entry +/// (an embedded repository `-uall` cannot expand) keeps its trailing `/`. +fn status_paths(root: &Path, repo: &Repo) -> Option> { + let out = git( + root, + &[ + "status", + "--porcelain=v1", + "-z", + "--no-renames", + "-uall", + "--ignore-submodules=none", + "--", + ".", + ], + )?; + let mut paths = BTreeSet::new(); + for entry in nul_fields(&out) { + let path = entry.get(3..)?; + if let Some(relative) = repo.rebase(path) { + paths.insert(relative); + } + } + Some(paths) +} + +/// Project-relative paths that differ between `commit` and HEAD. +fn committed_paths(root: &Path, repo: &Repo, commit: &str) -> Option> { + let out = git( + root, + &[ + "diff", + "--name-status", + "--no-renames", + "--no-relative", + "-z", + "--ignore-submodules=none", + commit, + "HEAD", + "--", + ".", + ], + )?; + let fields = nul_fields(&out).collect::>(); + let mut paths = BTreeSet::new(); + for pair in fields.chunks(2) { + let [_, path] = pair else { + return None; + }; + if let Some(relative) = repo.rebase(path) { + paths.insert(relative); + } + } + Some(paths) +} + +/// What the index file says that `git status` cannot: a gitlink (submodule), +/// or an entry marked `assume-unchanged` or `skip-worktree`, whose working-tree +/// changes git hides. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +struct IndexFlags { + submodules: bool, + hidden_changes: bool, +} + +fn index_flags(root: &Path, repo: &Repo) -> Option { + let out = git( + root, + &["ls-files", "-s", "-v", "-z", "--full-name", "--", "."], + )?; + let mut flags = IndexFlags { + submodules: Path::new(&repo.toplevel).join(".gitmodules").exists(), + hidden_changes: false, + }; + for entry in nul_fields(&out) { + // ` \t`. + let mut fields = entry.split(' '); + let tag = fields.next()?; + let mode = fields.next()?; + flags.submodules |= mode == "160000"; + flags.hidden_changes |= tag == "S" || tag.chars().all(|c| c.is_ascii_lowercase()); + } + Some(flags) +} + +/// Whether git ignores a path the scan would keep: a working-tree change +/// there is invisible to `git status`. `None` when git fails. +fn ignores_scanned_path(root: &Path, repo: &Repo, options: &ExtractOptions) -> Option { + let out = git( + root, + &[ + "ls-files", + "--full-name", + "-z", + "--others", + "--ignored", + "--exclude-standard", + "--directory", + "--", + ".", + ], + )?; + for entry in nul_fields(&out) { + let Some(relative) = repo.rebase(&entry) else { + continue; + }; + let membership = match relative.strip_suffix('/') { + Some(dir) => scan_membership(root, options, dir, true), + None => scan_membership(root, options, &relative, false), + }; + if membership != Membership::Rejected { + return Some(true); + } + } + Some(false) +} + +/// The git state a full build or full sync starts from, captured before it +/// scans or reads a file. +pub struct GitIndexCapture { + repo: Repo, + status: BTreeSet, + submodules: bool, + scope: String, +} + +impl GitIndexCapture { + /// `None` outside a git work tree, without a HEAD, or when git fails; + /// such a build then leaves no record and the fast path stays declined. + pub fn begin(root: &Path, options: &ExtractOptions) -> Option { + let scope = scope_fingerprint(root, options); + let repo = Repo::read(root)?; + let status = status_paths(root, &repo)?; + let flags = index_flags(root, &repo)?; + let capture = Self { + repo, + status, + submodules: flags.submodules, + scope, + }; + test_hooks::after_git_begin(root); + Some(capture) + } + + /// Write the record after every row is written. A scope that changed + /// while the build ran, or any git failure, removes the record instead. + pub fn record( + self, + store: &Store, + root: &Path, + options: &ExtractOptions, + links_followed: bool, + ) -> Result<()> { + test_hooks::before_git_record(root); + let after = (scope_fingerprint(root, options) == self.scope) + .then(|| status_paths(root, &self.repo)) + .flatten(); + let Some(after) = after else { + store.delete_project_metadata(GIT_PENDING_KEY)?; + return Ok(()); + }; + let mut dirty = self.status; + dirty.extend(after); + // A row whose file changed after it was read (and possibly back), and + // any path a fresh scan and the database disagree on (a file the first + // scan missed while it was briefly gone or excluded). + let stored = store.all_files()?; + let mut stored_paths = BTreeSet::new(); + for file in stored { + let unchanged = fs::metadata(root.join(&file.path)).is_ok_and(|metadata| { + file.size == metadata.len() as i64 && file.modified_at == modified_millis(&metadata) + }); + if !unchanged { + dirty.insert(file.path.clone()); + } + stored_paths.insert(file.path); + } + let fresh = scan_project(root, options)? + .into_iter() + .collect::>(); + dirty.extend(fresh.symmetric_difference(&stored_paths).cloned()); + let record = Record { + v: RECORD_VERSION, + commit: self.repo.head, + dirty: dirty.into_iter().collect(), + scope: self.scope, + links: links_followed, + submodules: self.submodules, + }; + store.set_project_metadata(GIT_PENDING_KEY, &serde_json::to_string(&record)?)?; + Ok(()) + } +} + +/// Forget the record before a full sync rewrites rows, so an interrupted sync +/// leaves none rather than a stale one. +pub(crate) fn forget(store: &Store) -> Result<()> { + store.delete_project_metadata(GIT_PENDING_KEY)?; + Ok(()) +} + +/// Before an incremental sync writes a row: keep the record only if it was made +/// under the current scope, with every requested path added to `dirty`. +pub(crate) fn extend_for_incremental( + store: &Store, + root: &Path, + options: &ExtractOptions, + requested: &BTreeSet, +) -> Result<()> { + let record = store + .get_project_metadata(GIT_PENDING_KEY)? + .and_then(|value| serde_json::from_str::(&value).ok()); + match record { + Some(mut record) + if record.v == RECORD_VERSION && record.scope == scope_fingerprint(root, options) => + { + let mut dirty = record.dirty.into_iter().collect::>(); + dirty.extend(requested.iter().cloned()); + record.dirty = dirty.into_iter().collect(); + store.set_project_metadata(GIT_PENDING_KEY, &serde_json::to_string(&record)?)?; + } + _ => forget(store)?, + } + Ok(()) +} + +/// The pending changes from git's candidates, or `None` to defer to the full +/// inventory. +pub(crate) fn git_fast_pending( + root: &Path, + store: &Store, + options: &ExtractOptions, +) -> Result> { + let Some(record) = store + .get_project_metadata(GIT_PENDING_KEY)? + .and_then(|value| serde_json::from_str::(&value).ok()) + else { + return Ok(None); + }; + if record.v != RECORD_VERSION + || record.links + || record.submodules + || record.scope != scope_fingerprint(root, options) + { + return Ok(None); + } + let Some(repo) = Repo::read(root) else { + return Ok(None); + }; + let exists = format!("{}^{{commit}}", record.commit); + if git(root, &["cat-file", "-e", &exists]).is_none() { + return Ok(None); + } + let Some(flags) = index_flags(root, &repo) else { + return Ok(None); + }; + if flags.submodules || flags.hidden_changes { + return Ok(None); + } + let (Some(status), Some(committed)) = ( + status_paths(root, &repo), + committed_paths(root, &repo, &record.commit), + ) else { + return Ok(None); + }; + if status.iter().any(|path| path.ends_with('/')) { + return Ok(None); + } + if ignores_scanned_path(root, &repo, options) != Some(false) { + return Ok(None); + } + + let mut candidates = status; + candidates.extend(committed); + candidates.extend(record.dirty); + let mut pending = PendingChanges::default(); + for relative in candidates { + let full = root.join(&relative); + if fs::symlink_metadata(&full) + .is_ok_and(|metadata| metadata.file_type().is_symlink() || metadata.is_dir()) + { + return Ok(None); + } + let membership = scan_membership(root, options, &relative, false); + if membership == Membership::Unknown { + return Ok(None); + } + let stored = store.file_by_path(&relative)?; + match (stored, membership) { + (None, Membership::Admitted) => { + // An untracked video clip named `.ts` is not source sync would add. + if is_source_file(&full, &relative, options.max_file_size)? { + pending.added.push(relative); + } + } + (None, _) => {} + (Some(_), _) if membership != Membership::Admitted => pending.removed.push(relative), + (Some(stored), _) => { + let metadata = fs::metadata(&full)?; + if stored.size == metadata.len() as i64 + && stored.modified_at == modified_millis(&metadata) + { + continue; + } + let (_, source) = read_source_file(&full, &relative, options.max_file_size)?; + match source.hash_input() { + // A tracked file that became a video clip leaves the index. + None => pending.removed.push(relative), + Some(input) if stored.content_hash != hash_content(&input) => { + pending.modified.push(relative); + } + Some(_) => {} + } + } + } + } + pending.added.sort(); + pending.modified.sort(); + pending.removed.sort(); + Ok(Some(pending)) +} + +/// Seams for tests that race the writers: a hook that runs right after +/// [`GitIndexCapture::begin`] captures git (before the build scans), and one +/// right before [`GitIndexCapture::record`] (after every file was read). +pub mod test_hooks { + use std::path::Path; + + #[cfg(feature = "test-hooks")] + type Hook = Box; + + #[cfg(feature = "test-hooks")] + static AFTER_GIT_BEGIN: std::sync::Mutex> = std::sync::Mutex::new(None); + #[cfg(feature = "test-hooks")] + static BEFORE_GIT_RECORD: std::sync::Mutex> = std::sync::Mutex::new(None); + + #[cfg(feature = "test-hooks")] + pub fn set_after_git_begin(hook: Option) { + *AFTER_GIT_BEGIN.lock().unwrap() = hook; + } + + #[cfg(feature = "test-hooks")] + pub fn set_before_git_record(hook: Option) { + *BEFORE_GIT_RECORD.lock().unwrap() = hook; + } + + #[cfg(feature = "test-hooks")] + static GIT_PROGRAM: std::sync::Mutex> = std::sync::Mutex::new(None); + #[cfg(feature = "test-hooks")] + static GIT_TIMEOUT: std::sync::Mutex> = std::sync::Mutex::new(None); + + /// Run this program instead of `git` (a stand-in that hangs, say). + #[cfg(feature = "test-hooks")] + pub fn set_git_program(program: Option) { + *GIT_PROGRAM.lock().unwrap() = program; + } + + /// Give up on git after this long instead of the default. + #[cfg(feature = "test-hooks")] + pub fn set_git_timeout(timeout: Option) { + *GIT_TIMEOUT.lock().unwrap() = timeout; + } + + pub(super) fn git_program() -> std::ffi::OsString { + #[cfg(feature = "test-hooks")] + if let Some(program) = GIT_PROGRAM.lock().unwrap().as_ref() { + return program.clone().into_os_string(); + } + "git".into() + } + + pub(super) fn git_timeout(default: std::time::Duration) -> std::time::Duration { + #[cfg(feature = "test-hooks")] + if let Some(timeout) = *GIT_TIMEOUT.lock().unwrap() { + return timeout; + } + default + } + + pub(super) fn after_git_begin(root: &Path) { + #[cfg(feature = "test-hooks")] + if let Some(hook) = AFTER_GIT_BEGIN.lock().unwrap().as_ref() { + hook(root); + } + let _ = root; + } + + pub(super) fn before_git_record(root: &Path) { + #[cfg(feature = "test-hooks")] + if let Some(hook) = BEFORE_GIT_RECORD.lock().unwrap().as_ref() { + hook(root); + } + let _ = root; + } +} diff --git a/crates/codegraph-watch/src/lib.rs b/crates/codegraph-watch/src/lib.rs index 0fe2526d..d2c17d73 100644 --- a/crates/codegraph-watch/src/lib.rs +++ b/crates/codegraph-watch/src/lib.rs @@ -1,4 +1,5 @@ mod git; +mod git_pending; mod link_state; mod migrate; mod policy; @@ -10,13 +11,17 @@ pub use git::{ DEFAULT_SYNC_HOOKS, GitHookName, GitHookResult, install_git_sync_hooks, is_git_repo, is_sync_hook_installed, remove_git_sync_hooks, }; +#[cfg(feature = "test-hooks")] +pub use git_pending::test_hooks as git_pending_hooks; +pub use git_pending::{GIT_PENDING_KEY, GitIndexCapture, PendingSource, scope_fingerprint}; pub use link_state::{FOLLOWED_LINKS_KEY, record_followed_links}; pub use policy::{ CODEGRAPH_NO_WATCH, TooBroadRoot, WatchPolicy, too_broad_root_reason, watch_disabled_reason, }; pub use sync::{ - PendingChanges, SyncCancellation, SyncOutcome, pending_project_changes, sync_changed_paths, - sync_project_once, sync_project_once_cancellable, sync_project_once_with_progress, + PendingChanges, SyncCancellation, SyncOutcome, pending_full_inventory, pending_project_changes, + pending_project_changes_detailed, sync_changed_paths, sync_project_once, + sync_project_once_cancellable, sync_project_once_with_progress, }; pub use watcher::{ LOCK_RECOVERY_INTERVAL, PendingFile, ProjectWatcher, WatchHealth, WatchOptions, diff --git a/crates/codegraph-watch/src/migrate.rs b/crates/codegraph-watch/src/migrate.rs index 5225555a..20b895d7 100644 --- a/crates/codegraph-watch/src/migrate.rs +++ b/crates/codegraph-watch/src/migrate.rs @@ -73,6 +73,8 @@ pub(crate) fn migrate_project( let options = &scope.options; // `scan_project` returns a SORTED list, and every downstream pass keeps that // order, so no HashSet iteration order can reach the database or the outcome. + // Capture git before the scan, so the record this build writes is race-safe. + let git_capture = crate::git_pending::GitIndexCapture::begin(project_root, options); let scan = codegraph_extract::engine::scan_project_with_stats(project_root, options)?; let candidates = scan.files; let total = candidates.len(); @@ -182,6 +184,14 @@ pub(crate) fn migrate_project( .store() .set_project_metadata(INDEXED_WITH_VERSION_KEY, env!("CARGO_PKG_VERSION"))?; crate::link_state::record_followed_links(rebuild.store(), &scan.links)?; + if let Some(capture) = git_capture { + capture.record( + rebuild.store(), + project_root, + options, + !scan.links.is_empty(), + )?; + } // Explicit fallible finalization: pragma restore, checkpoint + compaction, // extraction stamp, stamp checkpoint, connection close, and only then the diff --git a/crates/codegraph-watch/src/sync.rs b/crates/codegraph-watch/src/sync.rs index 75d1ba27..0e951f4b 100644 --- a/crates/codegraph-watch/src/sync.rs +++ b/crates/codegraph-watch/src/sync.rs @@ -22,6 +22,7 @@ use codegraph_resolve::frameworks::godot_dsl_config::GodotDslConfig; use codegraph_store::queries::{FileReferenceSite, ReferenceSite}; use codegraph_store::{IndexLease, Store, StoreWriteOpen, StoreWritePurpose}; +use crate::git_pending::{GitIndexCapture, PendingSource}; use crate::link_state::{RehashUnder, record_followed_links}; use crate::policy::WatchPolicy; @@ -145,9 +146,45 @@ pub fn pending_project_changes( project_root: impl AsRef, store: &Store, ) -> Result { + Ok(pending_project_changes_detailed(project_root, store)?.0) +} + +/// [`pending_project_changes`], and which computation answered: the git fast +/// path when git can see every change since the index was built (#1878), +/// otherwise the full inventory. +pub fn pending_project_changes_detailed( + project_root: impl AsRef, + store: &Store, +) -> Result<(PendingChanges, PendingSource)> { let project_root = project_root.as_ref(); let paths = index_paths(project_root)?; let scope = ProjectScope::load(project_root, &paths)?; + if let Some(fast) = crate::git_pending::git_fast_pending(project_root, store, &scope.options)? { + return Ok((fast, PendingSource::GitFastPath)); + } + Ok(( + full_inventory(project_root, store, &scope)?, + PendingSource::FullInventory, + )) +} + +/// The reference answer: a full scan against the persisted inventory, never +/// consulting git. +pub fn pending_full_inventory( + project_root: impl AsRef, + store: &Store, +) -> Result { + let project_root = project_root.as_ref(); + let paths = index_paths(project_root)?; + let scope = ProjectScope::load(project_root, &paths)?; + full_inventory(project_root, store, &scope) +} + +fn full_inventory( + project_root: &Path, + store: &Store, + scope: &ProjectScope, +) -> Result { let scan = codegraph_extract::engine::scan_project_with_stats(project_root, &scope.options)?; // A path reached through a new, retargeted or unrecorded link is re-read: // its stored stat may describe a different file (#935). @@ -254,6 +291,10 @@ fn sync_project_once_with_scope( let _active = cancel.map(SyncCancellation::enter); match open_sync_writer(paths, cancel)? { SyncWriter::Incremental(mut store) => { + // Forget the git record before any row moves, and capture git + // before the scan, so the record this sync writes is race-safe. + crate::git_pending::forget(&store)?; + let git_capture = GitIndexCapture::begin(project_root, options); let scan = codegraph_extract::engine::scan_project_with_stats(project_root, options)?; // Re-read whatever a new, retargeted or unrecorded link reaches, so a // same-stat file behind a moved link cannot keep its old graph (#935). @@ -289,6 +330,9 @@ fn sync_project_once_with_scope( on_progress, )?; record_followed_links(&store, &scan.links)?; + if let Some(capture) = git_capture { + capture.record(&store, project_root, options, !scan.links.is_empty())?; + } store.finish_current_mutation()?; Ok(outcome) } @@ -433,6 +477,24 @@ fn sync_changed_paths_current_scope( let _active = cancel.map(SyncCancellation::enter); match open_sync_writer(&paths, cancel)? { SyncWriter::Incremental(mut store) => { + // Rows change only for these paths, so the git record stays valid + // once they are in its always-reclassified set (#1878). That is + // written before the first row moves. + let changed = changed + .into_iter() + .map(|path| path.as_ref().to_path_buf()) + .collect::>(); + let normalizer = WatchPolicy::new(project_root); + let requested = changed + .iter() + .filter_map(|path| normalizer.normalize_relative(path)) + .collect::>(); + crate::git_pending::extend_for_incremental( + &store, + project_root, + &scope.options, + &requested, + )?; let outcome = sync_paths_with_store( &mut store, project_root, diff --git a/crates/codegraph-watch/tests/git_pending_fast_path.rs b/crates/codegraph-watch/tests/git_pending_fast_path.rs new file mode 100644 index 00000000..f75924d7 --- /dev/null +++ b/crates/codegraph-watch/tests/git_pending_fast_path.rs @@ -0,0 +1,695 @@ +//! The git fast path for pending status (upstream #1829 / #1878). +//! +//! Every case compares the reported changes with the full inventory, the +//! reference answer. Cases git can see must take the fast path; cases it might +//! not see must decline to the full inventory. + +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::time::{Duration, Instant, SystemTime}; + +use codegraph_core::IndexPaths; +use codegraph_store::Store; +use codegraph_watch::git_pending_hooks; +use codegraph_watch::{ + GIT_PENDING_KEY, PendingChanges, PendingSource, pending_full_inventory, + pending_project_changes_detailed, sync_changed_paths, sync_project_once, +}; + +/// Repository-selection variables a git hook exports; a test repo must ignore +/// the ambient one (the pre-push hook runs this suite). +const GIT_ENV: &[&str] = &[ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_COMMON_DIR", + "GIT_INDEX_FILE", + "GIT_OBJECT_DIRECTORY", + "GIT_NAMESPACE", + "GIT_PREFIX", +]; + +/// The writer hooks are process-global: tests that set them, or that depend on +/// them being unset, run one at a time. +static HOOKS: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +fn hooks_guard() -> std::sync::MutexGuard<'static, ()> { + HOOKS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn git_available() -> bool { + Command::new("git") + .arg("--version") + .output() + .is_ok_and(|out| out.status.success()) +} + +struct Repo { + /// The project root (a subdirectory of the work tree for `in_subdir`). + root: PathBuf, + top: PathBuf, +} + +impl Repo { + fn new(tag: &str) -> Self { + Self::with_project(tag, None) + } + + fn in_subdir(tag: &str) -> Self { + Self::with_project(tag, Some("packages/app")) + } + + fn with_project(tag: &str, project: Option<&str>) -> Self { + static NEXT: AtomicU32 = AtomicU32::new(0); + let nanos = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let top = std::env::temp_dir().join(format!( + "cg_git_pending_{tag}_{}_{nanos}_{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&top).unwrap(); + let top = top.canonicalize().unwrap(); + let root = match project { + Some(sub) => top.join(sub), + None => top.clone(), + }; + fs::create_dir_all(&root).unwrap(); + let repo = Self { root, top }; + repo.git_at(&repo.top, &["init", "-q", "."]); + // Keep the index itself out of every commit, as a project does. + fs::write(repo.top.join(".git/info/exclude"), ".codegraph/\n").unwrap(); + repo + } + + fn git_at(&self, dir: &Path, args: &[&str]) -> String { + let mut command = Command::new("git"); + command.current_dir(dir); + for var in GIT_ENV { + command.env_remove(var); + } + let output = command + .args([ + "-c", + "user.name=cg", + "-c", + "user.email=cg@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "protocol.file.allow=always", + ]) + .args(args) + .output() + .expect("run git"); + assert!( + output.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).into_owned() + } + + fn git(&self, args: &[&str]) -> String { + self.git_at(&self.top, args) + } + + fn write(&self, relative: &str, contents: &str) { + let path = self.root.join(relative); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, contents).unwrap(); + } + + fn commit(&self, message: &str) { + self.git(&["add", "-A"]); + self.git(&["commit", "-q", "--allow-empty", "-m", message]); + } + + /// A full sync; on a fresh project it builds the index. + fn sync(&self) { + sync_project_once(&self.root).expect("full sync"); + } + + fn sync_paths(&self, paths: &[&str]) { + let db = IndexPaths::resolve(&self.root, None).unwrap().current_db(); + sync_changed_paths( + &self.root, + db, + paths.iter().map(|path| self.root.join(path)), + ) + .expect("incremental sync"); + } + + fn store(&self) -> Store { + let paths = IndexPaths::resolve(&self.root, None).unwrap(); + Store::open_for_read(&paths, Instant::now() + Duration::from_secs(30), || false) + .expect("open the index") + } + + fn pending(&self) -> (PendingChanges, PendingSource) { + pending_project_changes_detailed(&self.root, &self.store()).expect("pending") + } + + fn full(&self) -> PendingChanges { + pending_full_inventory(&self.root, &self.store()).expect("full inventory") + } + + fn record(&self) -> Option { + self.store() + .get_project_metadata(GIT_PENDING_KEY) + .unwrap() + .map(|value| serde_json::from_str(&value).unwrap()) + } + + fn dirty(&self) -> BTreeSet { + self.record().expect("a git record")["dirty"] + .as_array() + .unwrap() + .iter() + .map(|path| path.as_str().unwrap().to_string()) + .collect() + } + + /// The fast path answers, and agrees with the full inventory. + fn assert_fast(&self, context: &str) -> PendingChanges { + let (pending, source) = self.pending(); + assert_eq!( + source, + PendingSource::GitFastPath, + "{context}: fast path taken" + ); + assert_eq!( + pending, + self.full(), + "{context}: fast path equals the inventory" + ); + pending + } + + /// The fast path declines, and the answer is the full inventory's. + fn assert_declines(&self, context: &str) -> PendingChanges { + let (pending, source) = self.pending(); + assert_eq!(source, PendingSource::FullInventory, "{context}: declined"); + assert_eq!(pending, self.full(), "{context}"); + pending + } +} + +impl Drop for Repo { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.top); + } +} + +/// A project with two committed sources, indexed. +fn indexed(tag: &str) -> Option { + if !git_available() { + return None; + } + let repo = Repo::new(tag); + repo.write("src/a.ts", "export const a = 1;\n"); + repo.write("src/b.ts", "export const b = 1;\n"); + repo.commit("init"); + repo.sync(); + Some(repo) +} + +fn paths(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).to_string()).collect() +} + +#[test] +fn clean_edits_and_commits_take_the_fast_path() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("edits") else { return }; + assert!(repo.assert_fast("clean").is_empty()); + + repo.write("src/a.ts", "export const a = 2;\n"); + assert_eq!(repo.assert_fast("an edit").modified, paths(&["src/a.ts"])); + + // #1829: committing the edit must not make it read as indexed. + repo.commit("edit a"); + assert_eq!( + repo.assert_fast("an edit, committed").modified, + paths(&["src/a.ts"]) + ); + + repo.write("src/c.ts", "export const c = 1;\n"); + repo.commit("add c"); + assert_eq!( + repo.assert_fast("a new file, committed").added, + paths(&["src/c.ts"]) + ); + + fs::remove_file(repo.root.join("src/b.ts")).unwrap(); + assert_eq!( + repo.assert_fast("a tracked delete").removed, + paths(&["src/b.ts"]) + ); +} + +#[test] +fn a_commit_reverted_by_a_new_commit_reads_clean() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("revert") else { + return; + }; + repo.write("src/a.ts", "export const a = 2;\n"); + repo.commit("edit a"); + repo.git(&["revert", "--no-edit", "HEAD"]); + assert!(repo.assert_fast("edit then revert").is_empty()); +} + +#[test] +fn a_file_indexed_dirty_then_restored_is_modified() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("restore") else { + return; + }; + repo.write("src/a.ts", "export const a = 99;\n"); + repo.sync(); // indexes the dirty content + repo.git(&["checkout", "--", "."]); + assert_eq!(repo.assert_fast("restored").modified, paths(&["src/a.ts"])); +} + +#[test] +fn untracked_files_are_followed_through_add_edit_and_delete() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("untracked") else { + return; + }; + repo.write("src/u.ts", "export const u = 1;\n"); + assert_eq!( + repo.assert_fast("untracked add").added, + paths(&["src/u.ts"]) + ); + repo.sync(); + repo.write("src/u.ts", "export const u = 22;\n"); + assert_eq!( + repo.assert_fast("untracked edit").modified, + paths(&["src/u.ts"]) + ); + fs::remove_file(repo.root.join("src/u.ts")).unwrap(); + assert_eq!( + repo.assert_fast("untracked delete").removed, + paths(&["src/u.ts"]) + ); +} + +#[test] +fn an_amended_stamp_that_still_exists_keeps_the_fast_path() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("amend") else { return }; + repo.write("src/a.ts", "export const a = 3;\n"); + repo.git(&["add", "-A"]); + repo.git(&["commit", "-q", "--amend", "--no-edit"]); + assert_eq!(repo.assert_fast("amended").modified, paths(&["src/a.ts"])); +} + +#[test] +fn an_incremental_sync_then_a_commit_reads_clean() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("incremental") else { + return; + }; + repo.write("src/a.ts", "export const a = 4;\n"); + repo.sync_paths(&["src/a.ts"]); + assert!( + repo.dirty().contains("src/a.ts"), + "the requested path is recorded" + ); + repo.commit("edit a"); + assert!(repo.assert_fast("incremental then commit").is_empty()); +} + +/// The scan keeps `.cache` although the watcher does not watch it. +#[test] +fn a_file_the_watcher_skips_but_the_scan_keeps_is_classified_by_the_scan() { + let _hooks = hooks_guard(); + if !git_available() { + return; + } + let repo = Repo::new("cache"); + repo.write("src/a.ts", "export const a = 1;\n"); + repo.write(".cache/c.ts", "export const c = 1;\n"); + repo.commit("init"); + repo.sync(); + repo.write(".cache/c.ts", "export const c = 2;\n"); + assert_eq!( + repo.assert_fast(".cache edit").modified, + paths(&[".cache/c.ts"]) + ); +} + +#[test] +fn a_project_below_its_work_tree_root_rebases_git_paths() { + let _hooks = hooks_guard(); + if !git_available() { + return; + } + let repo = Repo::in_subdir("subdir"); + repo.write("src/a.ts", "export const a = 1;\n"); + fs::write(repo.top.join("outside.ts"), "export const o = 1;\n").unwrap(); + repo.commit("init"); + repo.sync(); + assert!(repo.assert_fast("subdir clean").is_empty()); + repo.write("src/a.ts", "export const a = 2;\n"); + repo.commit("edit"); + assert_eq!( + repo.assert_fast("subdir commit").modified, + paths(&["src/a.ts"]) + ); + repo.sync_paths(&["src/a.ts"]); + assert!( + repo.dirty().contains("src/a.ts"), + "dirty is project-relative" + ); + assert!(repo.assert_fast("subdir after incremental").is_empty()); + + // An ignored file the scan keeps is invisible to git status: decline. + fs::write(repo.root.join("src/.gitignore"), "local.ts\n").unwrap(); + repo.write("src/local.ts", "export const l = 1;\n"); + repo.assert_declines("subdir nested .gitignore"); +} + +#[test] +fn no_git_an_empty_repo_or_no_record_declines() { + let _hooks = hooks_guard(); + if !git_available() { + return; + } + let plain = std::env::temp_dir().join(format!("cg_git_pending_plain_{}", std::process::id())); + let _ = fs::remove_dir_all(&plain); + fs::create_dir_all(plain.join("src")).unwrap(); + fs::write(plain.join("src/a.ts"), "export const a = 1;\n").unwrap(); + sync_project_once(&plain).unwrap(); + let paths = IndexPaths::resolve(&plain, None).unwrap(); + let store = + Store::open_for_read(&paths, Instant::now() + Duration::from_secs(30), || false).unwrap(); + let (_, source) = pending_project_changes_detailed(&plain, &store).unwrap(); + assert_eq!(source, PendingSource::FullInventory, "not a git work tree"); + drop(store); + fs::remove_dir_all(&plain).ok(); + + let empty = Repo::new("empty"); + empty.write("src/a.ts", "export const a = 1;\n"); + empty.sync(); + empty.assert_declines("a repository without a commit"); + + let Some(repo) = indexed("no-record") else { + return; + }; + let db = IndexPaths::resolve(&repo.root, None).unwrap().current_db(); + rusqlite::Connection::open(db) + .unwrap() + .execute( + "DELETE FROM project_metadata WHERE key = ?1", + [GIT_PENDING_KEY], + ) + .unwrap(); + repo.assert_declines("an index from before the record existed"); +} + +#[test] +fn an_unknown_stamp_or_a_new_scope_declines() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("stamp") else { return }; + // Rebuild on a commit, then make that commit vanish entirely. + repo.write("src/a.ts", "export const a = 5;\n"); + repo.commit("doomed"); + repo.sync(); + repo.git(&["reset", "-q", "--hard", "HEAD~1"]); + repo.git(&["reflog", "expire", "--expire=now", "--all"]); + repo.git(&["gc", "-q", "--prune=now"]); + repo.assert_declines("the stamp is gone"); + + let Some(repo) = indexed("scope") else { return }; + fs::write(repo.root.join(".gitignore"), "src/b.ts\n").unwrap(); + repo.assert_declines("the root .gitignore changed"); +} + +#[test] +fn nested_repositories_submodules_and_hidden_changes_decline() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("nested") else { + return; + }; + let nested = repo.root.join("vendored"); + fs::create_dir_all(&nested).unwrap(); + repo.git_at(&nested, &["init", "-q", "."]); + fs::write(nested.join("v.ts"), "export const v = 1;\n").unwrap(); + repo.assert_declines("an untracked nested repository"); + + let Some(repo) = indexed("assume") else { + return; + }; + repo.git(&["update-index", "--assume-unchanged", "src/a.ts"]); + repo.write("src/a.ts", "export const a = 6;\n"); + assert_eq!( + repo.assert_declines("assume-unchanged").modified, + paths(&["src/a.ts"]) + ); + + if !git_available() { + return; + } + let source = Repo::new("submodule-source"); + source.write("lib.ts", "export const lib = 1;\n"); + source.commit("lib"); + let host = Repo::new("submodule-host"); + host.write("src/a.ts", "export const a = 1;\n"); + host.git(&[ + "submodule", + "add", + "-q", + source.top.to_str().unwrap(), + "sub", + ]); + host.git(&["config", "-f", ".gitmodules", "submodule.sub.ignore", "all"]); + host.commit("submodule"); + host.sync(); + fs::write(host.root.join("sub/lib.ts"), "export const lib = 2;\n").unwrap(); + assert_eq!( + host.assert_declines("a submodule with ignore = all") + .modified, + paths(&["sub/lib.ts"]) + ); +} + +#[test] +fn an_ignored_file_the_scan_includes_declines() { + let _hooks = hooks_guard(); + if !git_available() { + return; + } + let repo = Repo::new("include"); + repo.write(".gitignore", "gen/\n"); + repo.write("src/a.ts", "export const a = 1;\n"); + repo.write("gen/keep.ts", "export const k = 1;\n"); + repo.commit("init"); + repo.sync(); + // Configure the include once the index exists, then let a full sync take it. + repo.write( + ".codegraph/config.toml", + "[app]\nname = \"cg\"\n\n[indexing]\ninclude = [\"gen/keep.ts\"]\n", + ); + repo.sync(); + repo.write("gen/keep.ts", "export const k = 2;\n"); + assert_eq!( + repo.assert_declines("git ignores a file the scan includes") + .modified, + paths(&["gen/keep.ts"]) + ); +} + +#[test] +fn symlinks_decline() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("links") else { return }; + let outside = repo.top.with_extension("outside"); + fs::create_dir_all(&outside).unwrap(); + fs::write(outside.join("o.ts"), "export const o = 1;\n").unwrap(); + #[cfg(unix)] + let made = std::os::unix::fs::symlink(&outside, repo.root.join("linked")); + #[cfg(windows)] + let made = std::os::windows::fs::symlink_dir(&outside, repo.root.join("linked")); + match made { + Ok(()) => {} + Err(error) if std::env::var_os("CI").is_some() => { + panic!("CI must be able to create symlinks: {error}") + } + Err(_) => return, + } + repo.assert_declines("a new symlink to a directory"); + repo.sync(); + assert_eq!(repo.record().unwrap()["links"], serde_json::json!(true)); + repo.assert_declines("an index built through a symlink"); + fs::remove_dir_all(&outside).ok(); +} + +#[cfg(unix)] +#[test] +fn a_hanging_git_declines_within_the_timeout() { + use std::os::unix::fs::PermissionsExt as _; + + let _hooks = hooks_guard(); + let Some(repo) = indexed("timeout") else { + return; + }; + let fake = repo.top.with_extension("fake-git"); + fs::write(&fake, "#!/bin/sh\nsleep 30\n").unwrap(); + fs::set_permissions(&fake, fs::Permissions::from_mode(0o755)).unwrap(); + git_pending_hooks::set_git_program(Some(fake.clone())); + git_pending_hooks::set_git_timeout(Some(Duration::from_millis(300))); + let started = Instant::now(); + let declined = repo.pending(); + git_pending_hooks::set_git_program(None); + git_pending_hooks::set_git_timeout(None); + assert_eq!(declined.1, PendingSource::FullInventory); + assert!( + started.elapsed() < Duration::from_secs(20), + "the timeout bounds the wait" + ); + fs::remove_file(&fake).ok(); +} + +#[test] +fn a_full_build_and_a_full_sync_write_the_record() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("writers") else { + return; + }; + let first = repo.record().expect("a full build writes the record"); + let head = repo.git(&["rev-parse", "HEAD"]).trim().to_string(); + assert_eq!(first["commit"], serde_json::json!(head)); + repo.write("src/a.ts", "export const a = 7;\n"); + repo.commit("next"); + repo.sync(); + let second = repo.record().expect("a full sync replaces the record"); + let head = repo.git(&["rev-parse", "HEAD"]).trim().to_string(); + assert_eq!(second["commit"], serde_json::json!(head)); +} + +/// A file read in a transient state and restored before the record is written +/// still reaches `dirty` (its stat moved). +#[test] +fn a_file_changed_and_restored_during_the_build_stays_dirty() { + let _hooks = hooks_guard(); + if !git_available() { + return; + } + let repo = Repo::new("race-read"); + repo.write("src/a.ts", "export const a = 1;\n"); + repo.commit("init"); + let file = repo.root.join("src/a.ts"); + let original = fs::read(&file).unwrap(); + let mutate = file.clone(); + git_pending_hooks::set_after_git_begin(Some(Box::new(move |_| { + fs::write(&mutate, "export const a = 123;\n").unwrap(); + }))); + let restore = file.clone(); + git_pending_hooks::set_before_git_record(Some(Box::new(move |_| { + fs::write(&restore, &original).unwrap(); + let later = SystemTime::now() + Duration::from_secs(2); + fs::File::options() + .write(true) + .open(&restore) + .unwrap() + .set_modified(later) + .unwrap(); + }))); + repo.sync(); + git_pending_hooks::set_after_git_begin(None); + git_pending_hooks::set_before_git_record(None); + assert!(repo.dirty().contains("src/a.ts"), "{:?}", repo.dirty()); + assert_eq!( + repo.assert_fast("restored after read").modified, + paths(&["src/a.ts"]) + ); +} + +/// A file the scan never saw — briefly gone, or briefly excluded — reaches +/// `dirty` through the fresh-scan comparison. +#[test] +fn a_file_missing_from_the_scan_and_then_restored_stays_dirty() { + let _hooks = hooks_guard(); + if !git_available() { + return; + } + for exclude in [false, true] { + let repo = Repo::new(if exclude { "race-ignore" } else { "race-gone" }); + repo.write("src/a.ts", "export const a = 1;\n"); + repo.write("src/z.ts", "export const z = 1;\n"); + repo.commit("init"); + let root = repo.root.clone(); + git_pending_hooks::set_after_git_begin(Some(Box::new(move |_| { + if exclude { + fs::write(root.join(".gitignore"), "src/z.ts\n").unwrap(); + } else { + fs::rename(root.join("src/z.ts"), root.join("z.ts.away")).unwrap(); + } + }))); + let root = repo.root.clone(); + git_pending_hooks::set_before_git_record(Some(Box::new(move |_| { + if exclude { + fs::remove_file(root.join(".gitignore")).unwrap(); + } else { + fs::rename(root.join("z.ts.away"), root.join("src/z.ts")).unwrap(); + } + }))); + repo.sync(); + git_pending_hooks::set_after_git_begin(None); + git_pending_hooks::set_before_git_record(None); + assert!( + repo.dirty().contains("src/z.ts"), + "{exclude}: {:?}", + repo.dirty() + ); + assert_eq!( + repo.assert_fast("omitted then restored").added, + paths(&["src/z.ts"]), + "exclude={exclude}" + ); + } +} + +/// A full sync forgets the record before any row moves, so an interrupted +/// one can never leave a stale record behind. +#[test] +fn a_full_sync_forgets_the_record_before_it_scans() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("forget") else { + return; + }; + assert!(repo.record().is_some()); + let db = IndexPaths::resolve(&repo.root, None).unwrap().current_db(); + let absent = Arc::new(AtomicBool::new(false)); + let seen = Arc::clone(&absent); + git_pending_hooks::set_after_git_begin(Some(Box::new(move |_| { + let conn = + rusqlite::Connection::open_with_flags(&db, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .unwrap(); + let present: Option = conn + .query_row( + "SELECT value FROM project_metadata WHERE key = ?1", + [GIT_PENDING_KEY], + |row| row.get(0), + ) + .ok(); + seen.store(present.is_none(), Ordering::SeqCst); + }))); + repo.sync(); + git_pending_hooks::set_after_git_begin(None); + assert!( + absent.load(Ordering::SeqCst), + "the record was gone during the sync" + ); + assert!(repo.record().is_some(), "and written again at its end"); +} diff --git a/docs/cli.md b/docs/cli.md index c3baab66..1261fceb 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -60,6 +60,27 @@ reuses full-sync include/exclude/custom-extension rules and the same non-empty. Running status acquires only the normal read lease and does not mutate the database. +In a git work tree, a full `index` or `sync` also records the commit it started at. +It records every path that may differ from that commit as well: paths git reported +dirty during the build, files whose size or mtime moved while it ran, and paths a +fresh scan and the database disagree on. Incremental syncs add the paths they +handle. `status` then classifies only git's candidates and those recorded paths, +by the same rules: what changed between the recorded commit and `HEAD`, plus what +`git status` reports now. The answer is the same as the full inventory's. The full +inventory still answers whenever git might not see a change: + +- there is no record, the commit no longer exists, or the repository has no commit; +- the scope changed: config, root `.gitignore`, extension overrides, index root, + or binary version; +- the index was built through symlinks, or the repository has submodules; +- an untracked nested repository or an `assume-unchanged` or `skip-worktree` entry + is present; +- git ignores a path the scan keeps, such as one under a nested `.gitignore` or an + `include` override; +- git is missing or does not answer within 10 seconds. + +Git runs without optional locks, so `status` never rewrites `.git/index`. + ### Project-path argument contract Lifecycle commands (`init`, `uninit`, `index`, `sync`, `status`, `unlock`) diff --git a/docs/mcp.md b/docs/mcp.md index ea15669c..ac06ea3e 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -441,6 +441,8 @@ scope differs from the persisted file inventory. It sees committed-but-unindexed work as well as working-tree and non-Git changes, applies the same project scope and content-hash gates as whole-tree sync, and lists each added/modified/removed path. The check is read-only and holds the engine's existing shared index lease. +In a git work tree it takes the same git fast path as `codegraph status`, and it +falls back to the full inventory under the same conditions. **`codegraph_export`** dumps the complete graph as NetworkX node-link JSON. Useful for external visualization tools, custom analysis scripts, or feeding an From e2a1fa48e1b7306707cb930e5cf0affd8db343c8 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 17:42:08 +0800 Subject: [PATCH 02/10] docs(upstream): record the #1878 pending-status fast path --- docs/upstream-sync/UPSTREAM.md | 48 ++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/docs/upstream-sync/UPSTREAM.md b/docs/upstream-sync/UPSTREAM.md index afbc2d67..cbe60a43 100644 --- a/docs/upstream-sync/UPSTREAM.md +++ b/docs/upstream-sync/UPSTREAM.md @@ -46,8 +46,9 @@ below remain immutable historical evidence. > did not follow symlinks, so a watcher that did would have watched what was > never indexed. It landed afterwards in #288, together with the #935 scan > port; see the 2026-10-01 symlink entry below. -> - **#1829 / #1878 git-stamped pending status stays KEEP-RUST** (a full -> inventory); its fast path is deferred pending a measurement. +> - **#1829 / #1878 git-stamped pending status was a full inventory at +> `v0.51.0`**, with the fast path deferred pending a measurement. The fast +> path landed afterwards in #289; see the 2026-10-01 pending-status entry below. > - **KEEP-RUST divergences** are recorded row by row in the audit — among them > a parameter-pack constructor declines its overload set (upstream admits it > unbounded), a self-closing `` is never a statement-map root, and a @@ -118,6 +119,49 @@ below remain immutable historical evidence. ## Sync log +### 2026-10-01 — #1878 git-stamped pending status LANDED (#289) + +Owner decision, 2026-10-01: port #1878's fast path. The design passed the +kirocodex plan-convergence review in round 4. + +#289 keeps the full inventory as the reference answer. A full build or full sync +records, under `project_metadata` key `git_pending_state`: + +- the commit it started at; +- every path that may differ from that commit: paths git reported dirty, rows + whose stat moved during the build, and paths a fresh scan and the database + disagree on. + +Incremental syncs add the paths they handle. `status` then classifies only git's +candidates plus the recorded paths, by the full inventory's own rules. + +KEEP-RUST: + +- **One record, not two keys.** Upstream keeps a commit key and a dirty-path key. +- **Race guards:** the stat-moved and fresh-scan comparisons. +- **Declines to the full inventory**, beyond upstream's missing- or + unknown-stamp checks: + - a changed scope; + - an index built through symlinks; + - submodules; + - an untracked nested repository; + - `assume-unchanged` or `skip-worktree` entries; + - an ignored path the scan keeps (the scan honors only the root + `.gitignore`); + - a git timeout. + +Measured on the code at `ea0d069`, release build, warm cache, five runs after +a warm-up, `codegraph status . --json`. The corpus was a generated repository of +40,000 TypeScript files on ext4, on a 32-CPU Linux host. Medians, with +min–max: + +| Scenario | Fast path | Full inventory | +| ------------------- | ----------------- | ----------------- | +| Clean | 183 ms (175–183) | 262 ms (259–263) | +| 18 committed edits | 171 ms (167–181) | 260 ms (255–260) | + +Both columns reported the same pending changes in every run. + ### 2026-10-01 — #935 row CORRECTED; #935 and #770 symlink following LANDED on main (#288) The 2026-06-24 `v1.0.1 → v1.1.0` entry below records "In-root symlink indexing From 5cc457ffc8b4d72a63ee4aa0dc88a3489febd1aa Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 17:55:32 +0800 Subject: [PATCH 03/10] fix(status): expose git_pending, test interrupted writers, and note the unreleased landings --- crates/codegraph-watch/src/git_pending.rs | 71 ++++++-- crates/codegraph-watch/src/lib.rs | 2 +- .../tests/git_pending_fast_path.rs | 172 +++++++++++++++++- docs/upstream-sync/UPSTREAM.md | 7 + 4 files changed, 236 insertions(+), 16 deletions(-) diff --git a/crates/codegraph-watch/src/git_pending.rs b/crates/codegraph-watch/src/git_pending.rs index afaa1de2..87b32c6b 100644 --- a/crates/codegraph-watch/src/git_pending.rs +++ b/crates/codegraph-watch/src/git_pending.rs @@ -406,6 +406,7 @@ pub(crate) fn extend_for_incremental( } _ => forget(store)?, } + test_hooks::after_incremental_extend(root); Ok(()) } @@ -505,9 +506,11 @@ pub(crate) fn git_fast_pending( Ok(Some(pending)) } -/// Seams for tests that race the writers: a hook that runs right after -/// [`GitIndexCapture::begin`] captures git (before the build scans), and one -/// right before [`GitIndexCapture::record`] (after every file was read). +/// Seams for tests that race or break the writers: hooks right after +/// [`GitIndexCapture::begin`] captures git (before the build scans), right +/// before [`GitIndexCapture::record`] (after every file was read), and right +/// after an incremental sync updated the record (before any row moves). A +/// panicking hook must not poison the next test, so locks tolerate poison. pub mod test_hooks { use std::path::Path; @@ -518,15 +521,42 @@ pub mod test_hooks { static AFTER_GIT_BEGIN: std::sync::Mutex> = std::sync::Mutex::new(None); #[cfg(feature = "test-hooks")] static BEFORE_GIT_RECORD: std::sync::Mutex> = std::sync::Mutex::new(None); + #[cfg(feature = "test-hooks")] + static AFTER_INCREMENTAL_EXTEND: std::sync::Mutex> = std::sync::Mutex::new(None); + + /// Runs once an incremental sync has extended (or forgotten) the record, + /// before it writes a row. + #[cfg(feature = "test-hooks")] + pub fn set_after_incremental_extend(hook: Option) { + *AFTER_INCREMENTAL_EXTEND + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = hook; + } + + pub(super) fn after_incremental_extend(root: &Path) { + #[cfg(feature = "test-hooks")] + if let Some(hook) = AFTER_INCREMENTAL_EXTEND + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + { + hook(root); + } + let _ = root; + } #[cfg(feature = "test-hooks")] pub fn set_after_git_begin(hook: Option) { - *AFTER_GIT_BEGIN.lock().unwrap() = hook; + *AFTER_GIT_BEGIN + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = hook; } #[cfg(feature = "test-hooks")] pub fn set_before_git_record(hook: Option) { - *BEFORE_GIT_RECORD.lock().unwrap() = hook; + *BEFORE_GIT_RECORD + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = hook; } #[cfg(feature = "test-hooks")] @@ -537,18 +567,26 @@ pub mod test_hooks { /// Run this program instead of `git` (a stand-in that hangs, say). #[cfg(feature = "test-hooks")] pub fn set_git_program(program: Option) { - *GIT_PROGRAM.lock().unwrap() = program; + *GIT_PROGRAM + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = program; } /// Give up on git after this long instead of the default. #[cfg(feature = "test-hooks")] pub fn set_git_timeout(timeout: Option) { - *GIT_TIMEOUT.lock().unwrap() = timeout; + *GIT_TIMEOUT + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = timeout; } pub(super) fn git_program() -> std::ffi::OsString { #[cfg(feature = "test-hooks")] - if let Some(program) = GIT_PROGRAM.lock().unwrap().as_ref() { + if let Some(program) = GIT_PROGRAM + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + { return program.clone().into_os_string(); } "git".into() @@ -556,7 +594,10 @@ pub mod test_hooks { pub(super) fn git_timeout(default: std::time::Duration) -> std::time::Duration { #[cfg(feature = "test-hooks")] - if let Some(timeout) = *GIT_TIMEOUT.lock().unwrap() { + if let Some(timeout) = *GIT_TIMEOUT + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + { return timeout; } default @@ -564,7 +605,11 @@ pub mod test_hooks { pub(super) fn after_git_begin(root: &Path) { #[cfg(feature = "test-hooks")] - if let Some(hook) = AFTER_GIT_BEGIN.lock().unwrap().as_ref() { + if let Some(hook) = AFTER_GIT_BEGIN + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + { hook(root); } let _ = root; @@ -572,7 +617,11 @@ pub mod test_hooks { pub(super) fn before_git_record(root: &Path) { #[cfg(feature = "test-hooks")] - if let Some(hook) = BEFORE_GIT_RECORD.lock().unwrap().as_ref() { + if let Some(hook) = BEFORE_GIT_RECORD + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_ref() + { hook(root); } let _ = root; diff --git a/crates/codegraph-watch/src/lib.rs b/crates/codegraph-watch/src/lib.rs index d2c17d73..ab5bb2ac 100644 --- a/crates/codegraph-watch/src/lib.rs +++ b/crates/codegraph-watch/src/lib.rs @@ -1,5 +1,5 @@ mod git; -mod git_pending; +pub mod git_pending; mod link_state; mod migrate; mod policy; diff --git a/crates/codegraph-watch/tests/git_pending_fast_path.rs b/crates/codegraph-watch/tests/git_pending_fast_path.rs index f75924d7..c0c8f3e7 100644 --- a/crates/codegraph-watch/tests/git_pending_fast_path.rs +++ b/crates/codegraph-watch/tests/git_pending_fast_path.rs @@ -42,6 +42,53 @@ fn hooks_guard() -> std::sync::MutexGuard<'static, ()> { .unwrap_or_else(std::sync::PoisonError::into_inner) } +fn codegraph_dir() -> Option { + std::env::var("CODEGRAPH_DIR").ok() +} + +/// Sets `CODEGRAPH_DIR` for one test (every test here holds [`hooks_guard`], +/// so none runs alongside) and restores it on drop. +struct IndexDirGuard(Option); + +impl IndexDirGuard { + fn set(name: &str) -> Self { + let previous = std::env::var_os("CODEGRAPH_DIR"); + // SAFETY: tests in this binary are serialized by `hooks_guard`. + unsafe { std::env::set_var("CODEGRAPH_DIR", name) }; + Self(previous) + } +} + +impl Drop for IndexDirGuard { + fn drop(&mut self) { + // SAFETY: as in `set`. + unsafe { + match self.0.take() { + Some(value) => std::env::set_var("CODEGRAPH_DIR", value), + None => std::env::remove_var("CODEGRAPH_DIR"), + } + } + } +} + +/// The record straight from the database file, readable even when an +/// interrupted writer left the namespace unreadable through the state gate. +fn raw_record(root: &Path) -> Option { + let db = IndexPaths::resolve(root, codegraph_dir().as_deref()) + .unwrap() + .current_db(); + let conn = + rusqlite::Connection::open_with_flags(&db, rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY) + .unwrap(); + conn.query_row( + "SELECT value FROM project_metadata WHERE key = ?1", + [GIT_PENDING_KEY], + |row| row.get::<_, String>(0), + ) + .ok() + .map(|value| serde_json::from_str(&value).unwrap()) +} + fn git_available() -> bool { Command::new("git") .arg("--version") @@ -138,7 +185,9 @@ impl Repo { } fn sync_paths(&self, paths: &[&str]) { - let db = IndexPaths::resolve(&self.root, None).unwrap().current_db(); + let db = IndexPaths::resolve(&self.root, codegraph_dir().as_deref()) + .unwrap() + .current_db(); sync_changed_paths( &self.root, db, @@ -148,7 +197,7 @@ impl Repo { } fn store(&self) -> Store { - let paths = IndexPaths::resolve(&self.root, None).unwrap(); + let paths = IndexPaths::resolve(&self.root, codegraph_dir().as_deref()).unwrap(); Store::open_for_read(&paths, Instant::now() + Duration::from_secs(30), || false) .expect("open the index") } @@ -406,7 +455,9 @@ fn no_git_an_empty_repo_or_no_record_declines() { let Some(repo) = indexed("no-record") else { return; }; - let db = IndexPaths::resolve(&repo.root, None).unwrap().current_db(); + let db = IndexPaths::resolve(&repo.root, codegraph_dir().as_deref()) + .unwrap() + .current_db(); rusqlite::Connection::open(db) .unwrap() .execute( @@ -669,7 +720,9 @@ fn a_full_sync_forgets_the_record_before_it_scans() { return; }; assert!(repo.record().is_some()); - let db = IndexPaths::resolve(&repo.root, None).unwrap().current_db(); + let db = IndexPaths::resolve(&repo.root, codegraph_dir().as_deref()) + .unwrap() + .current_db(); let absent = Arc::new(AtomicBool::new(false)); let seen = Arc::clone(&absent); git_pending_hooks::set_after_git_begin(Some(Box::new(move |_| { @@ -693,3 +746,114 @@ fn a_full_sync_forgets_the_record_before_it_scans() { ); assert!(repo.record().is_some(), "and written again at its end"); } + +/// `CODEGRAPH_DIR` names a single project-local index root. The watcher's +/// policy only knows `.codegraph*`, but the scan prunes the exact root, so +/// the index's own files never become candidates. +#[test] +fn a_custom_index_root_is_excluded_by_the_scan_rules() { + let _hooks = hooks_guard(); + if !git_available() { + return; + } + let _dir = IndexDirGuard::set("cgidx"); + let repo = Repo::new("custom-root"); + fs::write(repo.top.join(".git/info/exclude"), "").unwrap(); + repo.write("src/a.ts", "export const a = 1;\n"); + repo.commit("init"); + repo.sync(); + assert!( + repo.root.join("cgidx").is_dir(), + "the index lives in the custom root" + ); + assert!(repo.assert_fast("custom root, clean").is_empty()); + repo.write("src/a.ts", "export const a = 2;\n"); + assert_eq!( + repo.assert_fast("custom root, an edit").modified, + paths(&["src/a.ts"]) + ); +} + +#[test] +fn a_submodule_below_a_subdirectory_project_declines() { + let _hooks = hooks_guard(); + if !git_available() { + return; + } + let source = Repo::new("subdir-submodule-source"); + source.write("lib.ts", "export const lib = 1;\n"); + source.commit("lib"); + let repo = Repo::in_subdir("subdir-submodule"); + repo.write("src/a.ts", "export const a = 1;\n"); + repo.commit("init"); + repo.git(&[ + "submodule", + "add", + "-q", + source.top.to_str().unwrap(), + "packages/app/vendor", + ]); + repo.commit("submodule"); + repo.sync(); + assert_eq!( + repo.record().expect("a record")["submodules"], + serde_json::json!(true), + "the --full-name gitlink check sees a submodule below the project" + ); + repo.assert_declines("a submodule below a subdirectory project"); +} + +/// A full sync that dies after forgetting the record and before writing the +/// new one leaves no record, never a stale one. +#[test] +fn an_interrupted_full_sync_leaves_no_record() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("interrupted-full") else { + return; + }; + assert!(raw_record(&repo.root).is_some()); + repo.write("src/a.ts", "export const a = 8;\n"); + git_pending_hooks::set_before_git_record(Some(Box::new(|_| { + panic!("injected failure before the record is written") + }))); + let root = repo.root.clone(); + let outcome = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| sync_project_once(&root))); + git_pending_hooks::set_before_git_record(None); + assert!( + outcome.is_err(), + "the injected failure interrupted the sync" + ); + assert!(raw_record(&repo.root).is_none(), "no stale record survives"); +} + +/// An incremental sync records its paths before any row moves, so one that +/// dies right after still leaves them in `dirty`. +#[test] +fn an_interrupted_incremental_sync_keeps_its_paths_recorded() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("interrupted-incremental") else { + return; + }; + repo.write("src/a.ts", "export const a = 9;\n"); + git_pending_hooks::set_after_incremental_extend(Some(Box::new(|_| { + panic!("injected failure before any row moves") + }))); + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + repo.sync_paths(&["src/a.ts"]) + })); + git_pending_hooks::set_after_incremental_extend(None); + assert!( + outcome.is_err(), + "the injected failure interrupted the sync" + ); + let record = raw_record(&repo.root).expect("the record survives"); + assert!( + record["dirty"] + .as_array() + .unwrap() + .iter() + .any(|path| path == "src/a.ts"), + "{record}" + ); +} diff --git a/docs/upstream-sync/UPSTREAM.md b/docs/upstream-sync/UPSTREAM.md index cbe60a43..e69a8d93 100644 --- a/docs/upstream-sync/UPSTREAM.md +++ b/docs/upstream-sync/UPSTREAM.md @@ -23,6 +23,13 @@ `f4ddf508516332419ea3c95702810765936cf679..origin/main`. Do not re-triage the audited `v1.6.1` range. - **Upstream repository:** . +- **Landed after `v0.51.0`, not yet released:** + - #286, which makes a full `sync` keep what `index` keeps; + - #288, symlink following (#935, #770); + - #289, the #1878 git fast path for pending status. + + All three have dated entries below and ship in the next release. The two + `v0.51.0` exceptions #770 and #1829/#1878 are closed on main. - **Deferred UI family:** the browser viewer and the graph-semantic families F1–F12 that shipped beside it in `v1.6.1` (framework navigation edges and routers, the synthesis layer and its schema, cross-tier channels, server route From f75033541637ae6db03b8f30c1b2399a60e683a0 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 18:01:08 +0800 Subject: [PATCH 04/10] fix(status): decline the git fast path when git may call changed bytes or names clean --- crates/codegraph-watch/src/git_pending.rs | 73 ++++++++++++++++++- .../tests/git_pending_fast_path.rs | 60 +++++++++++++-- 2 files changed, 123 insertions(+), 10 deletions(-) diff --git a/crates/codegraph-watch/src/git_pending.rs b/crates/codegraph-watch/src/git_pending.rs index 87b32c6b..e5b51552 100644 --- a/crates/codegraph-watch/src/git_pending.rs +++ b/crates/codegraph-watch/src/git_pending.rs @@ -20,7 +20,9 @@ //! - the recorded commit is no longer in the repository; //! - an untracked nested repository, a directory or symlink candidate, an //! `assume-unchanged` or `skip-worktree` entry, or an ignored path the scan -//! would keep. +//! would keep; +//! - git may call changed bytes clean: `core.autocrlf`, any gitattributes +//! source, or a case-insensitive or Unicode-precomposing name match. use std::collections::BTreeSet; use std::fs; @@ -148,23 +150,34 @@ struct Repo { /// empty at the top level. prefix: String, toplevel: String, + /// The repository's common git directory (holding `info/attributes`). + common_dir: std::path::PathBuf, } impl Repo { fn read(root: &Path) -> Option { let out = git( root, - &["rev-parse", "HEAD", "--show-prefix", "--show-toplevel"], + &[ + "rev-parse", + "HEAD", + "--show-prefix", + "--show-toplevel", + "--git-common-dir", + ], )?; let text = String::from_utf8(out).ok()?; let mut lines = text.lines(); let head = lines.next()?.trim().to_string(); let prefix = lines.next()?.to_string(); let toplevel = lines.next()?.to_string(); + // Printed relative to the working directory unless it lies elsewhere. + let common_dir = root.join(lines.next()?); (!head.is_empty()).then_some(Self { head, prefix, toplevel, + common_dir, }) } @@ -239,6 +252,55 @@ fn committed_paths(root: &Path, repo: &Repo, commit: &str) -> Option Option { + let config = git(root, &["config", "--list", "-z"])?; + for entry in nul_fields(&config) { + let (key, value) = entry.split_once('\n').unwrap_or((entry.as_str(), "")); + let value = value.trim().to_ascii_lowercase(); + let truthy = value.is_empty() || matches!(value.as_str(), "true" | "yes" | "on" | "1"); + let falsy = matches!(value.as_str(), "false" | "no" | "off" | "0"); + let risky = match key.to_ascii_lowercase().as_str() { + "core.autocrlf" => !falsy, + "core.ignorecase" | "core.precomposeunicode" => truthy, + "core.attributesfile" => !value.is_empty(), + _ => false, + }; + if risky { + return Some(true); + } + } + // Git's default global attributes file applies without any config. + let global_attributes = std::env::var_os("XDG_CONFIG_HOME") + .filter(|dir| !dir.is_empty()) + .map(std::path::PathBuf::from) + .or_else(|| std::env::var_os("HOME").map(|home| Path::new(&home).join(".config"))) + .map(|config| config.join("git").join("attributes")); + if global_attributes.is_some_and(|path| path.exists()) + || repo.common_dir.join("info").join("attributes").exists() + { + return Some(true); + } + let attributes = git( + root, + &[ + "ls-files", + "--full-name", + "-z", + "--cached", + "--others", + "--exclude-standard", + "--", + ":(glob)**/.gitattributes", + ], + )?; + Some(nul_fields(&attributes).next().is_some()) +} + /// What the index file says that `git status` cannot: a gitlink (submodule), /// or an entry marked `assume-unchanged` or `skip-worktree`, whose working-tree /// changes git hides. @@ -289,6 +351,10 @@ fn ignores_scanned_path(root: &Path, repo: &Repo, options: &ExtractOptions) -> O let Some(relative) = repo.rebase(&entry) else { continue; }; + // An attributes file applies whether or not git ignores it. + if Path::new(&relative).file_name() == Some(std::ffi::OsStr::new(".gitattributes")) { + return Some(true); + } let membership = match relative.strip_suffix('/') { Some(dir) => scan_membership(root, options, dir, true), None => scan_membership(root, options, &relative, false), @@ -443,6 +509,9 @@ pub(crate) fn git_fast_pending( if flags.submodules || flags.hidden_changes { return Ok(None); } + if conversion_risk(root, &repo) != Some(false) { + return Ok(None); + } let (Some(status), Some(committed)) = ( status_paths(root, &repo), committed_paths(root, &repo, &record.commit), diff --git a/crates/codegraph-watch/tests/git_pending_fast_path.rs b/crates/codegraph-watch/tests/git_pending_fast_path.rs index c0c8f3e7..8c80417c 100644 --- a/crates/codegraph-watch/tests/git_pending_fast_path.rs +++ b/crates/codegraph-watch/tests/git_pending_fast_path.rs @@ -89,6 +89,14 @@ fn raw_record(root: &Path) -> Option { .map(|value| serde_json::from_str(&value).unwrap()) } +/// A path git accepts as a clone source on every platform: no Windows +/// verbatim prefix, forward slashes. +fn git_path(path: &Path) -> String { + let text = path.to_string_lossy().replace('\\', "/"); + text.strip_prefix("//?/") + .map_or(text.clone(), str::to_string) +} + fn git_available() -> bool { Command::new("git") .arg("--version") @@ -133,6 +141,16 @@ impl Repo { repo.git_at(&repo.top, &["init", "-q", "."]); // Keep the index itself out of every commit, as a project does. fs::write(repo.top.join(".git/info/exclude"), ".codegraph/\n").unwrap(); + // The same byte- and case-exact git on every runner (Windows defaults + // to `core.autocrlf=true`, macOS and Windows to `core.ignorecase`); + // the decline tests set them back on purpose. + for (key, value) in [ + ("core.autocrlf", "false"), + ("core.ignorecase", "false"), + ("core.precomposeunicode", "false"), + ] { + repo.git_at(&repo.top, &["config", key, value]); + } repo } @@ -516,13 +534,7 @@ fn nested_repositories_submodules_and_hidden_changes_decline() { source.commit("lib"); let host = Repo::new("submodule-host"); host.write("src/a.ts", "export const a = 1;\n"); - host.git(&[ - "submodule", - "add", - "-q", - source.top.to_str().unwrap(), - "sub", - ]); + host.git(&["submodule", "add", "-q", &git_path(&source.top), "sub"]); host.git(&["config", "-f", ".gitmodules", "submodule.sub.ignore", "all"]); host.commit("submodule"); host.sync(); @@ -790,7 +802,7 @@ fn a_submodule_below_a_subdirectory_project_declines() { "submodule", "add", "-q", - source.top.to_str().unwrap(), + &git_path(&source.top), "packages/app/vendor", ]); repo.commit("submodule"); @@ -857,3 +869,35 @@ fn an_interrupted_incremental_sync_keeps_its_paths_recorded() { "{record}" ); } + +/// Git compares content after line-ending conversion, so a revert can rewrite +/// a file's bytes (LF to CRLF) while `git status` calls it clean. The fast path +/// must decline, and the full inventory reports the changed bytes. +#[test] +fn line_ending_conversion_declines() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("autocrlf") else { + return; + }; + repo.git(&["config", "core.autocrlf", "true"]); + repo.write("src/a.ts", "export const a = 2;\n"); + repo.commit("edit a"); + repo.git(&["revert", "--no-edit", "HEAD"]); + repo.assert_declines("core.autocrlf"); +} + +#[test] +fn gitattributes_and_case_insensitive_names_decline() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("attributes") else { + return; + }; + repo.write(".gitattributes", "* text=auto\n"); + repo.assert_declines("a .gitattributes file"); + + let Some(repo) = indexed("ignorecase") else { + return; + }; + repo.git(&["config", "core.ignorecase", "true"]); + repo.assert_declines("core.ignorecase"); +} From 98c4aea8ff8860d7a678a259f1b1532b95ee3a14 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 18:03:27 +0800 Subject: [PATCH 05/10] docs: list the conversion declines and the final measurement --- docs/cli.md | 5 +++++ docs/upstream-sync/UPSTREAM.md | 10 +++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 1261fceb..c77eab24 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -77,6 +77,11 @@ inventory still answers whenever git might not see a change: is present; - git ignores a path the scan keeps, such as one under a nested `.gitignore` or an `include` override; +- git might call changed bytes or names clean: `core.autocrlf`, any gitattributes + source (`.gitattributes`, `info/attributes`, `core.attributesFile`), or a + case-insensitive or Unicode-precomposing name match (`core.ignorecase`, the + macOS and Windows default; `core.precomposeunicode`). On those systems the full + inventory answers; - git is missing or does not answer within 10 seconds. Git runs without optional locks, so `status` never rewrites `.git/index`. diff --git a/docs/upstream-sync/UPSTREAM.md b/docs/upstream-sync/UPSTREAM.md index e69a8d93..491aa682 100644 --- a/docs/upstream-sync/UPSTREAM.md +++ b/docs/upstream-sync/UPSTREAM.md @@ -155,17 +155,21 @@ KEEP-RUST: - `assume-unchanged` or `skip-worktree` entries; - an ignored path the scan keeps (the scan honors only the root `.gitignore`); + - content or name conversions that let git call changed bytes clean: + `core.autocrlf`, any gitattributes source, `core.ignorecase`, + `core.precomposeunicode`. Windows CI caught this: a revert rewrote LF as + CRLF while `git status` stayed clean; - a git timeout. -Measured on the code at `ea0d069`, release build, warm cache, five runs after +Measured on the code at `f750335`, release build, warm cache, five runs after a warm-up, `codegraph status . --json`. The corpus was a generated repository of 40,000 TypeScript files on ext4, on a 32-CPU Linux host. Medians, with min–max: | Scenario | Fast path | Full inventory | | ------------------- | ----------------- | ----------------- | -| Clean | 183 ms (175–183) | 262 ms (259–263) | -| 18 committed edits | 171 ms (167–181) | 260 ms (255–260) | +| Clean | 198 ms (197–208) | 259 ms (257–263) | +| 18 committed edits | 196 ms (194–209) | 261 ms (255–266) | Both columns reported the same pending changes in every run. From 87d680478364d2dd9818408cc8282e41bbe4c5ab Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 18:12:30 +0800 Subject: [PATCH 06/10] fix(status): resolve the system and global attributes files through git --- crates/codegraph-watch/src/git_pending.rs | 76 +++++++++++++++++-- .../tests/git_pending_fast_path.rs | 39 +++++++++- 2 files changed, 106 insertions(+), 9 deletions(-) diff --git a/crates/codegraph-watch/src/git_pending.rs b/crates/codegraph-watch/src/git_pending.rs index e5b51552..c7fc3c86 100644 --- a/crates/codegraph-watch/src/git_pending.rs +++ b/crates/codegraph-watch/src/git_pending.rs @@ -274,13 +274,15 @@ fn conversion_risk(root: &Path, repo: &Repo) -> Option { return Some(true); } } - // Git's default global attributes file applies without any config. - let global_attributes = std::env::var_os("XDG_CONFIG_HOME") - .filter(|dir| !dir.is_empty()) - .map(std::path::PathBuf::from) - .or_else(|| std::env::var_os("HOME").map(|home| Path::new(&home).join(".config"))) - .map(|config| config.join("git").join("attributes")); - if global_attributes.is_some_and(|path| path.exists()) + // The system (`$(prefix)/etc/gitattributes`) and global (`core.attributesFile`, + // else the XDG default) attributes files, exactly as git resolves them. + // `GIT_ATTR_NOSYSTEM` switches the system file off, as it does for git. + let system_disabled = std::env::var_os("GIT_ATTR_NOSYSTEM") + .is_some_and(|value| !matches!(value.to_str(), Some("" | "0" | "false" | "no" | "off"))); + if !system_disabled && attributes_file_applies(root, "GIT_ATTR_SYSTEM") { + return Some(true); + } + if attributes_file_applies(root, "GIT_ATTR_GLOBAL") || repo.common_dir.join("info").join("attributes").exists() { return Some(true); @@ -301,6 +303,18 @@ fn conversion_risk(root: &Path, repo: &Repo) -> Option { Some(nul_fields(&attributes).next().is_some()) } +/// Whether the attributes file git names for `var` (`GIT_ATTR_SYSTEM`, +/// `GIT_ATTR_GLOBAL`, git 2.42+) exists. A git that cannot say counts as one +/// that applies it. +fn attributes_file_applies(root: &Path, var: &str) -> bool { + let Some(out) = git(root, &["var", var]) else { + return true; + }; + let text = String::from_utf8_lossy(&out); + let path = text.trim_end_matches(['\r', '\n']); + !path.is_empty() && Path::new(path).exists() +} + /// What the index file says that `git status` cannot: a gitlink (submodule), /// or an entry marked `assume-unchanged` or `skip-worktree`, whose working-tree /// changes git hides. @@ -696,3 +710,51 @@ pub mod test_hooks { let _ = root; } } + +#[cfg(all(test, unix, feature = "test-hooks"))] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt as _; + + /// A stand-in git that answers only what `conversion_risk` asks. + fn fake_git(dir: &Path, system_attributes: &Path) -> std::path::PathBuf { + let script = dir.join("git"); + fs::write( + &script, + format!( + "#!/bin/sh\nfor a in \"$@\"; do\n case \"$a\" in\n GIT_ATTR_SYSTEM) echo '{}'; exit 0;;\n GIT_ATTR_GLOBAL) echo '{}'; exit 0;;\n esac\ndone\nexit 0\n", + system_attributes.display(), + dir.join("no-global-attributes").display() + ), + ) + .unwrap(); + fs::set_permissions(&script, fs::Permissions::from_mode(0o755)).unwrap(); + script + } + + #[test] + fn a_system_attributes_file_is_a_conversion_risk() { + let dir = std::env::temp_dir().join(format!("cg_fake_git_{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + let system = dir.join("etc-gitattributes"); + let repo = Repo { + head: "0".repeat(40), + prefix: String::new(), + toplevel: dir.to_string_lossy().into_owned(), + common_dir: dir.join("dot-git"), + }; + test_hooks::set_git_program(Some(fake_git(&dir, &system))); + let without = conversion_risk(&dir, &repo); + fs::write(&system, "* text=auto\n").unwrap(); + let with = conversion_risk(&dir, &repo); + test_hooks::set_git_program(None); + fs::remove_dir_all(&dir).ok(); + assert_eq!(without, Some(false), "no attributes anywhere"); + assert_eq!( + with, + Some(true), + "git reads $(prefix)/etc/gitattributes too" + ); + } +} diff --git a/crates/codegraph-watch/tests/git_pending_fast_path.rs b/crates/codegraph-watch/tests/git_pending_fast_path.rs index 8c80417c..6f3ff616 100644 --- a/crates/codegraph-watch/tests/git_pending_fast_path.rs +++ b/crates/codegraph-watch/tests/git_pending_fast_path.rs @@ -36,10 +36,28 @@ const GIT_ENV: &[&str] = &[ /// them being unset, run one at a time. static HOOKS: std::sync::Mutex<()> = std::sync::Mutex::new(()); +/// An empty `XDG_CONFIG_HOME`: no global git attributes or config leak in. +fn isolated_config_home() -> PathBuf { + std::env::temp_dir().join(format!("cg_git_pending_xdg_{}", std::process::id())) +} + fn hooks_guard() -> std::sync::MutexGuard<'static, ()> { - HOOKS + static ISOLATE: std::sync::Once = std::sync::Once::new(); + let guard = HOOKS .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) + .unwrap_or_else(std::sync::PoisonError::into_inner); + ISOLATE.call_once(|| { + let home = isolated_config_home(); + fs::create_dir_all(&home).unwrap(); + // SAFETY: every test here holds `HOOKS`, so none runs alongside. + unsafe { + // A system attributes file (Git for Windows ships one) or a global + // one would make every case decline; the decline tests add their own. + std::env::set_var("GIT_ATTR_NOSYSTEM", "1"); + std::env::set_var("XDG_CONFIG_HOME", &home); + } + }); + guard } fn codegraph_dir() -> Option { @@ -901,3 +919,20 @@ fn gitattributes_and_case_insensitive_names_decline() { repo.git(&["config", "core.ignorecase", "true"]); repo.assert_declines("core.ignorecase"); } + +/// A global attributes file (the XDG default, as `git var GIT_ATTR_GLOBAL` +/// resolves it) converts content as much as a `.gitattributes` does. +#[test] +fn a_global_attributes_file_declines() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("global-attributes") else { + return; + }; + assert!(repo.assert_fast("no attributes anywhere").is_empty()); + let attributes = isolated_config_home().join("git").join("attributes"); + fs::create_dir_all(attributes.parent().unwrap()).unwrap(); + fs::write(&attributes, "* text=auto\n").unwrap(); + let declined = repo.pending().1; + fs::remove_file(&attributes).unwrap(); + assert_eq!(declined, PendingSource::FullInventory); +} From 8a0cd1cd892f08f55121945d6c38691406b6e178 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 18:16:00 +0800 Subject: [PATCH 07/10] docs: name the attributes sources and the final measurement --- docs/cli.md | 3 ++- docs/upstream-sync/UPSTREAM.md | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index c77eab24..6cdfa9ed 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -78,7 +78,8 @@ inventory still answers whenever git might not see a change: - git ignores a path the scan keeps, such as one under a nested `.gitignore` or an `include` override; - git might call changed bytes or names clean: `core.autocrlf`, any gitattributes - source (`.gitattributes`, `info/attributes`, `core.attributesFile`), or a + source (`.gitattributes`, `info/attributes`, and the global and system attributes + files git resolves, including `core.attributesFile`), or a case-insensitive or Unicode-precomposing name match (`core.ignorecase`, the macOS and Windows default; `core.precomposeunicode`). On those systems the full inventory answers; diff --git a/docs/upstream-sync/UPSTREAM.md b/docs/upstream-sync/UPSTREAM.md index 491aa682..e3119a13 100644 --- a/docs/upstream-sync/UPSTREAM.md +++ b/docs/upstream-sync/UPSTREAM.md @@ -161,15 +161,15 @@ KEEP-RUST: CRLF while `git status` stayed clean; - a git timeout. -Measured on the code at `f750335`, release build, warm cache, five runs after +Measured on the code at `87d6804`, release build, warm cache, five runs after a warm-up, `codegraph status . --json`. The corpus was a generated repository of 40,000 TypeScript files on ext4, on a 32-CPU Linux host. Medians, with min–max: | Scenario | Fast path | Full inventory | | ------------------- | ----------------- | ----------------- | -| Clean | 198 ms (197–208) | 259 ms (257–263) | -| 18 committed edits | 196 ms (194–209) | 261 ms (255–266) | +| Clean | 215 ms (213–225) | 265 ms (265–267) | +| 18 committed edits | 217 ms (209–222) | 263 ms (257–269) | Both columns reported the same pending changes in every run. From 742bcb4c13b952fcdf950fd582dc4febd515a7bb Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 18:17:55 +0800 Subject: [PATCH 08/10] fix(status): judge git config by the value git uses --- crates/codegraph-watch/src/git_pending.rs | 9 +++++-- .../tests/git_pending_fast_path.rs | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/crates/codegraph-watch/src/git_pending.rs b/crates/codegraph-watch/src/git_pending.rs index c7fc3c86..d9fdfdc5 100644 --- a/crates/codegraph-watch/src/git_pending.rs +++ b/crates/codegraph-watch/src/git_pending.rs @@ -259,12 +259,17 @@ fn committed_paths(root: &Path, repo: &Repo, commit: &str) -> Option Option { let config = git(root, &["config", "--list", "-z"])?; + // `--list` prints every scope in precedence order (system, global, local, + // worktree), so the last value of a key is the one git uses. + let mut effective = std::collections::BTreeMap::new(); for entry in nul_fields(&config) { let (key, value) = entry.split_once('\n').unwrap_or((entry.as_str(), "")); - let value = value.trim().to_ascii_lowercase(); + effective.insert(key.to_ascii_lowercase(), value.trim().to_ascii_lowercase()); + } + for (key, value) in &effective { let truthy = value.is_empty() || matches!(value.as_str(), "true" | "yes" | "on" | "1"); let falsy = matches!(value.as_str(), "false" | "no" | "off" | "0"); - let risky = match key.to_ascii_lowercase().as_str() { + let risky = match key.as_str() { "core.autocrlf" => !falsy, "core.ignorecase" | "core.precomposeunicode" => truthy, "core.attributesfile" => !value.is_empty(), diff --git a/crates/codegraph-watch/tests/git_pending_fast_path.rs b/crates/codegraph-watch/tests/git_pending_fast_path.rs index 6f3ff616..f092473e 100644 --- a/crates/codegraph-watch/tests/git_pending_fast_path.rs +++ b/crates/codegraph-watch/tests/git_pending_fast_path.rs @@ -936,3 +936,30 @@ fn a_global_attributes_file_declines() { fs::remove_file(&attributes).unwrap(); assert_eq!(declined, PendingSource::FullInventory); } + +/// Only the value git uses counts: a global `core.autocrlf=true` that the +/// repository overrides with `false` converts nothing (Git for Windows ships +/// `true` in its system config). +#[test] +fn an_overridden_conversion_setting_keeps_the_fast_path() { + let _hooks = hooks_guard(); + let Some(repo) = indexed("overridden") else { + return; + }; + let global = isolated_config_home().join("gitconfig-autocrlf"); + fs::write(&global, "[core]\n\tautocrlf = true\n\tignorecase = true\n").unwrap(); + let previous = std::env::var_os("GIT_CONFIG_GLOBAL"); + // SAFETY: tests in this binary are serialized by `hooks_guard`. + unsafe { std::env::set_var("GIT_CONFIG_GLOBAL", &global) }; + let (pending, source) = repo.pending(); + // SAFETY: as above. + unsafe { + match previous { + Some(value) => std::env::set_var("GIT_CONFIG_GLOBAL", value), + None => std::env::remove_var("GIT_CONFIG_GLOBAL"), + } + } + fs::remove_file(&global).ok(); + assert_eq!(source, PendingSource::GitFastPath, "the local false wins"); + assert!(pending.is_empty()); +} From 4a0f5aff12ea83d476a49c5321867631fd7fac53 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 18:19:45 +0800 Subject: [PATCH 09/10] docs: cite the final pending-status measurement --- docs/upstream-sync/UPSTREAM.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/upstream-sync/UPSTREAM.md b/docs/upstream-sync/UPSTREAM.md index e3119a13..f0508143 100644 --- a/docs/upstream-sync/UPSTREAM.md +++ b/docs/upstream-sync/UPSTREAM.md @@ -161,15 +161,15 @@ KEEP-RUST: CRLF while `git status` stayed clean; - a git timeout. -Measured on the code at `87d6804`, release build, warm cache, five runs after +Measured on the code at `742bcb4`, release build, warm cache, five runs after a warm-up, `codegraph status . --json`. The corpus was a generated repository of 40,000 TypeScript files on ext4, on a 32-CPU Linux host. Medians, with min–max: | Scenario | Fast path | Full inventory | | ------------------- | ----------------- | ----------------- | -| Clean | 215 ms (213–225) | 265 ms (265–267) | -| 18 committed edits | 217 ms (209–222) | 263 ms (257–269) | +| Clean | 212 ms (208–215) | 266 ms (264–266) | +| 18 committed edits | 211 ms (206–219) | 263 ms (262–264) | Both columns reported the same pending changes in every run. From a603d9dcfbe0900710ee67ddb2c82c4630aaa6f8 Mon Sep 17 00:00:00 2001 From: CodeGraph Test Date: Thu, 1 Oct 2026 18:26:23 +0800 Subject: [PATCH 10/10] fix(status): decline when a .gitattributes above a subdirectory project applies --- crates/codegraph-watch/src/git_pending.rs | 21 ++++++++++++++++ .../tests/git_pending_fast_path.rs | 25 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/crates/codegraph-watch/src/git_pending.rs b/crates/codegraph-watch/src/git_pending.rs index d9fdfdc5..76580fef 100644 --- a/crates/codegraph-watch/src/git_pending.rs +++ b/crates/codegraph-watch/src/git_pending.rs @@ -289,6 +289,7 @@ fn conversion_risk(root: &Path, repo: &Repo) -> Option { } if attributes_file_applies(root, "GIT_ATTR_GLOBAL") || repo.common_dir.join("info").join("attributes").exists() + || ancestor_attributes_exist(repo) { return Some(true); } @@ -308,6 +309,26 @@ fn conversion_risk(root: &Path, repo: &Repo) -> Option { Some(nul_fields(&attributes).next().is_some()) } +/// Whether a `.gitattributes` sits in a directory between the work-tree root +/// and the project root: git applies it to the project's files, but a pathspec +/// rooted at the project cannot list it. The project root itself and +/// everything below are covered by the `ls-files` query. +fn ancestor_attributes_exist(repo: &Repo) -> bool { + let top = Path::new(&repo.toplevel); + let segments = repo + .prefix + .split('/') + .filter(|segment| !segment.is_empty()) + .collect::>(); + (0..segments.len()).any(|depth| { + segments[..depth] + .iter() + .fold(top.to_path_buf(), |dir, segment| dir.join(segment)) + .join(".gitattributes") + .exists() + }) +} + /// Whether the attributes file git names for `var` (`GIT_ATTR_SYSTEM`, /// `GIT_ATTR_GLOBAL`, git 2.42+) exists. A git that cannot say counts as one /// that applies it. diff --git a/crates/codegraph-watch/tests/git_pending_fast_path.rs b/crates/codegraph-watch/tests/git_pending_fast_path.rs index f092473e..f0079b68 100644 --- a/crates/codegraph-watch/tests/git_pending_fast_path.rs +++ b/crates/codegraph-watch/tests/git_pending_fast_path.rs @@ -963,3 +963,28 @@ fn an_overridden_conversion_setting_keeps_the_fast_path() { assert_eq!(source, PendingSource::GitFastPath, "the local false wins"); assert!(pending.is_empty()); } + +/// A `.gitattributes` above a project rooted below its work tree still +/// applies to the project's files. +#[test] +fn an_ancestor_gitattributes_above_a_subdirectory_project_declines() { + let _hooks = hooks_guard(); + if !git_available() { + return; + } + let repo = Repo::in_subdir("ancestor-attributes"); + repo.write("src/a.ts", "export const a = 1;\n"); + repo.commit("init"); + repo.sync(); + assert!(repo.assert_fast("no attributes yet").is_empty()); + for (dir, label) in [ + (repo.top.clone(), "the work-tree root"), + (repo.top.join("packages"), "an intermediate directory"), + ] { + let attributes = dir.join(".gitattributes"); + fs::write(&attributes, "* text=auto\n").unwrap(); + repo.assert_declines(label); + fs::remove_file(&attributes).unwrap(); + } + assert!(repo.assert_fast("attributes removed again").is_empty()); +}