From 7a755cff3c9cbb31d94c11c88287a582ffab5017 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:23:25 -0400 Subject: [PATCH 1/5] test: each walk thread matches a .gitignore through its own copy --- packages/rust/src/gitignore_frame.rs | 33 ++++++++++++++++++++++++++++ packages/rust/src/lib.rs | 1 + 2 files changed, 34 insertions(+) create mode 100644 packages/rust/src/gitignore_frame.rs diff --git a/packages/rust/src/gitignore_frame.rs b/packages/rust/src/gitignore_frame.rs new file mode 100644 index 000000000..e6eb75e8b --- /dev/null +++ b/packages/rust/src/gitignore_frame.rs @@ -0,0 +1,33 @@ +#[cfg(test)] +mod tests { + use super::*; + use ignore::gitignore::GitignoreBuilder; + + fn gitignore(lines: &[&str]) -> Gitignore { + let mut builder = GitignoreBuilder::new("/r"); + for line in lines { + builder.add_line(None, line).unwrap(); + } + builder.build().unwrap() + } + + #[test] + fn a_thread_matches_against_the_copy_rebuilt_for_it() { + let frame = Frame::new(gitignore(&["*.log"]), || Some(gitignore(&["*.md"]))); + assert!(frame.matched(Path::new("/r/a.md"), false).is_ignore()); + assert!(frame.matched(Path::new("/r/a.log"), false).is_none()); + } + + #[test] + fn a_frame_whose_copy_cannot_be_rebuilt_matches_its_own_gitignore() { + let frame = Frame::new(gitignore(&["*.log", "!keep.log"]), || None); + assert!(frame.matched(Path::new("/r/a.log"), false).is_ignore()); + assert!(frame.matched(Path::new("/r/keep.log"), false).is_whitelist()); + } + + #[test] + fn a_frame_reports_the_gitignore_it_was_made_from() { + let frame = Frame::new(gitignore(&["*.log"]), || None); + assert_eq!(frame.gitignore().path(), Path::new("/r")); + } +} diff --git a/packages/rust/src/lib.rs b/packages/rust/src/lib.rs index 860e35ea0..bdd7279f3 100644 --- a/packages/rust/src/lib.rs +++ b/packages/rust/src/lib.rs @@ -20,6 +20,7 @@ pub mod differ; pub mod extension_resolution; #[doc(hidden)] pub mod functions; +mod gitignore_frame; #[doc(hidden)] pub mod infer; #[doc(hidden)] From 64a91dcf4851c02183542106f7230e7a237f482f Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:24:02 -0400 Subject: [PATCH 2/5] perf: match each .gitignore through a per-thread copy Gitignore::matched borrows a match buffer from a pool only its owner thread reaches lock-free; the walk's other threads contended on it. --- .../2026-10-15-gitignore-match-per-thread.md | 3 + .../2026-10-15-gitignore-match-per-thread.md | 25 ++++ packages/rust/src/gitignore_frame.rs | 134 +++++++++++++++--- packages/rust/src/scanner.rs | 62 +++----- 4 files changed, 167 insertions(+), 57 deletions(-) create mode 100644 packages/rust/changelog.d/2026-10-15-gitignore-match-per-thread.md create mode 100644 packages/rust/migrations.d/2026-10-15-gitignore-match-per-thread.md diff --git a/packages/rust/changelog.d/2026-10-15-gitignore-match-per-thread.md b/packages/rust/changelog.d/2026-10-15-gitignore-match-per-thread.md new file mode 100644 index 000000000..a14da8dad --- /dev/null +++ b/packages/rust/changelog.d/2026-10-15-gitignore-match-per-thread.md @@ -0,0 +1,3 @@ +**Changed** + +- **A path-table scan inside a git repo matches `.gitignore` rules without contention between walk threads.** Globs such as `./**` in a large repo answer faster. Which files are ignored is unchanged. diff --git a/packages/rust/migrations.d/2026-10-15-gitignore-match-per-thread.md b/packages/rust/migrations.d/2026-10-15-gitignore-match-per-thread.md new file mode 100644 index 000000000..62cd95df5 --- /dev/null +++ b/packages/rust/migrations.d/2026-10-15-gitignore-match-per-thread.md @@ -0,0 +1,25 @@ +### Faster `.gitignore` matching + +#### Summary + +Each walk thread matches against its own compiled copy of a `.gitignore`. Which files a scan returns, and their order, are unchanged; nothing breaks. + +#### Required changes + +_None._ + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +_None._ + +#### Verification + +```bash +dirsql query "SELECT path FROM './**'" +``` + +Prints the same rows as before, sooner, inside a git repo. diff --git a/packages/rust/src/gitignore_frame.rs b/packages/rust/src/gitignore_frame.rs index e6eb75e8b..ad692dc45 100644 --- a/packages/rust/src/gitignore_frame.rs +++ b/packages/rust/src/gitignore_frame.rs @@ -1,33 +1,135 @@ +use ignore::Match; +use ignore::gitignore::{Gitignore, GitignoreBuilder, Glob}; +use std::path::Path; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{LazyLock, OnceLock}; + +/// One directory's `.gitignore` as the walk matches against it. +/// +/// `Gitignore::matched` borrows a match buffer from a pool that only its +/// first caller's thread reaches without a lock; every other walk thread +/// contends on the rest. So each thread slot matches against a copy of its +/// own, compiled again on first use there. +pub(crate) struct Frame { + gitignore: Gitignore, + copies: Box<[OnceLock>]>, + compile: Box Option + Send + Sync>, +} + +static SLOTS: LazyLock = + LazyLock::new(|| std::thread::available_parallelism().map_or(1, usize::from)); + +static NEXT_SLOT: AtomicUsize = AtomicUsize::new(0); + +thread_local! { + static SLOT: usize = NEXT_SLOT.fetch_add(1, Ordering::Relaxed); +} + +impl Frame { + /// The `.gitignore` in `dir`, if one exists. An unparsable file degrades + /// to no frame rather than failing the scan. + pub(crate) fn load(dir: &Path) -> Option { + let dir = dir.to_path_buf(); + Frame::compiled_by(move || { + let file = dir.join(".gitignore"); + if !file.is_file() { + return None; + } + let mut builder = GitignoreBuilder::new(&dir); + builder.add(&file); + builder.build().ok() + }) + } + + /// A frame for `dir` holding `lines`, as though read from its `.gitignore`. + #[cfg(test)] + pub(crate) fn from_lines(dir: &Path, lines: &[&str]) -> Frame { + let dir = dir.to_path_buf(); + let lines: Vec = lines.iter().map(|line| line.to_string()).collect(); + Frame::compiled_by(move || { + let mut builder = GitignoreBuilder::new(&dir); + for line in &lines { + builder.add_line(None, line).ok()?; + } + builder.build().ok() + }) + .expect("in-memory gitignore lines compile") + } + + fn compiled_by( + compile: impl Fn() -> Option + Send + Sync + 'static, + ) -> Option { + Some(Frame { + gitignore: compile()?, + copies: (0..*SLOTS).map(|_| OnceLock::new()).collect(), + compile: Box::new(compile), + }) + } + + pub(crate) fn gitignore(&self) -> &Gitignore { + &self.gitignore + } + + /// `Gitignore::matched`, on this thread's copy, or the shared one when a + /// copy no longer compiles. + pub(crate) fn matched(&self, path: &Path, is_dir: bool) -> Match<&Glob> { + let slot = SLOT.with(|slot| slot % self.copies.len()); + self.copies[slot] + .get_or_init(&*self.compile) + .as_ref() + .unwrap_or(&self.gitignore) + .matched(path, is_dir) + } +} + #[cfg(test)] mod tests { use super::*; - use ignore::gitignore::GitignoreBuilder; - fn gitignore(lines: &[&str]) -> Gitignore { - let mut builder = GitignoreBuilder::new("/r"); - for line in lines { - builder.add_line(None, line).unwrap(); - } - builder.build().unwrap() + #[test] + fn a_frame_matches_as_its_gitignore_does() { + let frame = Frame::from_lines(Path::new("/r"), &["*.log", "!keep.log", "dist/"]); + assert!(frame.matched(Path::new("/r/a.log"), false).is_ignore()); + assert!( + frame + .matched(Path::new("/r/keep.log"), false) + .is_whitelist() + ); + assert!(frame.matched(Path::new("/r/dist"), true).is_ignore()); + assert!(frame.matched(Path::new("/r/dist"), false).is_none()); } #[test] - fn a_thread_matches_against_the_copy_rebuilt_for_it() { - let frame = Frame::new(gitignore(&["*.log"]), || Some(gitignore(&["*.md"]))); - assert!(frame.matched(Path::new("/r/a.md"), false).is_ignore()); - assert!(frame.matched(Path::new("/r/a.log"), false).is_none()); + fn each_thread_compiles_a_copy_of_its_own_once() { + let compiled = std::sync::Arc::new(AtomicUsize::new(0)); + let count = std::sync::Arc::clone(&compiled); + let frame = Frame::compiled_by(move || { + count.fetch_add(1, Ordering::Relaxed); + Some(Gitignore::empty()) + }) + .unwrap(); + frame.matched(Path::new("/r/a"), false); + frame.matched(Path::new("/r/b"), false); + assert_eq!(compiled.load(Ordering::Relaxed), 2); } #[test] - fn a_frame_whose_copy_cannot_be_rebuilt_matches_its_own_gitignore() { - let frame = Frame::new(gitignore(&["*.log", "!keep.log"]), || None); + fn a_thread_whose_copy_no_longer_compiles_matches_the_first() { + let first = std::sync::atomic::AtomicBool::new(true); + let frame = Frame::compiled_by(move || { + first.swap(false, Ordering::Relaxed).then(|| { + let mut builder = GitignoreBuilder::new("/r"); + builder.add_line(None, "*.log").ok()?; + builder.build().ok() + })? + }) + .unwrap(); assert!(frame.matched(Path::new("/r/a.log"), false).is_ignore()); - assert!(frame.matched(Path::new("/r/keep.log"), false).is_whitelist()); } #[test] - fn a_frame_reports_the_gitignore_it_was_made_from() { - let frame = Frame::new(gitignore(&["*.log"]), || None); + fn a_frame_reports_the_directory_of_its_gitignore() { + let frame = Frame::from_lines(Path::new("/r"), &["*.log"]); assert_eq!(frame.gitignore().path(), Path::new("/r")); } } diff --git a/packages/rust/src/scanner.rs b/packages/rust/src/scanner.rs index 46b0c4bca..5f860ef0c 100644 --- a/packages/rust/src/scanner.rs +++ b/packages/rust/src/scanner.rs @@ -1,8 +1,8 @@ +use crate::gitignore_frame::Frame; use crate::listing::{Listing, Seen}; use crate::matcher::{GlobError, Pattern, TableMatcher}; use crate::tree_walk::{Step, walk_in_order}; use ignore::Match; -use ignore::gitignore::{Gitignore, GitignoreBuilder}; use std::borrow::Cow; use std::ffi::OsStr; use std::fs; @@ -116,9 +116,7 @@ fn scan_below( glob: Some(matcher.walk_glob()), gitignore, in_repo: repo.is_some(), - frames: repo.map_or_else(Vec::new, |top| { - gitignores_above(start, top, &load_gitignore) - }), + frames: repo.map_or_else(Vec::new, |top| gitignores_above(start, top, &Frame::load)), dirs: false, }; walk( @@ -215,7 +213,7 @@ pub(crate) fn scan_glob_rows( glob: Some(glob), gitignore, in_repo: repo.is_some(), - frames: repo.map_or_else(Vec::new, |top| gitignores_above(root, top, &load_gitignore)), + frames: repo.map_or_else(Vec::new, |top| gitignores_above(root, top, &Frame::load)), dirs: false, }; let mut results = Vec::new(); @@ -560,9 +558,9 @@ impl<'a> Place<'a> { walk.frames.clear(); } if walk.in_repo - && let Some(matcher) = load_gitignore(&dir) + && let Some(frame) = Frame::load(&dir) { - walk.frames.push(Arc::new(matcher)); + walk.frames.push(Arc::new(frame)); } let listing = Listing::read(&dir); let taken = judge_all(listing.entries(), &|listed| { @@ -642,7 +640,7 @@ struct Walk<'a> { in_repo: bool, /// The `.gitignore` files in force at the walk's current position, root /// first. - frames: Vec>, + frames: Vec>, /// Whether each directory entered is also visited, as a leaf. dirs: bool, } @@ -843,8 +841,8 @@ fn classify(is_dir: bool, is_file: bool, linked: bool) -> Option { fn gitignores_above( start: &Path, top: &Path, - load: &dyn Fn(&Path) -> Option, -) -> Vec> { + load: &dyn Fn(&Path) -> Option, +) -> Vec> { dirs_above(start, top) .into_iter() .filter_map(load) @@ -871,18 +869,6 @@ fn dirs_above<'a>(start: &'a Path, top: &Path) -> Vec<&'a Path> { dirs } -/// Compile the `.gitignore` in `dir`, if one exists. An unparsable file -/// degrades to no matcher rather than failing the scan. -fn load_gitignore(dir: &Path) -> Option { - let file = dir.join(".gitignore"); - if !file.is_file() { - return None; - } - let mut builder = GitignoreBuilder::new(dir); - builder.add(&file); - builder.build().ok() -} - /// Whether the `.gitignore` files of the repo enclosing `path` ignore it, or /// a directory above it, as the walk would have: for a path a watcher reports /// without having walked to it. @@ -893,8 +879,8 @@ pub(crate) fn is_gitignored_path(path: &Path, is_dir: bool) -> bool { else { return false; }; - for frame in gitignores_above(path, top, &load_gitignore).iter().rev() { - match frame.matched_path_or_any_parents(path, is_dir) { + for frame in gitignores_above(path, top, &Frame::load).iter().rev() { + match frame.gitignore().matched_path_or_any_parents(path, is_dir) { Match::Ignore(_) => return true, Match::Whitelist(_) => return false, Match::None => {} @@ -905,7 +891,7 @@ pub(crate) fn is_gitignored_path(path: &Path, is_dir: bool) -> bool { /// Whether the `.gitignore` files in force mark `path` ignored. Deeper files /// take precedence (git's rule), and a whitelisting `!pattern` un-ignores. -fn is_gitignored(frames: &[Arc], path: &Path, is_dir: bool) -> bool { +fn is_gitignored(frames: &[Arc], path: &Path, is_dir: bool) -> bool { for frame in frames.iter().rev() { match frame.matched(path, is_dir) { Match::Ignore(_) => return true, @@ -1362,12 +1348,8 @@ mod tests { } /// A gitignore frame compiled from in-memory lines; no filesystem. - fn frame(dir: &str, lines: &[&str]) -> Arc { - let mut builder = GitignoreBuilder::new(Path::new(dir)); - for line in lines { - builder.add_line(None, line).unwrap(); - } - Arc::new(builder.build().unwrap()) + fn frame(dir: &str, lines: &[&str]) -> Arc { + Arc::new(Frame::from_lines(Path::new(dir), lines)) } #[test] @@ -1418,20 +1400,18 @@ mod tests { let root = package.ancestors().nth(2).unwrap(); let start = package.join("src"); - assert!(!gitignores_above(&start, root, &load_gitignore).is_empty()); + assert!(!gitignores_above(&start, root, &Frame::load).is_empty()); } #[test] fn gitignores_above_loads_the_ancestors_holding_one_outermost_first() { - let load = |dir: &Path| { - (dir != Path::new("/r/a")).then(|| { - let mut builder = GitignoreBuilder::new(dir); - builder.add_line(None, "*.log").unwrap(); - builder.build().unwrap() - }) - }; + let load = + |dir: &Path| (dir != Path::new("/r/a")).then(|| Frame::from_lines(dir, &["*.log"])); let frames = gitignores_above(Path::new("/r/a/b/c"), Path::new("/r"), &load); - let roots: Vec<&Path> = frames.iter().map(|frame| frame.path()).collect(); + let roots: Vec<&Path> = frames + .iter() + .map(|frame| frame.gitignore().path()) + .collect(); assert_eq!(roots, vec![Path::new("/r"), Path::new("/r/a/b")]); } @@ -1454,7 +1434,7 @@ mod tests { fn walk_with<'a>( ignore: &'a TableMatcher, glob: Option<&'a PathGlob>, - frames: Vec>, + frames: Vec>, ) -> Walk<'a> { Walk { ignore, From bc29b63042a3efed29f46f79cf24fafb2f4ecef7 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:37:59 -0400 Subject: [PATCH 3/5] e2e attestation for bot/1481-gitrepo-speed-1015 --- .../e2e-attestations/bot-1481-gitrepo-speed-1015.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/python/e2e-attestations/bot-1481-gitrepo-speed-1015.json diff --git a/packages/python/e2e-attestations/bot-1481-gitrepo-speed-1015.json b/packages/python/e2e-attestations/bot-1481-gitrepo-speed-1015.json new file mode 100644 index 000000000..28b775826 --- /dev/null +++ b/packages/python/e2e-attestations/bot-1481-gitrepo-speed-1015.json @@ -0,0 +1,7 @@ +{ + "command": "uv run python -m pytest tests/e2e/glob_speed_gitignore_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/query_contract_test.py -x -q -s", + "ran_at": 1792078679, + "exit_code": 0, + "commit": "64a91dcf4851c02183542106f7230e7a237f482f", + "branch": "bot/1481-gitrepo-speed-1015" +} From 496817e1aeec8173385410d3b6fc35775cf35efc Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:39:22 -0400 Subject: [PATCH 4/5] e2e attestation for bot/1481-gitrepo-speed-1015 --- .../ts/e2e-attestations/bot-1481-gitrepo-speed-1015.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/ts/e2e-attestations/bot-1481-gitrepo-speed-1015.json diff --git a/packages/ts/e2e-attestations/bot-1481-gitrepo-speed-1015.json b/packages/ts/e2e-attestations/bot-1481-gitrepo-speed-1015.json new file mode 100644 index 000000000..a4bf9dba3 --- /dev/null +++ b/packages/ts/e2e-attestations/bot-1481-gitrepo-speed-1015.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1792078762, + "exit_code": 0, + "commit": "bc29b63042a3efed29f46f79cf24fafb2f4ecef7", + "branch": "bot/1481-gitrepo-speed-1015" +} From 64693ff75646ece363b43a7eeef122d4c5233bd2 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:50:53 -0400 Subject: [PATCH 5/5] test: a frame serves more threads than it has copies --- packages/rust/src/gitignore_frame.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/rust/src/gitignore_frame.rs b/packages/rust/src/gitignore_frame.rs index ad692dc45..784cc69c9 100644 --- a/packages/rust/src/gitignore_frame.rs +++ b/packages/rust/src/gitignore_frame.rs @@ -70,15 +70,18 @@ impl Frame { &self.gitignore } - /// `Gitignore::matched`, on this thread's copy, or the shared one when a - /// copy no longer compiles. + /// `Gitignore::matched`, on this thread's copy. pub(crate) fn matched(&self, path: &Path, is_dir: bool) -> Match<&Glob> { - let slot = SLOT.with(|slot| slot % self.copies.len()); - self.copies[slot] + SLOT.with(|slot| self.copy(*slot)).matched(path, is_dir) + } + + /// The copy for thread `slot`, or the shared one when a copy no longer + /// compiles. + fn copy(&self, slot: usize) -> &Gitignore { + self.copies[slot % self.copies.len()] .get_or_init(&*self.compile) .as_ref() .unwrap_or(&self.gitignore) - .matched(path, is_dir) } } @@ -127,6 +130,13 @@ mod tests { assert!(frame.matched(Path::new("/r/a.log"), false).is_ignore()); } + #[test] + fn a_frame_serves_more_threads_than_it_has_copies() { + let frame = Frame::from_lines(Path::new("/r"), &["*.log"]); + let slot = frame.copies.len() * frame.copies.len(); + assert!(frame.copy(slot).matched("/r/a.log", false).is_ignore()); + } + #[test] fn a_frame_reports_the_directory_of_its_gitignore() { let frame = Frame::from_lines(Path::new("/r"), &["*.log"]);