perf: match .gitignore without contention between walk threads

packages/python/e2e-attestations/bot-1481-gitrepo-speed-1015.json

@@ -0,0 +1,7 @@
+{
+  "command": "uv run python -m pytest tests/e2e/glob_speed_gitignore_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/query_contract_test.py -x -q -s",
+  "ran_at": 1792078679,
+  "exit_code": 0,
+  "commit": "64a91dcf4851c02183542106f7230e7a237f482f",
+  "branch": "bot/1481-gitrepo-speed-1015"
+}

packages/rust/changelog.d/2026-10-15-gitignore-match-per-thread.md

@@ -0,0 +1,3 @@
+**Changed**
+
+- **A path-table scan inside a git repo matches `.gitignore` rules without contention between walk threads.** Globs such as `./**` in a large repo answer faster. Which files are ignored is unchanged.

packages/rust/migrations.d/2026-10-15-gitignore-match-per-thread.md

@@ -0,0 +1,25 @@
+### Faster `.gitignore` matching
+
+#### Summary
+
+Each walk thread matches against its own compiled copy of a `.gitignore`. Which files a scan returns, and their order, are unchanged; nothing breaks.
+
+#### Required changes
+
+_None._
+
+#### Deprecations removed
+
+_None._
+
+#### Behavior changes without code changes
+
+_None._
+
+#### Verification
+
+```bash
+dirsql query "SELECT path FROM './**'"
+```
+
+Prints the same rows as before, sooner, inside a git repo.

packages/rust/src/gitignore_frame.rs

@@ -0,0 +1,145 @@
+use ignore::Match;
+use ignore::gitignore::{Gitignore, GitignoreBuilder, Glob};
+use std::path::Path;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{LazyLock, OnceLock};
+
+/// One directory's `.gitignore` as the walk matches against it.
+///
+/// `Gitignore::matched` borrows a match buffer from a pool that only its
+/// first caller's thread reaches without a lock; every other walk thread
+/// contends on the rest. So each thread slot matches against a copy of its
+/// own, compiled again on first use there.
+pub(crate) struct Frame {
+    gitignore: Gitignore,
+    copies: Box<[OnceLock<Option<Gitignore>>]>,
+    compile: Box<dyn Fn() -> Option<Gitignore> + Send + Sync>,
+}
+
+static SLOTS: LazyLock<usize> =
+    LazyLock::new(|| std::thread::available_parallelism().map_or(1, usize::from));
+
+static NEXT_SLOT: AtomicUsize = AtomicUsize::new(0);
+
+thread_local! {
+    static SLOT: usize = NEXT_SLOT.fetch_add(1, Ordering::Relaxed);
+}
+
+impl Frame {
+    /// The `.gitignore` in `dir`, if one exists. An unparsable file degrades
+    /// to no frame rather than failing the scan.
+    pub(crate) fn load(dir: &Path) -> Option<Frame> {
+        let dir = dir.to_path_buf();
+        Frame::compiled_by(move || {
+            let file = dir.join(".gitignore");
+            if !file.is_file() {
+                return None;
+            }
+            let mut builder = GitignoreBuilder::new(&dir);
+            builder.add(&file);
+            builder.build().ok()
+        })
+    }
+
+    /// A frame for `dir` holding `lines`, as though read from its `.gitignore`.
+    #[cfg(test)]
+    pub(crate) fn from_lines(dir: &Path, lines: &[&str]) -> Frame {
+        let dir = dir.to_path_buf();
+        let lines: Vec<String> = lines.iter().map(|line| line.to_string()).collect();
+        Frame::compiled_by(move || {
+            let mut builder = GitignoreBuilder::new(&dir);
+            for line in &lines {
+                builder.add_line(None, line).ok()?;
+            }
+            builder.build().ok()
+        })
+        .expect("in-memory gitignore lines compile")
+    }
+
+    fn compiled_by(
+        compile: impl Fn() -> Option<Gitignore> + Send + Sync + 'static,
+    ) -> Option<Frame> {
+        Some(Frame {
+            gitignore: compile()?,
+            copies: (0..*SLOTS).map(|_| OnceLock::new()).collect(),
+            compile: Box::new(compile),
+        })
+    }
+
+    pub(crate) fn gitignore(&self) -> &Gitignore {
+        &self.gitignore
+    }
+
+    /// `Gitignore::matched`, on this thread's copy.
+    pub(crate) fn matched(&self, path: &Path, is_dir: bool) -> Match<&Glob> {
+        SLOT.with(|slot| self.copy(*slot)).matched(path, is_dir)
+    }
+
+    /// The copy for thread `slot`, or the shared one when a copy no longer
+    /// compiles.
+    fn copy(&self, slot: usize) -> &Gitignore {
+        self.copies[slot % self.copies.len()]
+            .get_or_init(&*self.compile)
+            .as_ref()
+            .unwrap_or(&self.gitignore)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_frame_matches_as_its_gitignore_does() {
+        let frame = Frame::from_lines(Path::new("/r"), &["*.log", "!keep.log", "dist/"]);
+        assert!(frame.matched(Path::new("/r/a.log"), false).is_ignore());
+        assert!(
+            frame
+                .matched(Path::new("/r/keep.log"), false)
+                .is_whitelist()
+        );
+        assert!(frame.matched(Path::new("/r/dist"), true).is_ignore());
+        assert!(frame.matched(Path::new("/r/dist"), false).is_none());
+    }
+
+    #[test]
+    fn each_thread_compiles_a_copy_of_its_own_once() {
+        let compiled = std::sync::Arc::new(AtomicUsize::new(0));
+        let count = std::sync::Arc::clone(&compiled);
+        let frame = Frame::compiled_by(move || {
+            count.fetch_add(1, Ordering::Relaxed);
+            Some(Gitignore::empty())
+        })
+        .unwrap();
+        frame.matched(Path::new("/r/a"), false);
+        frame.matched(Path::new("/r/b"), false);
+        assert_eq!(compiled.load(Ordering::Relaxed), 2);
+    }
+
+    #[test]
+    fn a_thread_whose_copy_no_longer_compiles_matches_the_first() {
+        let first = std::sync::atomic::AtomicBool::new(true);
+        let frame = Frame::compiled_by(move || {
+            first.swap(false, Ordering::Relaxed).then(|| {
+                let mut builder = GitignoreBuilder::new("/r");
+                builder.add_line(None, "*.log").ok()?;
+                builder.build().ok()
+            })?
+        })
+        .unwrap();
+        assert!(frame.matched(Path::new("/r/a.log"), false).is_ignore());
+    }
+
+    #[test]
+    fn a_frame_serves_more_threads_than_it_has_copies() {
+        let frame = Frame::from_lines(Path::new("/r"), &["*.log"]);
+        let slot = frame.copies.len() * frame.copies.len();
+        assert!(frame.copy(slot).matched("/r/a.log", false).is_ignore());
+    }
+
+    #[test]
+    fn a_frame_reports_the_directory_of_its_gitignore() {
+        let frame = Frame::from_lines(Path::new("/r"), &["*.log"]);
+        assert_eq!(frame.gitignore().path(), Path::new("/r"));
+    }
+}

packages/rust/src/lib.rs

@@ -20,6 +20,7 @@ pub mod differ;
 pub mod extension_resolution;
 #[doc(hidden)]
 pub mod functions;
+mod gitignore_frame;
 #[doc(hidden)]
 pub mod infer;
 #[doc(hidden)]

packages/rust/src/scanner.rs

@@ -1,8 +1,8 @@
+use crate::gitignore_frame::Frame;
 use crate::listing::{Listing, Seen};
 use crate::matcher::{GlobError, Pattern, TableMatcher};
 use crate::tree_walk::{Step, walk_in_order};
 use ignore::Match;
-use ignore::gitignore::{Gitignore, GitignoreBuilder};
 use std::borrow::Cow;
 use std::ffi::OsStr;
 use std::fs;
@@ -116,9 +116,7 @@ fn scan_below(
         glob: Some(matcher.walk_glob()),
         gitignore,
         in_repo: repo.is_some(),
-        frames: repo.map_or_else(Vec::new, |top| {
-            gitignores_above(start, top, &load_gitignore)
-        }),
+        frames: repo.map_or_else(Vec::new, |top| gitignores_above(start, top, &Frame::load)),
         dirs: false,
     };
     walk(
@@ -215,7 +213,7 @@ pub(crate) fn scan_glob_rows<T: Send>(
         glob: Some(glob),
         gitignore,
         in_repo: repo.is_some(),
-        frames: repo.map_or_else(Vec::new, |top| gitignores_above(root, top, &load_gitignore)),
+        frames: repo.map_or_else(Vec::new, |top| gitignores_above(root, top, &Frame::load)),
         dirs: false,
     };
     let mut results = Vec::new();
@@ -560,9 +558,9 @@ impl<'a> Place<'a> {
             walk.frames.clear();
         }
         if walk.in_repo
-            && let Some(matcher) = load_gitignore(&dir)
+            && let Some(frame) = Frame::load(&dir)
         {
-            walk.frames.push(Arc::new(matcher));
+            walk.frames.push(Arc::new(frame));
         }
         let listing = Listing::read(&dir);
         let taken = judge_all(listing.entries(), &|listed| {
@@ -642,7 +640,7 @@ struct Walk<'a> {
     in_repo: bool,
     /// The `.gitignore` files in force at the walk's current position, root
     /// first.
-    frames: Vec<Arc<Gitignore>>,
+    frames: Vec<Arc<Frame>>,
     /// Whether each directory entered is also visited, as a leaf.
     dirs: bool,
 }
@@ -843,8 +841,8 @@ fn classify(is_dir: bool, is_file: bool, linked: bool) -> Option<Kind> {
 fn gitignores_above(
     start: &Path,
     top: &Path,
-    load: &dyn Fn(&Path) -> Option<Gitignore>,
-) -> Vec<Arc<Gitignore>> {
+    load: &dyn Fn(&Path) -> Option<Frame>,
+) -> Vec<Arc<Frame>> {
     dirs_above(start, top)
         .into_iter()
         .filter_map(load)
@@ -871,18 +869,6 @@ fn dirs_above<'a>(start: &'a Path, top: &Path) -> Vec<&'a Path> {
     dirs
 }
 
-/// Compile the `.gitignore` in `dir`, if one exists. An unparsable file
-/// degrades to no matcher rather than failing the scan.
-fn load_gitignore(dir: &Path) -> Option<Gitignore> {
-    let file = dir.join(".gitignore");
-    if !file.is_file() {
-        return None;
-    }
-    let mut builder = GitignoreBuilder::new(dir);
-    builder.add(&file);
-    builder.build().ok()
-}
-
 /// Whether the `.gitignore` files of the repo enclosing `path` ignore it, or
 /// a directory above it, as the walk would have: for a path a watcher reports
 /// without having walked to it.
@@ -893,8 +879,8 @@ pub(crate) fn is_gitignored_path(path: &Path, is_dir: bool) -> bool {
     else {
         return false;
     };
-    for frame in gitignores_above(path, top, &load_gitignore).iter().rev() {
-        match frame.matched_path_or_any_parents(path, is_dir) {
+    for frame in gitignores_above(path, top, &Frame::load).iter().rev() {
+        match frame.gitignore().matched_path_or_any_parents(path, is_dir) {
             Match::Ignore(_) => return true,
             Match::Whitelist(_) => return false,
             Match::None => {}
@@ -905,7 +891,7 @@ pub(crate) fn is_gitignored_path(path: &Path, is_dir: bool) -> bool {
 
 /// Whether the `.gitignore` files in force mark `path` ignored. Deeper files
 /// take precedence (git's rule), and a whitelisting `!pattern` un-ignores.
-fn is_gitignored(frames: &[Arc<Gitignore>], path: &Path, is_dir: bool) -> bool {
+fn is_gitignored(frames: &[Arc<Frame>], path: &Path, is_dir: bool) -> bool {
     for frame in frames.iter().rev() {
         match frame.matched(path, is_dir) {
             Match::Ignore(_) => return true,
@@ -1362,12 +1348,8 @@ mod tests {
     }
 
     /// A gitignore frame compiled from in-memory lines; no filesystem.
-    fn frame(dir: &str, lines: &[&str]) -> Arc<Gitignore> {
-        let mut builder = GitignoreBuilder::new(Path::new(dir));
-        for line in lines {
-            builder.add_line(None, line).unwrap();
-        }
-        Arc::new(builder.build().unwrap())
+    fn frame(dir: &str, lines: &[&str]) -> Arc<Frame> {
+        Arc::new(Frame::from_lines(Path::new(dir), lines))
     }
 
     #[test]
@@ -1418,20 +1400,18 @@ mod tests {
         let root = package.ancestors().nth(2).unwrap();
         let start = package.join("src");
 
-        assert!(!gitignores_above(&start, root, &load_gitignore).is_empty());
+        assert!(!gitignores_above(&start, root, &Frame::load).is_empty());
     }
 
     #[test]
     fn gitignores_above_loads_the_ancestors_holding_one_outermost_first() {
-        let load = |dir: &Path| {
-            (dir != Path::new("/r/a")).then(|| {
-                let mut builder = GitignoreBuilder::new(dir);
-                builder.add_line(None, "*.log").unwrap();
-                builder.build().unwrap()
-            })
-        };
+        let load =
+            |dir: &Path| (dir != Path::new("/r/a")).then(|| Frame::from_lines(dir, &["*.log"]));
         let frames = gitignores_above(Path::new("/r/a/b/c"), Path::new("/r"), &load);
-        let roots: Vec<&Path> = frames.iter().map(|frame| frame.path()).collect();
+        let roots: Vec<&Path> = frames
+            .iter()
+            .map(|frame| frame.gitignore().path())
+            .collect();
         assert_eq!(roots, vec![Path::new("/r"), Path::new("/r/a/b")]);
     }
 
@@ -1454,7 +1434,7 @@ mod tests {
     fn walk_with<'a>(
         ignore: &'a TableMatcher,
         glob: Option<&'a PathGlob>,
-        frames: Vec<Arc<Gitignore>>,
+        frames: Vec<Arc<Frame>>,
     ) -> Walk<'a> {
         Walk {
             ignore,

packages/ts/e2e-attestations/bot-1481-gitrepo-speed-1015.json

@@ -0,0 +1,7 @@
+{
+  "command": "pnpm test:e2e",
+  "ran_at": 1792078762,
+  "exit_code": 0,
+  "commit": "bc29b63042a3efed29f46f79cf24fafb2f4ecef7",
+  "branch": "bot/1481-gitrepo-speed-1015"
+}