From f6fdddfc1d2152ac7208fb0defcc9d6806afc062 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:06:38 -0400 Subject: [PATCH 01/10] perf: drop unmatchable file names before building paths; order the walk's reads The walk rejects a regular file whose name no glob word's last component matches before building its path, checks for .git and .gitignore in the listing it already read, opens directories without duplicating the fd, and has workers claim the open directory the walk reaches first, each directory behind its own lock. --- .../2026-10-15-matcher-glob-walk-speed.md | 3 + .../2026-10-15-matcher-glob-walk-speed.md | 25 ++ packages/rust/src/listing.rs | 22 +- packages/rust/src/scanner.rs | 104 ++++++-- packages/rust/src/tree_walk.rs | 240 ++++++++++++------ 5 files changed, 303 insertions(+), 91 deletions(-) create mode 100644 packages/rust/changelog.d/2026-10-15-matcher-glob-walk-speed.md create mode 100644 packages/rust/migrations.d/2026-10-15-matcher-glob-walk-speed.md diff --git a/packages/rust/changelog.d/2026-10-15-matcher-glob-walk-speed.md b/packages/rust/changelog.d/2026-10-15-matcher-glob-walk-speed.md new file mode 100644 index 000000000..a0546aa0f --- /dev/null +++ b/packages/rust/changelog.d/2026-10-15-matcher-glob-walk-speed.md @@ -0,0 +1,3 @@ +**Changed** + +- **Recursive globs with a narrow file name list faster.** For globs such as `./**/file?.md` and `./**/[[:digit:]]*.md`, the walk drops a file whose name cannot match before building its path, skips the `.git` and `.gitignore` lookups in directories whose listing holds neither, and has its threads read the directories the output reaches next. Output and its order are unchanged. diff --git a/packages/rust/migrations.d/2026-10-15-matcher-glob-walk-speed.md b/packages/rust/migrations.d/2026-10-15-matcher-glob-walk-speed.md new file mode 100644 index 000000000..c70250f06 --- /dev/null +++ b/packages/rust/migrations.d/2026-10-15-matcher-glob-walk-speed.md @@ -0,0 +1,25 @@ +### Faster recursive globs with a narrow file name + +#### Summary + +The tree walk rejects a file by its name before building its path, checks for `.git` and `.gitignore` in the directory listing it already read, and reads directories ahead of the output in the order the output reaches them. Query results and their order are unchanged; nothing breaks. + +#### Required changes + +_None._ + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +_None._ + +#### Verification + +```bash +dirsql query "SELECT count(*) FROM './**/file?.md'" +``` + +Prints the same count as before, sooner. diff --git a/packages/rust/src/listing.rs b/packages/rust/src/listing.rs index a2d6647e5..520da1fdd 100644 --- a/packages/rust/src/listing.rs +++ b/packages/rust/src/listing.rs @@ -100,6 +100,13 @@ impl Listing { }); } + /// Whether an entry is named `name`; the listing must be sorted. + pub(crate) fn holds(&self, name: &OsStr) -> bool { + self.entries + .binary_search_by(|listed| self.name(listed).cmp(name)) + .is_ok() + } + pub(crate) fn name(&self, listed: &Listed) -> &OsStr { self.names.get(listed.start, listed.end) } @@ -139,7 +146,7 @@ fn read_into(dir: &Path, listing: &mut Listing) { let Ok(fd) = rustix::fs::open(dir, flags, Mode::empty()) else { return; }; - let Ok(mut entries) = Dir::read_from(&fd) else { + let Ok(mut entries) = Dir::new(fd) else { return; }; while let Some(Ok(entry)) = entries.read() { @@ -231,6 +238,19 @@ mod tests { assert_eq!(names(&listing), ["first", "second"]); } + #[test] + fn a_sorted_listing_holds_exactly_the_names_pushed() { + let mut listing = Listing::default(); + for name in [".gitignore", "b", ".git", "a"] { + listing.push(OsStr::new(name), Seen::File); + } + listing.sort(); + assert!(listing.holds(OsStr::new(".git"))); + assert!(listing.holds(OsStr::new("a"))); + assert!(!listing.holds(OsStr::new(".gi"))); + assert!(!Listing::default().holds(OsStr::new(".git"))); + } + #[test] fn name_prefix_orders_names_as_their_bytes_do() { let mut names = ["abcdefghZ", "abcdefgh", "abcdefghA", "ab", "b", "abc", ""]; diff --git a/packages/rust/src/scanner.rs b/packages/rust/src/scanner.rs index 46b0c4bca..75e4719e7 100644 --- a/packages/rust/src/scanner.rs +++ b/packages/rust/src/scanner.rs @@ -86,6 +86,7 @@ pub fn scan_dirs(root: &Path, start: &Path, ignore: &TableMatcher) -> Vec( root, root, walker, + &|name| glob.may_match_name(name), &|rel_path| is_wanted(glob, ignore, rel_path), &|rel_path, _| row(rel_path), &mut |found| results.push(found), @@ -261,6 +264,8 @@ pub struct PathGlob { components: Vec, /// Where each word's components begin. starts: Vec, + /// Where each word's closing `End` sits. + ends: Vec, /// Whether any word spells a `.` or `..` component, which the walk has to /// enter itself: a directory listing never names them. spells_relative: bool, @@ -319,6 +324,18 @@ impl PathGlob { .all(|name| self.spells_dot_name(name)) } + /// Whether a file named `name` may match: false when no word could match + /// it, since only `**` crosses a `/` and so a word's last component, unless + /// it is `**`, has to match the name alone. + pub(crate) fn may_match_name(&self, name: &OsStr) -> bool { + self.ends + .iter() + .any(|&end| match &self.components[end - 1] { + Component::Name(p) | Component::DotName(p) => p.is_match(Path::new(name)), + Component::AnyDepth | Component::End => true, + }) + } + fn has_components(&self) -> bool { !self.components.is_empty() } @@ -422,10 +439,12 @@ pub fn compile_globs(patterns: &[&str]) -> Result { .collect(); let mut components = Vec::new(); let mut starts = Vec::new(); + let mut ends = Vec::new(); for word in &words { let word_components = compile_components(word)?; starts.push(components.len()); components.extend(word_components); + ends.push(components.len()); components.push(Component::End); } Ok(PathGlob { @@ -433,6 +452,7 @@ pub fn compile_globs(patterns: &[&str]) -> Result { spelled_dot_names, components, starts, + ends, spells_relative, named, }) @@ -504,6 +524,7 @@ fn walk( root: &Path, start: &Path, walker: Walk<'_>, + keep_name: &(dyn Fn(&OsStr) -> bool + Sync), keep: &(dyn Fn(&Path) -> bool + Sync), leaf: &(dyn Fn(PathBuf, &Path) -> L + Sync), visit: &mut dyn FnMut(L), @@ -523,7 +544,7 @@ fn walk( }; walk_in_order( Box::new(place), - &|place: Box>| (*place).explore(keep, leaf), + &|place: Box>| (*place).explore(keep_name, keep, leaf), visit, ); } @@ -542,9 +563,11 @@ struct Place<'a> { impl<'a> Place<'a> { /// The directory's entries in walk order: each file `keep` takes, made /// into a `leaf` with the directory it sits in, and each directory to - /// enter. + /// enter. A regular file whose name `keep_name` rejects is dropped before + /// its path is built. fn explore( self, + keep_name: &(dyn Fn(&OsStr) -> bool + Sync), keep: &(dyn Fn(&Path) -> bool + Sync), leaf: &(dyn Fn(PathBuf, &Path) -> L + Sync), ) -> Vec>, L>> { @@ -555,29 +578,27 @@ impl<'a> Place<'a> { states, linked, } = self; - if walk.gitignore && holds_git(&dir) { + let listing = Listing::read(&dir); + if walk.gitignore && listing.holds(OsStr::new(".git")) && holds_git(&dir) { walk.in_repo = true; walk.frames.clear(); } if walk.in_repo + && listing.holds(OsStr::new(".gitignore")) && let Some(matcher) = load_gitignore(&dir) { walk.frames.push(Arc::new(matcher)); } - let listing = Listing::read(&dir); let taken = judge_all(listing.entries(), &|listed| { - walk.take( - &dir, - listing.name(listed), - listed.seen, - &rel, - &states, - linked, - ) - .filter(|taken| match taken { - Taken::File(child) => keep(child), - Taken::Dir { .. } => true, - }) + let name = listing.name(listed); + if listed.seen == Seen::File && !keep_name(name) { + return None; + } + walk.take(&dir, name, listed.seen, &rel, &states, linked) + .filter(|taken| match taken { + Taken::File(child) => keep(child), + Taken::Dir { .. } => true, + }) }); let mut steps: Vec>, L>> = if walk.dirs { taken @@ -1148,7 +1169,7 @@ mod tests { fn explored_dirs(place: Place<'_>) -> Vec> { place - .explore(&|_| true, &|_, _| ()) + .explore(&|_| true, &|_| true, &|_, _| ()) .into_iter() .filter_map(|step| match step { Step::Dir(child) => Some(*child), @@ -1182,6 +1203,40 @@ mod tests { ); } + #[test] + fn explore_drops_a_file_whose_name_is_rejected_before_judging_its_path() { + let root = tempfile::tempdir().unwrap(); + fs::write(root.path().join("a.md"), "").unwrap(); + fs::write(root.path().join("b.md"), "").unwrap(); + fs::create_dir(root.path().join("d")).unwrap(); + let ignore = TableMatcher::new(&[], &[]).unwrap(); + let top = Place { + walk: walk_with(&ignore, None, Vec::new()), + dir: root.path().to_path_buf(), + rel: PathBuf::new(), + states: Vec::new(), + linked: false, + }; + let judged = std::sync::Mutex::new(Vec::new()); + let steps = top.explore( + &|name| name != OsStr::new("b.md"), + &|rel| { + judged.lock().unwrap().push(rel.to_path_buf()); + true + }, + &|rel, _| rel, + ); + let leaves: Vec = steps + .into_iter() + .filter_map(|step| match step { + Step::Leaf(rel) => Some(rel), + Step::Dir(_) => None, + }) + .collect(); + assert_eq!(leaves, vec![PathBuf::from("a.md")]); + assert_eq!(judged.into_inner().unwrap(), vec![PathBuf::from("a.md")]); + } + #[test] fn explore_enters_a_dirsql_directory_below_the_top_level() { let root = tempfile::tempdir().unwrap(); @@ -1605,6 +1660,21 @@ mod tests { }) } + #[test] + fn a_name_may_match_only_when_some_word_ends_in_a_component_matching_it() { + let glob = compile_glob("./**/{a,b}*.md").unwrap(); + assert!(glob.may_match_name(OsStr::new("a1.md"))); + assert!(glob.may_match_name(OsStr::new("b1.md"))); + assert!(!glob.may_match_name(OsStr::new("c1.md"))); + assert!(!glob.may_match_name(OsStr::new("a1.txt"))); + } + + #[test] + fn a_word_ending_in_a_double_star_may_match_any_name() { + let glob = compile_glob("{docs/**,*.md}").unwrap(); + assert!(glob.may_match_name(OsStr::new("a.txt"))); + } + #[test] fn a_double_star_starts_matched_or_not_yet_consumed() { assert_eq!(compile_glob("**/*.md").unwrap().start(), vec![0, 1]); diff --git a/packages/rust/src/tree_walk.rs b/packages/rust/src/tree_walk.rs index 18c52bce9..b61bdcba5 100644 --- a/packages/rust/src/tree_walk.rs +++ b/packages/rust/src/tree_walk.rs @@ -4,7 +4,10 @@ //! of every other directory, so workers explore ahead of the walk while the //! calling thread hands out what they found in depth-first order. -use std::sync::{Condvar, Mutex}; +use std::cmp::Ordering as Order; +use std::collections::BinaryHeap; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; use std::thread; /// What exploring a directory yields: a leaf to hand out, or a directory to @@ -18,24 +21,69 @@ pub(crate) enum Step { /// cores; below it, spawning workers costs more than the exploring. const PARALLEL_DIRS: usize = 8; -enum Item { - Dir(usize), +enum Item { + Dir(Arc>), Leaf(L), } -struct Queue { - /// Each directory found, until a thread claims it. - unclaimed: Vec>, - /// Each explored directory's items, until the walk reaches it. - explored: Vec>>>, - /// Unclaimed directories, the next one the walk will want on top. - open: Vec, +/// One directory found: unexplored until a thread takes it, then its items +/// once that thread is through. Each directory has its own locks, so the walk +/// waiting on one never contends with workers exploring others. +struct Slot { + /// The directory's place in the walk: each step's index among its + /// parent's directories, root first, so keys order as the walk reaches + /// them. + key: Vec, + dir: Mutex>, + explored: Mutex>, + ready: Condvar, +} + +struct Explored { + items: Option>>, + /// Whether the walk is waiting on these items, so a worker signals only + /// then. + awaited: bool, +} + +/// Directories found and not yet claimed. Workers take the one the walk will +/// reach first, so they explore just ahead of it rather than deep in a subtree +/// it is far from, which would leave it to explore its next directory itself. +struct Open { + slots: BinaryHeap>, + /// Workers waiting for a directory, so a find wakes only as many as it + /// has work for. + idle: usize, done: bool, } +/// A found directory, ordered so the heap's greatest is the earliest key. +struct Pending(Arc>); + +impl PartialEq for Pending { + fn eq(&self, other: &Self) -> bool { + self.0.key == other.0.key + } +} + +impl Eq for Pending {} + +impl PartialOrd for Pending { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Pending { + fn cmp(&self, other: &Self) -> Order { + other.0.key.cmp(&self.0.key) + } +} + struct Shared<'e, D, L> { - queue: Mutex>, - changed: Condvar, + open: Mutex>, + found_dir: Condvar, + found: AtomicUsize, explore: &'e (dyn Fn(D) -> Vec> + Sync), } @@ -47,118 +95,140 @@ pub(crate) fn walk_in_order( visit: &mut dyn FnMut(L), ) { let shared = Shared { - queue: Mutex::new(Queue { - unclaimed: vec![Some(root)], - explored: vec![None], - open: Vec::new(), + open: Mutex::new(Open { + slots: BinaryHeap::new(), + idle: 0, done: false, }), - changed: Condvar::new(), + found_dir: Condvar::new(), + found: AtomicUsize::new(1), explore, }; thread::scope(|scope| { let mut spawned = false; - let mut stack = vec![shared.obtain(0).into_iter()]; + let mut stack = vec![shared.explore(root, &[]).into_iter()]; while let Some(items) = stack.last_mut() { match items.next() { Some(Item::Leaf(leaf)) => visit(leaf), - Some(Item::Dir(id)) => { - if !spawned && shared.found() >= PARALLEL_DIRS { + Some(Item::Dir(slot)) => { + if !spawned && shared.found.load(Ordering::Relaxed) >= PARALLEL_DIRS { spawned = true; let workers = thread::available_parallelism().map_or(1, usize::from); for _ in 1..workers { scope.spawn(|| shared.work()); } } - stack.push(shared.obtain(id).into_iter()); + stack.push(shared.obtain(&slot).into_iter()); } None => { stack.pop(); } } } - shared.lock().done = true; - shared.changed.notify_all(); + shared.open_lock().done = true; + shared.found_dir.notify_all(); }); } -impl Shared<'_, D, L> { - fn lock(&self) -> std::sync::MutexGuard<'_, Queue> { - self.queue - .lock() - .expect("exploring a directory does not panic") - } +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().expect("exploring a directory does not panic") +} - fn found(&self) -> usize { - self.lock().unclaimed.len() +impl Shared<'_, D, L> { + fn open_lock(&self) -> MutexGuard<'_, Open> { + lock(&self.open) } - /// Directory `id`'s items: explored here if no worker has claimed it, + /// Directory `slot`'s items: explored here if no worker has claimed it, /// else once the worker that has is through. - fn obtain(&self, id: usize) -> Vec> { - let mut queue = self.lock(); + fn obtain(&self, slot: &Slot) -> Vec> { + let unclaimed = lock(&slot.dir).take(); + if let Some(dir) = unclaimed { + return self.explore(dir, &slot.key); + } + let mut explored = lock(&slot.explored); loop { - if let Some(items) = queue.explored[id].take() { + if let Some(items) = explored.items.take() { return items; } - if let Some(dir) = queue.unclaimed[id].take() { - drop(queue); - return self.explore(dir); - } - queue = self - .changed - .wait(queue) + explored.awaited = true; + explored = slot + .ready + .wait(explored) .expect("exploring a directory does not panic"); } } fn work(&self) { - let mut queue = self.lock(); + while let Some(slot) = self.claim() { + let unclaimed = lock(&slot.dir).take(); + let Some(dir) = unclaimed else { + continue; + }; + let items = self.explore(dir, &slot.key); + let mut explored = lock(&slot.explored); + explored.items = Some(items); + if explored.awaited { + slot.ready.notify_one(); + } + } + } + + /// The next directory found, or `None` once the walk is over. + fn claim(&self) -> Option>> { + let mut open = self.open_lock(); loop { - if queue.done { - return; + if open.done { + return None; } - let claimed = queue.open.pop().map(|id| (id, queue.unclaimed[id].take())); - match claimed { - Some((id, Some(dir))) => { - drop(queue); - let items = self.explore(dir); - queue = self.lock(); - queue.explored[id] = Some(items); - self.changed.notify_all(); - } - Some((_, None)) => {} - None => { - queue = self - .changed - .wait(queue) - .expect("exploring a directory does not panic"); - } + if let Some(Pending(slot)) = open.slots.pop() { + return Some(slot); } + open.idle += 1; + open = self + .found_dir + .wait(open) + .expect("exploring a directory does not panic"); + open.idle -= 1; } } - /// Explore `dir`, registering the directories it holds for any thread - /// to claim. - fn explore(&self, dir: D) -> Vec> { + /// Explore `dir`, at `key`, registering the directories it holds for any + /// thread to claim. + fn explore(&self, dir: D, key: &[u32]) -> Vec> { let steps = (self.explore)(dir); - let mut queue = self.lock(); let mut dirs = Vec::new(); - let items = steps + let items: Vec> = steps .into_iter() .map(|step| match step { Step::Leaf(leaf) => Item::Leaf(leaf), Step::Dir(dir) => { - let id = queue.unclaimed.len(); - queue.unclaimed.push(Some(dir)); - queue.explored.push(None); - dirs.push(id); - Item::Dir(id) + let index = u32::try_from(dirs.len()).unwrap_or(u32::MAX); + let slot = Arc::new(Slot { + key: [key, &[index]].concat(), + dir: Mutex::new(Some(dir)), + explored: Mutex::new(Explored { + items: None, + awaited: false, + }), + ready: Condvar::new(), + }); + dirs.push(Arc::clone(&slot)); + Item::Dir(slot) } }) .collect(); - queue.open.extend(dirs.into_iter().rev()); - self.changed.notify_all(); + if dirs.is_empty() { + return items; + } + self.found.fetch_add(dirs.len(), Ordering::Relaxed); + let mut open = self.open_lock(); + let wake = open.idle.min(dirs.len()); + open.slots.extend(dirs.into_iter().map(Pending)); + drop(open); + for _ in 0..wake { + self.found_dir.notify_one(); + } items } } @@ -226,6 +296,30 @@ mod tests { threads.into_inner().unwrap().len() } + fn pending(key: &[u32]) -> Pending<(), ()> { + Pending(Arc::new(Slot { + key: key.to_vec(), + dir: Mutex::new(None), + explored: Mutex::new(Explored { + items: None, + awaited: false, + }), + ready: Condvar::new(), + })) + } + + #[test] + fn the_open_directories_yield_the_one_the_walk_reaches_first() { + let mut open = BinaryHeap::new(); + for key in [&[1][..], &[0, 2], &[0], &[0, 1, 5]] { + open.push(pending(key)); + } + let order: Vec> = std::iter::from_fn(|| open.pop()) + .map(|Pending(slot)| slot.key.clone()) + .collect(); + assert_eq!(order, vec![vec![0], vec![0, 1, 5], vec![0, 2], vec![1]]); + } + #[test] fn a_walk_of_few_directories_stays_on_the_calling_thread() { assert_eq!(threads_exploring(3, 1), 1); From 5927ded592edcd86a6af16efc38a38a80c61e8f2 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:45:41 -0400 Subject: [PATCH 02/10] e2e attestation for bot/1480-matcher-speed-1015 --- .../e2e-attestations/bot-1480-matcher-speed-1015.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/python/e2e-attestations/bot-1480-matcher-speed-1015.json diff --git a/packages/python/e2e-attestations/bot-1480-matcher-speed-1015.json b/packages/python/e2e-attestations/bot-1480-matcher-speed-1015.json new file mode 100644 index 000000000..c4bd30d38 --- /dev/null +++ b/packages/python/e2e-attestations/bot-1480-matcher-speed-1015.json @@ -0,0 +1,7 @@ +{ + "command": "uv run python -m pytest tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_gitignore_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_symlink_test.py tests/e2e/glob_speed_node_modules_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/glob_speed_depth_bound_test.py tests/e2e/stat_speed_test.py tests/e2e/fanout_test.py tests/e2e/query_contract_test.py tests/e2e/declared_table_name_test.py -x -q -s", + "ran_at": 1792075541, + "exit_code": 0, + "commit": "f6fdddfc1d2152ac7208fb0defcc9d6806afc062", + "branch": "bot/1480-matcher-speed-1015" +} From cfc70ceffffcc2730f98e25a2a36e137f53d3931 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:47:07 -0400 Subject: [PATCH 03/10] e2e attestation for bot/1480-matcher-speed-1015 --- .../ts/e2e-attestations/bot-1480-matcher-speed-1015.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/ts/e2e-attestations/bot-1480-matcher-speed-1015.json diff --git a/packages/ts/e2e-attestations/bot-1480-matcher-speed-1015.json b/packages/ts/e2e-attestations/bot-1480-matcher-speed-1015.json new file mode 100644 index 000000000..dc561248c --- /dev/null +++ b/packages/ts/e2e-attestations/bot-1480-matcher-speed-1015.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1792075627, + "exit_code": 0, + "commit": "5927ded592edcd86a6af16efc38a38a80c61e8f2", + "branch": "bot/1480-matcher-speed-1015" +} From 1978ca49a59fc6537aa47c6e1cd4f23826d6ecbe Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:15:02 -0400 Subject: [PATCH 04/10] test: kill the walk's surviving mutants; wake one worker per directory found --- packages/rust/src/scanner.rs | 25 +++++++++++++++++++++++++ packages/rust/src/tree_walk.rs | 20 +++++++++----------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/packages/rust/src/scanner.rs b/packages/rust/src/scanner.rs index 75e4719e7..5e9a53c71 100644 --- a/packages/rust/src/scanner.rs +++ b/packages/rust/src/scanner.rs @@ -1237,6 +1237,31 @@ mod tests { assert_eq!(judged.into_inner().unwrap(), vec![PathBuf::from("a.md")]); } + #[test] + fn explore_outside_gitignore_mode_ignores_a_repo_it_enters() { + let root = tempfile::tempdir().unwrap(); + fs::create_dir(root.path().join(".git")).unwrap(); + fs::write(root.path().join(".gitignore"), "x.md\n").unwrap(); + fs::write(root.path().join("x.md"), "").unwrap(); + let ignore = TableMatcher::new(&[], &[]).unwrap(); + let top = Place { + walk: walk_with(&ignore, None, Vec::new()), + dir: root.path().to_path_buf(), + rel: PathBuf::new(), + states: Vec::new(), + linked: false, + }; + let leaves: Vec = top + .explore(&|_| true, &|_| true, &|rel, _| rel) + .into_iter() + .filter_map(|step| match step { + Step::Leaf(rel) => Some(rel), + Step::Dir(_) => None, + }) + .collect(); + assert!(leaves.contains(&PathBuf::from("x.md")), "{leaves:?}"); + } + #[test] fn explore_enters_a_dirsql_directory_below_the_top_level() { let root = tempfile::tempdir().unwrap(); diff --git a/packages/rust/src/tree_walk.rs b/packages/rust/src/tree_walk.rs index b61bdcba5..9bafd2146 100644 --- a/packages/rust/src/tree_walk.rs +++ b/packages/rust/src/tree_walk.rs @@ -51,9 +51,6 @@ struct Explored { /// it is far from, which would leave it to explore its next directory itself. struct Open { slots: BinaryHeap>, - /// Workers waiting for a directory, so a find wakes only as many as it - /// has work for. - idle: usize, done: bool, } @@ -97,7 +94,6 @@ pub(crate) fn walk_in_order( let shared = Shared { open: Mutex::new(Open { slots: BinaryHeap::new(), - idle: 0, done: false, }), found_dir: Condvar::new(), @@ -184,12 +180,10 @@ impl Shared<'_, D, L> { if let Some(Pending(slot)) = open.slots.pop() { return Some(slot); } - open.idle += 1; open = self .found_dir .wait(open) .expect("exploring a directory does not panic"); - open.idle -= 1; } } @@ -222,11 +216,9 @@ impl Shared<'_, D, L> { return items; } self.found.fetch_add(dirs.len(), Ordering::Relaxed); - let mut open = self.open_lock(); - let wake = open.idle.min(dirs.len()); - open.slots.extend(dirs.into_iter().map(Pending)); - drop(open); - for _ in 0..wake { + let found = dirs.len(); + self.open_lock().slots.extend(dirs.into_iter().map(Pending)); + for _ in 0..found { self.found_dir.notify_one(); } items @@ -308,6 +300,12 @@ mod tests { })) } + #[test] + fn open_directories_are_equal_exactly_when_their_keys_are() { + assert!(pending(&[0, 1]) == pending(&[0, 1])); + assert!(pending(&[0, 1]) != pending(&[0, 2])); + } + #[test] fn the_open_directories_yield_the_one_the_walk_reaches_first() { let mut open = BinaryHeap::new(); From 581d7e2f57df9c78060a43a2bf9f81e600878921 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 16 Oct 2026 07:47:54 -0400 Subject: [PATCH 05/10] perf: read directory listings through Dir on Linux too --- packages/rust/src/listing.rs | 49 +++--------------------------------- 1 file changed, 4 insertions(+), 45 deletions(-) diff --git a/packages/rust/src/listing.rs b/packages/rust/src/listing.rs index b08217dec..b9125daa2 100644 --- a/packages/rust/src/listing.rs +++ b/packages/rust/src/listing.rs @@ -145,40 +145,11 @@ fn open_dir(dir: &Path) -> rustix::io::Result { rustix::fs::open(dir, flags, Mode::empty()) } -#[cfg(target_os = "linux")] -thread_local! { - /// Where a walk thread reads its directories' entries: big enough that a - /// directory of millions takes few `getdents` calls, and reused so a deep - /// tree of small directories pays for it once. - static ENTRY_BUFFER: std::cell::RefCell> = - std::cell::RefCell::new(Vec::with_capacity(1 << 20)); -} - -/// Reads entries straight out of the kernel's buffer, where `Dir` would copy -/// each name into its own allocation. -#[cfg(target_os = "linux")] +/// Uses `Dir`, not `RawDir`: on a deep tree of small directories the walk ran +/// about twice as slow reading through `RawDir`, with no gain on a flat +/// directory of a million entries. +#[cfg(unix)] fn read_into(dir: &Path, listing: &mut Listing) { - use rustix::fs::RawDir; - use std::os::unix::ffi::OsStrExt; - let Ok(fd) = open_dir(dir) else { - return; - }; - ENTRY_BUFFER.with_borrow_mut(|buffer| { - let mut entries = RawDir::new(&fd, buffer.spare_capacity_mut()); - while let Some(Ok(entry)) = entries.next() { - let name = entry.file_name().to_bytes(); - if name == b"." || name == b".." { - continue; - } - listing.push(OsStr::from_bytes(name), seen(entry.file_type())); - } - }); -} - -/// `read_into` for Unix systems without `RawDir`. Built under test too, so the -/// Linux unit run covers it. -#[cfg(all(unix, any(test, not(target_os = "linux"))))] -fn read_dir_with_copies_into(dir: &Path, listing: &mut Listing) { use rustix::fs::Dir; use std::os::unix::ffi::OsStrExt; let Ok(fd) = open_dir(dir) else { @@ -221,8 +192,6 @@ fn seen_flags(symlink: bool, dir: bool, file: bool) -> Seen { #[cfg(not(unix))] use read_dir_into as read_into; -#[cfg(all(unix, not(target_os = "linux")))] -use read_dir_with_copies_into as read_into; /// A name's first eight bytes as a big-endian word, which orders names as /// their bytes do as far as those bytes go. @@ -364,16 +333,6 @@ mod tests { assert_eq!(read_with(read_into, dir.path()), fixture_entries()); } - #[cfg(unix)] - #[test] - fn read_dir_with_copies_into_lists_the_entries_without_dot_and_dot_dot() { - let dir = fixture(); - assert_eq!( - read_with(read_dir_with_copies_into, dir.path()), - fixture_entries() - ); - } - #[test] fn read_dir_into_classifies_what_std_reports() { let dir = fixture(); From 1e1c34268c0eaf68930338bbf09a8ed093dc47be Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:49:58 -0400 Subject: [PATCH 06/10] e2e attestation for bot/1480-matcher-speed-1015 --- .../e2e-attestations/bot-1480-matcher-speed-1015.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/python/e2e-attestations/bot-1480-matcher-speed-1015.json b/packages/python/e2e-attestations/bot-1480-matcher-speed-1015.json index c4bd30d38..3be038055 100644 --- a/packages/python/e2e-attestations/bot-1480-matcher-speed-1015.json +++ b/packages/python/e2e-attestations/bot-1480-matcher-speed-1015.json @@ -1,7 +1,7 @@ { - "command": "uv run python -m pytest tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_gitignore_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_symlink_test.py tests/e2e/glob_speed_node_modules_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/glob_speed_depth_bound_test.py tests/e2e/stat_speed_test.py tests/e2e/fanout_test.py tests/e2e/query_contract_test.py tests/e2e/declared_table_name_test.py -x -q -s", - "ran_at": 1792075541, + "command": "uv run python -m pytest tests/e2e/glob_speed_one_level_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_depth_bound_test.py tests/e2e/glob_speed_gitignore_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_node_modules_test.py tests/e2e/glob_speed_symlink_test.py tests/e2e/stat_speed_test.py tests/e2e/query_contract_test.py -q", + "ran_at": 1792154998, "exit_code": 0, - "commit": "f6fdddfc1d2152ac7208fb0defcc9d6806afc062", + "commit": "581d7e2f57df9c78060a43a2bf9f81e600878921", "branch": "bot/1480-matcher-speed-1015" } From b87110efc75ac63b7f51b4cd90b419ebcb162080 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:51:14 -0400 Subject: [PATCH 07/10] e2e attestation for bot/1480-matcher-speed-1015 --- packages/ts/e2e-attestations/bot-1480-matcher-speed-1015.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ts/e2e-attestations/bot-1480-matcher-speed-1015.json b/packages/ts/e2e-attestations/bot-1480-matcher-speed-1015.json index dc561248c..ff80ee10e 100644 --- a/packages/ts/e2e-attestations/bot-1480-matcher-speed-1015.json +++ b/packages/ts/e2e-attestations/bot-1480-matcher-speed-1015.json @@ -1,7 +1,7 @@ { "command": "pnpm test:e2e", - "ran_at": 1792075627, + "ran_at": 1792155074, "exit_code": 0, - "commit": "5927ded592edcd86a6af16efc38a38a80c61e8f2", + "commit": "1e1c34268c0eaf68930338bbf09a8ed093dc47be", "branch": "bot/1480-matcher-speed-1015" } From e01da64bd1c48112f77059fab894961c5dc241ce Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:18:13 -0400 Subject: [PATCH 08/10] perf: a thread that finds directories takes one itself rather than waking a worker for it --- packages/rust/src/tree_walk.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/rust/src/tree_walk.rs b/packages/rust/src/tree_walk.rs index a7b0babc6..5f26f65b6 100644 --- a/packages/rust/src/tree_walk.rs +++ b/packages/rust/src/tree_walk.rs @@ -242,7 +242,9 @@ impl Shared<'_, D, L> { self.found.fetch_add(dirs.len(), Ordering::Relaxed); let found = dirs.len(); self.open_lock().slots.extend(dirs.into_iter().map(Pending)); - for _ in 0..found { + // The thread that found them takes one next, so only the rest need + // a waiting worker. + for _ in 1..found { self.found_dir.notify_one(); } items From 9cdbfeabf2b256d17a51bdc00b161efe438f5dd5 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:55:25 -0400 Subject: [PATCH 09/10] e2e attestation for bot/1480-matcher-speed-1015 --- .../e2e-attestations/bot-1480-matcher-speed-1015.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/python/e2e-attestations/bot-1480-matcher-speed-1015.json b/packages/python/e2e-attestations/bot-1480-matcher-speed-1015.json index 3be038055..4eeeb8eb2 100644 --- a/packages/python/e2e-attestations/bot-1480-matcher-speed-1015.json +++ b/packages/python/e2e-attestations/bot-1480-matcher-speed-1015.json @@ -1,7 +1,7 @@ { - "command": "uv run python -m pytest tests/e2e/glob_speed_one_level_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_depth_bound_test.py tests/e2e/glob_speed_gitignore_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_node_modules_test.py tests/e2e/glob_speed_symlink_test.py tests/e2e/stat_speed_test.py tests/e2e/query_contract_test.py -q", - "ran_at": 1792154998, + "command": "uv run python -m pytest tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_gitignore_test.py tests/e2e/glob_speed_depth_bound_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/query_contract_test.py -q", + "ran_at": 1792158925, "exit_code": 0, - "commit": "581d7e2f57df9c78060a43a2bf9f81e600878921", + "commit": "e01da64bd1c48112f77059fab894961c5dc241ce", "branch": "bot/1480-matcher-speed-1015" } From 8635f17fb125a015821e0e8bc589c240b967a487 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:01:14 -0400 Subject: [PATCH 10/10] e2e attestation for bot/1480-matcher-speed-1015 --- packages/ts/e2e-attestations/bot-1480-matcher-speed-1015.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ts/e2e-attestations/bot-1480-matcher-speed-1015.json b/packages/ts/e2e-attestations/bot-1480-matcher-speed-1015.json index ff80ee10e..1b0e51cbb 100644 --- a/packages/ts/e2e-attestations/bot-1480-matcher-speed-1015.json +++ b/packages/ts/e2e-attestations/bot-1480-matcher-speed-1015.json @@ -1,7 +1,7 @@ { "command": "pnpm test:e2e", - "ran_at": 1792155074, + "ran_at": 1792159274, "exit_code": 0, - "commit": "1e1c34268c0eaf68930338bbf09a8ed093dc47be", + "commit": "9cdbfeabf2b256d17a51bdc00b161efe438f5dd5", "branch": "bot/1480-matcher-speed-1015" }