From 7ee50b0045ee01b980480a9f9d6753eba911e4d8 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:31:06 -0400 Subject: [PATCH 1/4] test: a walk entry's path is allocated at its exact length --- packages/rust/src/scanner.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/rust/src/scanner.rs b/packages/rust/src/scanner.rs index d2f377cbf..a31300c69 100644 --- a/packages/rust/src/scanner.rs +++ b/packages/rust/src/scanner.rs @@ -612,6 +612,10 @@ struct Walk<'a> { dirs: bool, } +fn child_path(rel: &Path, name: &OsStr) -> PathBuf { + rel.join(name) +} + impl Walk<'_> { /// What the walk makes of entry `name` of `dir`, seen as `seen`, the /// directory at `rel`, reached with glob `states` and `linked` as whether @@ -638,7 +642,7 @@ impl Walk<'_> { return None; } let is_dir = kind != Kind::File; - let child = rel.join(name); + let child = child_path(rel, name); // Only a `.gitignore` reads the full path, and building one per entry // is a large share of a big directory's walk. let path = if self.frames.is_empty() { @@ -1042,6 +1046,20 @@ mod tests { ) } + #[test] + fn child_path_allocates_exactly_its_length() { + let child = child_path(Path::new("alpha/beta/gamma"), OsStr::new("delta.md")); + assert_eq!(child, Path::new("alpha/beta/gamma/delta.md")); + assert_eq!(child.capacity(), child.as_os_str().len()); + } + + #[test] + fn child_path_of_the_root_is_the_bare_name() { + let child = child_path(Path::new(""), OsStr::new("c.md")); + assert_eq!(child, Path::new("c.md")); + assert_eq!(child.capacity(), 4); + } + #[test] fn take_enters_a_real_directory_a_component_can_enter_off_any_link() { let ignore = TableMatcher::new(&[], &[]).unwrap(); From fb7068d3883fd35c014146a174f4f342aaf93dab Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Sun, 11 Oct 2026 23:36:10 -0400 Subject: [PATCH 2/4] perf: build each walk entry's path in one allocation --- ...26-10-11-walk-entry-path-one-allocation.md | 3 +++ ...26-10-11-walk-entry-path-one-allocation.md | 25 +++++++++++++++++++ packages/rust/src/scanner.rs | 9 ++++++- 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 packages/rust/changelog.d/2026-10-11-walk-entry-path-one-allocation.md create mode 100644 packages/rust/migrations.d/2026-10-11-walk-entry-path-one-allocation.md diff --git a/packages/rust/changelog.d/2026-10-11-walk-entry-path-one-allocation.md b/packages/rust/changelog.d/2026-10-11-walk-entry-path-one-allocation.md new file mode 100644 index 000000000..6a0a3a948 --- /dev/null +++ b/packages/rust/changelog.d/2026-10-11-walk-entry-path-one-allocation.md @@ -0,0 +1,3 @@ +**Changed** + +- **A tree walk builds each entry's path in one allocation.** Recursive globs over large trees, such as `./**/{a,b}*.md`, list faster because the walk threads no longer contend the allocator. Output is unchanged. diff --git a/packages/rust/migrations.d/2026-10-11-walk-entry-path-one-allocation.md b/packages/rust/migrations.d/2026-10-11-walk-entry-path-one-allocation.md new file mode 100644 index 000000000..209d43ca5 --- /dev/null +++ b/packages/rust/migrations.d/2026-10-11-walk-entry-path-one-allocation.md @@ -0,0 +1,25 @@ +### Faster recursive walks + +#### Summary + +The tree walk builds each entry's path in one allocation, so its threads contend the allocator less. Query results and their order are unchanged; nothing breaks. + +#### Required changes + +_None._ + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +_None._ + +#### Verification + +```bash +dirsql query "SELECT count(*) FROM './*/*.md'" +``` + +Prints the same count as before, sooner. diff --git a/packages/rust/src/scanner.rs b/packages/rust/src/scanner.rs index a31300c69..250199eab 100644 --- a/packages/rust/src/scanner.rs +++ b/packages/rust/src/scanner.rs @@ -612,8 +612,15 @@ struct Walk<'a> { dirs: bool, } +/// `rel.join(name)` in one allocation: the walk builds one per entry on every +/// worker thread, and `join`'s grow-on-push contends the allocator's arenas. fn child_path(rel: &Path, name: &OsStr) -> PathBuf { - rel.join(name) + let rel_len = rel.as_os_str().len(); + let separator = usize::from(rel_len > 0); + let mut child = PathBuf::with_capacity(rel_len + separator + name.len()); + child.push(rel); + child.push(name); + child } impl Walk<'_> { From 68d5f9c6ff6063ccefe3f32c1bd394ef7d4b5ef9 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Mon, 12 Oct 2026 00:04:51 -0400 Subject: [PATCH 3/4] e2e attestation for bot/speed-gap-childpath-1011 --- .../e2e-attestations/bot-speed-gap-childpath-1011.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/python/e2e-attestations/bot-speed-gap-childpath-1011.json diff --git a/packages/python/e2e-attestations/bot-speed-gap-childpath-1011.json b/packages/python/e2e-attestations/bot-speed-gap-childpath-1011.json new file mode 100644 index 000000000..b1fe6f37a --- /dev/null +++ b/packages/python/e2e-attestations/bot-speed-gap-childpath-1011.json @@ -0,0 +1,7 @@ +{ + "command": "uv run python -m pytest tests/e2e/glob_speed_depth_bound_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_symlink_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_gitignore_test.py -x -q", + "ran_at": 1791777891, + "exit_code": 0, + "commit": "fb7068d3883fd35c014146a174f4f342aaf93dab", + "branch": "bot/speed-gap-childpath-1011" +} From 34f1517690e67547e30101325ff5e256671b8cb3 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Mon, 12 Oct 2026 00:06:06 -0400 Subject: [PATCH 4/4] e2e attestation for bot/speed-gap-childpath-1011 --- .../ts/e2e-attestations/bot-speed-gap-childpath-1011.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/ts/e2e-attestations/bot-speed-gap-childpath-1011.json diff --git a/packages/ts/e2e-attestations/bot-speed-gap-childpath-1011.json b/packages/ts/e2e-attestations/bot-speed-gap-childpath-1011.json new file mode 100644 index 000000000..456e973e1 --- /dev/null +++ b/packages/ts/e2e-attestations/bot-speed-gap-childpath-1011.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1791777966, + "exit_code": 0, + "commit": "68d5f9c6ff6063ccefe3f32c1bd394ef7d4b5ef9", + "branch": "bot/speed-gap-childpath-1011" +}