diff --git a/packages/python/e2e-attestations/bot-speed-gap-childpath-1011.json b/packages/python/e2e-attestations/bot-speed-gap-childpath-1011.json new file mode 100644 index 000000000..b1fe6f37a --- /dev/null +++ b/packages/python/e2e-attestations/bot-speed-gap-childpath-1011.json @@ -0,0 +1,7 @@ +{ + "command": "uv run python -m pytest tests/e2e/glob_speed_depth_bound_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_symlink_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_gitignore_test.py -x -q", + "ran_at": 1791777891, + "exit_code": 0, + "commit": "fb7068d3883fd35c014146a174f4f342aaf93dab", + "branch": "bot/speed-gap-childpath-1011" +} diff --git a/packages/rust/changelog.d/2026-10-11-walk-entry-path-one-allocation.md b/packages/rust/changelog.d/2026-10-11-walk-entry-path-one-allocation.md new file mode 100644 index 000000000..6a0a3a948 --- /dev/null +++ b/packages/rust/changelog.d/2026-10-11-walk-entry-path-one-allocation.md @@ -0,0 +1,3 @@ +**Changed** + +- **A tree walk builds each entry's path in one allocation.** Recursive globs over large trees, such as `./**/{a,b}*.md`, list faster because the walk threads no longer contend the allocator. Output is unchanged. diff --git a/packages/rust/migrations.d/2026-10-11-walk-entry-path-one-allocation.md b/packages/rust/migrations.d/2026-10-11-walk-entry-path-one-allocation.md new file mode 100644 index 000000000..209d43ca5 --- /dev/null +++ b/packages/rust/migrations.d/2026-10-11-walk-entry-path-one-allocation.md @@ -0,0 +1,25 @@ +### Faster recursive walks + +#### Summary + +The tree walk builds each entry's path in one allocation, so its threads contend the allocator less. Query results and their order are unchanged; nothing breaks. + +#### Required changes + +_None._ + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +_None._ + +#### Verification + +```bash +dirsql query "SELECT count(*) FROM './*/*.md'" +``` + +Prints the same count as before, sooner. diff --git a/packages/rust/src/scanner.rs b/packages/rust/src/scanner.rs index d2f377cbf..250199eab 100644 --- a/packages/rust/src/scanner.rs +++ b/packages/rust/src/scanner.rs @@ -612,6 +612,17 @@ struct Walk<'a> { dirs: bool, } +/// `rel.join(name)` in one allocation: the walk builds one per entry on every +/// worker thread, and `join`'s grow-on-push contends the allocator's arenas. +fn child_path(rel: &Path, name: &OsStr) -> PathBuf { + let rel_len = rel.as_os_str().len(); + let separator = usize::from(rel_len > 0); + let mut child = PathBuf::with_capacity(rel_len + separator + name.len()); + child.push(rel); + child.push(name); + child +} + impl Walk<'_> { /// What the walk makes of entry `name` of `dir`, seen as `seen`, the /// directory at `rel`, reached with glob `states` and `linked` as whether @@ -638,7 +649,7 @@ impl Walk<'_> { return None; } let is_dir = kind != Kind::File; - let child = rel.join(name); + let child = child_path(rel, name); // Only a `.gitignore` reads the full path, and building one per entry // is a large share of a big directory's walk. let path = if self.frames.is_empty() { @@ -1042,6 +1053,20 @@ mod tests { ) } + #[test] + fn child_path_allocates_exactly_its_length() { + let child = child_path(Path::new("alpha/beta/gamma"), OsStr::new("delta.md")); + assert_eq!(child, Path::new("alpha/beta/gamma/delta.md")); + assert_eq!(child.capacity(), child.as_os_str().len()); + } + + #[test] + fn child_path_of_the_root_is_the_bare_name() { + let child = child_path(Path::new(""), OsStr::new("c.md")); + assert_eq!(child, Path::new("c.md")); + assert_eq!(child.capacity(), 4); + } + #[test] fn take_enters_a_real_directory_a_component_can_enter_off_any_link() { let ignore = TableMatcher::new(&[], &[]).unwrap(); diff --git a/packages/ts/e2e-attestations/bot-speed-gap-childpath-1011.json b/packages/ts/e2e-attestations/bot-speed-gap-childpath-1011.json new file mode 100644 index 000000000..456e973e1 --- /dev/null +++ b/packages/ts/e2e-attestations/bot-speed-gap-childpath-1011.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1791777966, + "exit_code": 0, + "commit": "68d5f9c6ff6063ccefe3f32c1bd394ef7d4b5ef9", + "branch": "bot/speed-gap-childpath-1011" +}