Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"command": "uv run python -m pytest tests/e2e/glob_speed_depth_bound_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_symlink_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_gitignore_test.py -x -q",
"ran_at": 1791777891,
"exit_code": 0,
"commit": "fb7068d3883fd35c014146a174f4f342aaf93dab",
"branch": "bot/speed-gap-childpath-1011"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
**Changed**

- **A tree walk builds each entry's path in one allocation.** Recursive globs over large trees, such as `./**/{a,b}*.md`, list faster because the walk threads no longer contend the allocator. Output is unchanged.
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
### Faster recursive walks

#### Summary

The tree walk builds each entry's path in one allocation, so its threads contend the allocator less. Query results and their order are unchanged; nothing breaks.

#### Required changes

_None._

#### Deprecations removed

_None._

#### Behavior changes without code changes

_None._

#### Verification

```bash
dirsql query "SELECT count(*) FROM './*/*.md'"
```

Prints the same count as before, sooner.
27 changes: 26 additions & 1 deletion packages/rust/src/scanner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -612,6 +612,17 @@ struct Walk<'a> {
dirs: bool,
}

/// `rel.join(name)` in one allocation: the walk builds one per entry on every
/// worker thread, and `join`'s grow-on-push contends the allocator's arenas.
fn child_path(rel: &Path, name: &OsStr) -> PathBuf {
let rel_len = rel.as_os_str().len();
let separator = usize::from(rel_len > 0);
let mut child = PathBuf::with_capacity(rel_len + separator + name.len());
child.push(rel);
child.push(name);
child
}

impl Walk<'_> {
/// What the walk makes of entry `name` of `dir`, seen as `seen`, the
/// directory at `rel`, reached with glob `states` and `linked` as whether
Expand All @@ -638,7 +649,7 @@ impl Walk<'_> {
return None;
}
let is_dir = kind != Kind::File;
let child = rel.join(name);
let child = child_path(rel, name);
// Only a `.gitignore` reads the full path, and building one per entry
// is a large share of a big directory's walk.
let path = if self.frames.is_empty() {
Expand Down Expand Up @@ -1042,6 +1053,20 @@ mod tests {
)
}

#[test]
fn child_path_allocates_exactly_its_length() {
let child = child_path(Path::new("alpha/beta/gamma"), OsStr::new("delta.md"));
assert_eq!(child, Path::new("alpha/beta/gamma/delta.md"));
assert_eq!(child.capacity(), child.as_os_str().len());
}

#[test]
fn child_path_of_the_root_is_the_bare_name() {
let child = child_path(Path::new(""), OsStr::new("c.md"));
assert_eq!(child, Path::new("c.md"));
assert_eq!(child.capacity(), 4);
}

#[test]
fn take_enters_a_real_directory_a_component_can_enter_off_any_link() {
let ignore = TableMatcher::new(&[], &[]).unwrap();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"command": "pnpm test:e2e",
"ran_at": 1791777966,
"exit_code": 0,
"commit": "68d5f9c6ff6063ccefe3f32c1bd394ef7d4b5ef9",
"branch": "bot/speed-gap-childpath-1011"
}
Loading