Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"command": "uv run python -m pytest tests/e2e/glob_speed_depth_bound_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_gitignore_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_node_modules_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_symlink_test.py tests/e2e/stat_speed_test.py tests/e2e/frontmatter_parse_speed_test.py tests/e2e/jsonl_parse_speed_test.py tests/e2e/query_contract_test.py tests/e2e/fanout_test.py tests/e2e/repl_interactive_test.py tests/e2e/server_speed_test.py -q",
"ran_at": 1793727994,
"exit_code": 0,
"commit": "1404c15d5c3baf22a495ea5a700fe036f74c300d",
"branch": "bot/1475-prefix-speed-1016"
}
3 changes: 3 additions & 0 deletions packages/rust/changelog.d/2026-10-16-prefix-glob-speed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
**Changed**

- **One-shot queries over a glob under a prefix, such as `./a/b/**/*.md`, run faster.** `dirsql query` no longer starts an async worker per core, and the walk starts its workers as a tree rather than all from one thread. Query results are unchanged.
25 changes: 25 additions & 0 deletions packages/rust/migrations.d/2026-10-16-prefix-glob-speed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
### Prefix glob speed

#### Summary

One-shot CLI queries start fewer threads, and the walk starts its workers as a tree. No API changes.

#### Required changes

_None._

#### Deprecations removed

_None._

#### Behavior changes without code changes

_None._

#### Verification

```bash
dirsql query "SELECT path FROM './a/b/**/*.md'"
```

Prints the same rows as before, sooner.
33 changes: 32 additions & 1 deletion packages/rust/src/cli/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,13 @@ pub fn run_cli(argv: Vec<String>) -> i32 {
}
};

let runtime = match tokio::runtime::Runtime::new() {
let runtime = match if serves(&cli) {
tokio::runtime::Runtime::new()
} else {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
} {
Ok(runtime) => runtime,
Err(err) => {
eprintln!("dirsql: failed to start the async runtime: {err}");
Expand All @@ -352,6 +358,13 @@ pub fn run_cli(argv: Vec<String>) -> i32 {
i32::from(code)
}

/// Whether `cli` keeps serving, as the server and the REPL do, and so wants a
/// runtime with a worker per core; a one-shot command waits on one thing at a
/// time, and starting and stopping those workers is a cost it would only pay.
fn serves(cli: &Cli) -> bool {
matches!(cli.command, Some(Command::Server(_))) || (cli.command.is_none() && cli.sql.is_none())
}

/// The default (no-subcommand) behavior. With SQL, run it as a one-shot query,
/// exactly as `dirsql query "<sql>"` does. With no SQL, read statements until
/// EOF instead: bare `dirsql` is a REPL over the current directory, prompted
Expand Down Expand Up @@ -679,6 +692,24 @@ mod tests {
assert_eq!(query_body(" "), r#"{"sql":" "}"#);
}

#[test]
fn the_server_and_the_repl_serve() {
let serving =
[&["dirsql", "server"][..], &["dirsql"]].map(|argv| serves(&Cli::parse_from(argv)));
assert_eq!(serving, [true, true]);
}

#[test]
fn one_shot_commands_do_not_serve() {
let serving = [
&["dirsql", "query", "SELECT 1"][..],
&["dirsql", "SELECT 1"],
&["dirsql", "context"],
]
.map(|argv| serves(&Cli::parse_from(argv)));
assert_eq!(serving, [false, false, false]);
}

fn query_args(argv: &[&str]) -> QueryArgs {
match Cli::parse_from(argv).command {
Some(Command::Query(args)) => args,
Expand Down
34 changes: 31 additions & 3 deletions packages/rust/src/tree_walk.rs
Original file line number Diff line number Diff line change
Expand Up @@ -159,9 +159,7 @@ pub(crate) fn walk_in_order<D: Send, L: Send>(
if !spawned && shared.found.load(Ordering::Relaxed) >= PARALLEL_DIRS {
spawned = true;
let workers = thread::available_parallelism().map_or(1, usize::from);
for _ in 1..workers {
scope.spawn(|| shared.work());
}
spawn_workers(scope, &shared, (1..workers).collect());
}
stack.push(shared.obtain(&slot).into_iter());
}
Expand All @@ -179,6 +177,31 @@ fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
mutex.lock().expect("exploring a directory does not panic")
}

/// Start a worker for each of `team` as a tree: each worker starts half of
/// the rest before it explores, since spawning a thread takes tens of
/// microseconds and the walk hands out nothing while it spawns. Every worker
/// starts a strictly smaller team than it was handed, so the tree is finite.
fn spawn_workers<'scope, D: Send, L: Send>(
scope: &'scope thread::Scope<'scope, '_>,
shared: &'scope Shared<'_, D, L>,
mut team: Vec<usize>,
) {
if team.pop().is_none() {
return;
}
let (first, second) = halves(team);
scope.spawn(move || {
spawn_workers(scope, shared, first);
spawn_workers(scope, shared, second);
shared.work();
});
}

fn halves(mut team: Vec<usize>) -> (Vec<usize>, Vec<usize>) {
let second = team.split_off(team.len() / 2);
(team, second)
}

impl<D, L> Shared<'_, D, L> {
fn open_lock(&self) -> MutexGuard<'_, Open<D, L>> {
lock(&self.open)
Expand Down Expand Up @@ -457,6 +480,11 @@ mod tests {
assert!(threads_exploring(4, 2) > 1);
}

#[test]
fn a_team_splits_into_halves_the_second_taking_any_odd_one() {
assert_eq!(halves(vec![1, 2, 3, 4, 5]), (vec![1, 2], vec![3, 4, 5]));
}

#[test]
fn a_walk_spawns_its_workers_once() {
let cores = thread::available_parallelism().map_or(1, usize::from);
Expand Down
7 changes: 7 additions & 0 deletions packages/ts/e2e-attestations/bot-1475-prefix-speed-1016.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"command": "pnpm test:e2e",
"ran_at": 1793728044,
"exit_code": 0,
"commit": "26b351ed743d0a9352f95f176f6e3563fb862159",
"branch": "bot/1475-prefix-speed-1016"
}
Loading