diff --git a/packages/python/e2e-attestations/bot-1475-prefix-speed-1016.json b/packages/python/e2e-attestations/bot-1475-prefix-speed-1016.json new file mode 100644 index 000000000..7f3dab9c8 --- /dev/null +++ b/packages/python/e2e-attestations/bot-1475-prefix-speed-1016.json @@ -0,0 +1,7 @@ +{ + "command": "uv run python -m pytest tests/e2e/glob_speed_depth_bound_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_gitignore_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_node_modules_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_symlink_test.py tests/e2e/stat_speed_test.py tests/e2e/frontmatter_parse_speed_test.py tests/e2e/jsonl_parse_speed_test.py tests/e2e/query_contract_test.py tests/e2e/fanout_test.py tests/e2e/repl_interactive_test.py tests/e2e/server_speed_test.py -q", + "ran_at": 1793727994, + "exit_code": 0, + "commit": "1404c15d5c3baf22a495ea5a700fe036f74c300d", + "branch": "bot/1475-prefix-speed-1016" +} diff --git a/packages/rust/changelog.d/2026-10-16-prefix-glob-speed.md b/packages/rust/changelog.d/2026-10-16-prefix-glob-speed.md new file mode 100644 index 000000000..b34bfbda1 --- /dev/null +++ b/packages/rust/changelog.d/2026-10-16-prefix-glob-speed.md @@ -0,0 +1,3 @@ +**Changed** + +- **One-shot queries over a glob under a prefix, such as `./a/b/**/*.md`, run faster.** `dirsql query` no longer starts an async worker per core, and the walk starts its workers as a tree rather than all from one thread. Query results are unchanged. diff --git a/packages/rust/migrations.d/2026-10-16-prefix-glob-speed.md b/packages/rust/migrations.d/2026-10-16-prefix-glob-speed.md new file mode 100644 index 000000000..87692fb83 --- /dev/null +++ b/packages/rust/migrations.d/2026-10-16-prefix-glob-speed.md @@ -0,0 +1,25 @@ +### Prefix glob speed + +#### Summary + +One-shot CLI queries start fewer threads, and the walk starts its workers as a tree. No API changes. + +#### Required changes + +_None._ + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +_None._ + +#### Verification + +```bash +dirsql query "SELECT path FROM './a/b/**/*.md'" +``` + +Prints the same rows as before, sooner. diff --git a/packages/rust/src/cli/run.rs b/packages/rust/src/cli/run.rs index e4b700b31..46a07b504 100644 --- a/packages/rust/src/cli/run.rs +++ b/packages/rust/src/cli/run.rs @@ -329,7 +329,13 @@ pub fn run_cli(argv: Vec) -> i32 { } }; - let runtime = match tokio::runtime::Runtime::new() { + let runtime = match if serves(&cli) { + tokio::runtime::Runtime::new() + } else { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + } { Ok(runtime) => runtime, Err(err) => { eprintln!("dirsql: failed to start the async runtime: {err}"); @@ -352,6 +358,13 @@ pub fn run_cli(argv: Vec) -> i32 { i32::from(code) } +/// Whether `cli` keeps serving, as the server and the REPL do, and so wants a +/// runtime with a worker per core; a one-shot command waits on one thing at a +/// time, and starting and stopping those workers is a cost it would only pay. +fn serves(cli: &Cli) -> bool { + matches!(cli.command, Some(Command::Server(_))) || (cli.command.is_none() && cli.sql.is_none()) +} + /// The default (no-subcommand) behavior. With SQL, run it as a one-shot query, /// exactly as `dirsql query ""` does. With no SQL, read statements until /// EOF instead: bare `dirsql` is a REPL over the current directory, prompted @@ -679,6 +692,24 @@ mod tests { assert_eq!(query_body(" "), r#"{"sql":" "}"#); } + #[test] + fn the_server_and_the_repl_serve() { + let serving = + [&["dirsql", "server"][..], &["dirsql"]].map(|argv| serves(&Cli::parse_from(argv))); + assert_eq!(serving, [true, true]); + } + + #[test] + fn one_shot_commands_do_not_serve() { + let serving = [ + &["dirsql", "query", "SELECT 1"][..], + &["dirsql", "SELECT 1"], + &["dirsql", "context"], + ] + .map(|argv| serves(&Cli::parse_from(argv))); + assert_eq!(serving, [false, false, false]); + } + fn query_args(argv: &[&str]) -> QueryArgs { match Cli::parse_from(argv).command { Some(Command::Query(args)) => args, diff --git a/packages/rust/src/tree_walk.rs b/packages/rust/src/tree_walk.rs index 23f631ae5..bc79c9f15 100644 --- a/packages/rust/src/tree_walk.rs +++ b/packages/rust/src/tree_walk.rs @@ -159,9 +159,7 @@ pub(crate) fn walk_in_order( if !spawned && shared.found.load(Ordering::Relaxed) >= PARALLEL_DIRS { spawned = true; let workers = thread::available_parallelism().map_or(1, usize::from); - for _ in 1..workers { - scope.spawn(|| shared.work()); - } + spawn_workers(scope, &shared, (1..workers).collect()); } stack.push(shared.obtain(&slot).into_iter()); } @@ -179,6 +177,31 @@ fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { mutex.lock().expect("exploring a directory does not panic") } +/// Start a worker for each of `team` as a tree: each worker starts half of +/// the rest before it explores, since spawning a thread takes tens of +/// microseconds and the walk hands out nothing while it spawns. Every worker +/// starts a strictly smaller team than it was handed, so the tree is finite. +fn spawn_workers<'scope, D: Send, L: Send>( + scope: &'scope thread::Scope<'scope, '_>, + shared: &'scope Shared<'_, D, L>, + mut team: Vec, +) { + if team.pop().is_none() { + return; + } + let (first, second) = halves(team); + scope.spawn(move || { + spawn_workers(scope, shared, first); + spawn_workers(scope, shared, second); + shared.work(); + }); +} + +fn halves(mut team: Vec) -> (Vec, Vec) { + let second = team.split_off(team.len() / 2); + (team, second) +} + impl Shared<'_, D, L> { fn open_lock(&self) -> MutexGuard<'_, Open> { lock(&self.open) @@ -457,6 +480,11 @@ mod tests { assert!(threads_exploring(4, 2) > 1); } + #[test] + fn a_team_splits_into_halves_the_second_taking_any_odd_one() { + assert_eq!(halves(vec![1, 2, 3, 4, 5]), (vec![1, 2], vec![3, 4, 5])); + } + #[test] fn a_walk_spawns_its_workers_once() { let cores = thread::available_parallelism().map_or(1, usize::from); diff --git a/packages/ts/e2e-attestations/bot-1475-prefix-speed-1016.json b/packages/ts/e2e-attestations/bot-1475-prefix-speed-1016.json new file mode 100644 index 000000000..55e41f480 --- /dev/null +++ b/packages/ts/e2e-attestations/bot-1475-prefix-speed-1016.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1793728044, + "exit_code": 0, + "commit": "26b351ed743d0a9352f95f176f6e3563fb862159", + "branch": "bot/1475-prefix-speed-1016" +}