From d5d01de40c7a4dba9b6e7b4b880ed44030d6b219 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 16 Oct 2026 08:06:52 -0400 Subject: [PATCH 1/8] perf: one-shot queries on a current-thread runtime; walk on half the cores past sixteen Workers spawn as a tree, so the walking thread starts one rather than all of them. --- .../2026-10-16-prefix-glob-speed.md | 3 ++ .../2026-10-16-prefix-glob-speed.md | 25 ++++++++++ packages/rust/src/cli/run.rs | 33 ++++++++++++- packages/rust/src/tree_walk.rs | 49 +++++++++++++++++-- 4 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 packages/rust/changelog.d/2026-10-16-prefix-glob-speed.md create mode 100644 packages/rust/migrations.d/2026-10-16-prefix-glob-speed.md diff --git a/packages/rust/changelog.d/2026-10-16-prefix-glob-speed.md b/packages/rust/changelog.d/2026-10-16-prefix-glob-speed.md new file mode 100644 index 000000000..4d5b4a4fb --- /dev/null +++ b/packages/rust/changelog.d/2026-10-16-prefix-glob-speed.md @@ -0,0 +1,3 @@ +**Changed** + +- **One-shot queries over a glob under a prefix, such as `./a/b/**/*.md`, run faster.** `dirsql query` no longer starts an async worker per core, and on machines with more than sixteen cores the walk explores on half of them. Query results are unchanged. diff --git a/packages/rust/migrations.d/2026-10-16-prefix-glob-speed.md b/packages/rust/migrations.d/2026-10-16-prefix-glob-speed.md new file mode 100644 index 000000000..a8aa8a1ea --- /dev/null +++ b/packages/rust/migrations.d/2026-10-16-prefix-glob-speed.md @@ -0,0 +1,25 @@ +### Prefix glob speed + +#### Summary + +One-shot CLI queries start fewer threads, and the walk uses half the cores on machines with more than sixteen. No API changes. + +#### Required changes + +_None._ + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +_None._ + +#### Verification + +```bash +dirsql query "SELECT path FROM './a/b/**/*.md'" +``` + +Prints the same rows as before, sooner. diff --git a/packages/rust/src/cli/run.rs b/packages/rust/src/cli/run.rs index e4b700b31..46a07b504 100644 --- a/packages/rust/src/cli/run.rs +++ b/packages/rust/src/cli/run.rs @@ -329,7 +329,13 @@ pub fn run_cli(argv: Vec) -> i32 { } }; - let runtime = match tokio::runtime::Runtime::new() { + let runtime = match if serves(&cli) { + tokio::runtime::Runtime::new() + } else { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + } { Ok(runtime) => runtime, Err(err) => { eprintln!("dirsql: failed to start the async runtime: {err}"); @@ -352,6 +358,13 @@ pub fn run_cli(argv: Vec) -> i32 { i32::from(code) } +/// Whether `cli` keeps serving, as the server and the REPL do, and so wants a +/// runtime with a worker per core; a one-shot command waits on one thing at a +/// time, and starting and stopping those workers is a cost it would only pay. +fn serves(cli: &Cli) -> bool { + matches!(cli.command, Some(Command::Server(_))) || (cli.command.is_none() && cli.sql.is_none()) +} + /// The default (no-subcommand) behavior. With SQL, run it as a one-shot query, /// exactly as `dirsql query ""` does. With no SQL, read statements until /// EOF instead: bare `dirsql` is a REPL over the current directory, prompted @@ -679,6 +692,24 @@ mod tests { assert_eq!(query_body(" "), r#"{"sql":" "}"#); } + #[test] + fn the_server_and_the_repl_serve() { + let serving = + [&["dirsql", "server"][..], &["dirsql"]].map(|argv| serves(&Cli::parse_from(argv))); + assert_eq!(serving, [true, true]); + } + + #[test] + fn one_shot_commands_do_not_serve() { + let serving = [ + &["dirsql", "query", "SELECT 1"][..], + &["dirsql", "SELECT 1"], + &["dirsql", "context"], + ] + .map(|argv| serves(&Cli::parse_from(argv))); + assert_eq!(serving, [false, false, false]); + } + fn query_args(argv: &[&str]) -> QueryArgs { match Cli::parse_from(argv).command { Some(Command::Query(args)) => args, diff --git a/packages/rust/src/tree_walk.rs b/packages/rust/src/tree_walk.rs index 23f631ae5..7b6ee7552 100644 --- a/packages/rust/src/tree_walk.rs +++ b/packages/rust/src/tree_walk.rs @@ -158,10 +158,9 @@ pub(crate) fn walk_in_order( Some(Item::Dir(slot)) => { if !spawned && shared.found.load(Ordering::Relaxed) >= PARALLEL_DIRS { spawned = true; - let workers = thread::available_parallelism().map_or(1, usize::from); - for _ in 1..workers { - scope.spawn(|| shared.work()); - } + let workers = + worker_count(thread::available_parallelism().map_or(1, usize::from)); + spawn_worker(scope, &shared, 1, workers); } stack.push(shared.obtain(&slot).into_iter()); } @@ -179,6 +178,33 @@ fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { mutex.lock().expect("exploring a directory does not panic") } +/// How many threads explore on a machine of `cores` logical cores: half, as +/// past the physical cores a worker mostly takes time from the walking thread +/// and the thread reading its rows; but every core up to eight, as a small +/// machine walks slower on fewer. +fn worker_count(cores: usize) -> usize { + (cores / 2).max(cores.min(8)) +} + +/// Start worker `n` of `workers`, which starts workers `2n` and `2n + 1` +/// before it explores: spawning a thread takes tens of microseconds, and the +/// walk hands out nothing while it spawns. +fn spawn_worker<'scope, D: Send, L: Send>( + scope: &'scope thread::Scope<'scope, '_>, + shared: &'scope Shared<'_, D, L>, + n: usize, + workers: usize, +) { + if n >= workers { + return; + } + scope.spawn(move || { + spawn_worker(scope, shared, 2 * n, workers); + spawn_worker(scope, shared, 2 * n + 1, workers); + shared.work(); + }); +} + impl Shared<'_, D, L> { fn open_lock(&self) -> MutexGuard<'_, Open> { lock(&self.open) @@ -457,6 +483,21 @@ mod tests { assert!(threads_exploring(4, 2) > 1); } + #[test] + fn a_large_machine_explores_on_half_its_cores() { + assert_eq!(worker_count(32), 16); + } + + #[test] + fn a_small_machine_explores_on_every_core() { + assert_eq!([1, 4, 8].map(worker_count), [1, 4, 8]); + } + + #[test] + fn a_machine_explores_on_at_least_eight_cores() { + assert_eq!(worker_count(12), 8); + } + #[test] fn a_walk_spawns_its_workers_once() { let cores = thread::available_parallelism().map_or(1, usize::from); From 0eab1212547168048052571f1efc785fbfd39cb3 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 16 Oct 2026 09:13:10 -0400 Subject: [PATCH 2/8] test: kill the worker-numbering mutants --- packages/rust/src/tree_walk.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/rust/src/tree_walk.rs b/packages/rust/src/tree_walk.rs index 7b6ee7552..3b1b5a1ab 100644 --- a/packages/rust/src/tree_walk.rs +++ b/packages/rust/src/tree_walk.rs @@ -186,8 +186,8 @@ fn worker_count(cores: usize) -> usize { (cores / 2).max(cores.min(8)) } -/// Start worker `n` of `workers`, which starts workers `2n` and `2n + 1` -/// before it explores: spawning a thread takes tens of microseconds, and the +/// Start worker `n` of `workers`, which starts its [`children`] before it +/// explores: spawning a thread takes tens of microseconds, and the /// walk hands out nothing while it spawns. fn spawn_worker<'scope, D: Send, L: Send>( scope: &'scope thread::Scope<'scope, '_>, @@ -199,12 +199,19 @@ fn spawn_worker<'scope, D: Send, L: Send>( return; } scope.spawn(move || { - spawn_worker(scope, shared, 2 * n, workers); - spawn_worker(scope, shared, 2 * n + 1, workers); + for child in children(n) { + spawn_worker(scope, shared, child, workers); + } shared.work(); }); } +/// The workers worker `n` starts, numbered so that, from worker 1, each +/// number below the worker count is started exactly once. +fn children(n: usize) -> std::ops::Range { + 2 * n..2 * n + 2 +} + impl Shared<'_, D, L> { fn open_lock(&self) -> MutexGuard<'_, Open> { lock(&self.open) @@ -498,6 +505,11 @@ mod tests { assert_eq!(worker_count(12), 8); } + #[test] + fn a_worker_starts_the_two_numbered_after_twice_its_own() { + assert_eq!([1, 5].map(children), [2..4, 10..12]); + } + #[test] fn a_walk_spawns_its_workers_once() { let cores = thread::available_parallelism().map_or(1, usize::from); From 91474ddfe5a00cd0a7ae154b1ea87c1ef59e26b1 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Tue, 3 Nov 2026 07:23:28 -0500 Subject: [PATCH 3/8] perf: walk on every core --- .../2026-10-16-prefix-glob-speed.md | 2 +- .../2026-10-16-prefix-glob-speed.md | 2 +- packages/rust/src/tree_walk.rs | 26 +------------------ 3 files changed, 3 insertions(+), 27 deletions(-) diff --git a/packages/rust/changelog.d/2026-10-16-prefix-glob-speed.md b/packages/rust/changelog.d/2026-10-16-prefix-glob-speed.md index 4d5b4a4fb..b34bfbda1 100644 --- a/packages/rust/changelog.d/2026-10-16-prefix-glob-speed.md +++ b/packages/rust/changelog.d/2026-10-16-prefix-glob-speed.md @@ -1,3 +1,3 @@ **Changed** -- **One-shot queries over a glob under a prefix, such as `./a/b/**/*.md`, run faster.** `dirsql query` no longer starts an async worker per core, and on machines with more than sixteen cores the walk explores on half of them. Query results are unchanged. +- **One-shot queries over a glob under a prefix, such as `./a/b/**/*.md`, run faster.** `dirsql query` no longer starts an async worker per core, and the walk starts its workers as a tree rather than all from one thread. Query results are unchanged. diff --git a/packages/rust/migrations.d/2026-10-16-prefix-glob-speed.md b/packages/rust/migrations.d/2026-10-16-prefix-glob-speed.md index a8aa8a1ea..87692fb83 100644 --- a/packages/rust/migrations.d/2026-10-16-prefix-glob-speed.md +++ b/packages/rust/migrations.d/2026-10-16-prefix-glob-speed.md @@ -2,7 +2,7 @@ #### Summary -One-shot CLI queries start fewer threads, and the walk uses half the cores on machines with more than sixteen. No API changes. +One-shot CLI queries start fewer threads, and the walk starts its workers as a tree. No API changes. #### Required changes diff --git a/packages/rust/src/tree_walk.rs b/packages/rust/src/tree_walk.rs index 3b1b5a1ab..5a54ed91f 100644 --- a/packages/rust/src/tree_walk.rs +++ b/packages/rust/src/tree_walk.rs @@ -158,8 +158,7 @@ pub(crate) fn walk_in_order( Some(Item::Dir(slot)) => { if !spawned && shared.found.load(Ordering::Relaxed) >= PARALLEL_DIRS { spawned = true; - let workers = - worker_count(thread::available_parallelism().map_or(1, usize::from)); + let workers = thread::available_parallelism().map_or(1, usize::from); spawn_worker(scope, &shared, 1, workers); } stack.push(shared.obtain(&slot).into_iter()); @@ -178,14 +177,6 @@ fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { mutex.lock().expect("exploring a directory does not panic") } -/// How many threads explore on a machine of `cores` logical cores: half, as -/// past the physical cores a worker mostly takes time from the walking thread -/// and the thread reading its rows; but every core up to eight, as a small -/// machine walks slower on fewer. -fn worker_count(cores: usize) -> usize { - (cores / 2).max(cores.min(8)) -} - /// Start worker `n` of `workers`, which starts its [`children`] before it /// explores: spawning a thread takes tens of microseconds, and the /// walk hands out nothing while it spawns. @@ -490,21 +481,6 @@ mod tests { assert!(threads_exploring(4, 2) > 1); } - #[test] - fn a_large_machine_explores_on_half_its_cores() { - assert_eq!(worker_count(32), 16); - } - - #[test] - fn a_small_machine_explores_on_every_core() { - assert_eq!([1, 4, 8].map(worker_count), [1, 4, 8]); - } - - #[test] - fn a_machine_explores_on_at_least_eight_cores() { - assert_eq!(worker_count(12), 8); - } - #[test] fn a_worker_starts_the_two_numbered_after_twice_its_own() { assert_eq!([1, 5].map(children), [2..4, 10..12]); From 184cd851c08ade75502c713af497523161f8e8a6 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Tue, 3 Nov 2026 08:45:48 -0500 Subject: [PATCH 4/8] e2e attestation for bot/1475-prefix-speed-1016 --- .../e2e-attestations/bot-1475-prefix-speed-1016.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/python/e2e-attestations/bot-1475-prefix-speed-1016.json diff --git a/packages/python/e2e-attestations/bot-1475-prefix-speed-1016.json b/packages/python/e2e-attestations/bot-1475-prefix-speed-1016.json new file mode 100644 index 000000000..8a8b73ca7 --- /dev/null +++ b/packages/python/e2e-attestations/bot-1475-prefix-speed-1016.json @@ -0,0 +1,7 @@ +{ + "command": "uv run python -m pytest tests/e2e/glob_speed_depth_bound_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_gitignore_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_node_modules_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_symlink_test.py tests/e2e/stat_speed_test.py tests/e2e/frontmatter_parse_speed_test.py tests/e2e/jsonl_parse_speed_test.py tests/e2e/query_contract_test.py tests/e2e/fanout_test.py tests/e2e/repl_interactive_test.py tests/e2e/server_speed_test.py -q", + "ran_at": 1793713548, + "exit_code": 0, + "commit": "91474ddfe5a00cd0a7ae154b1ea87c1ef59e26b1", + "branch": "bot/1475-prefix-speed-1016" +} From 6e7d75d8a1b1c17789104764fbcc2e134a310584 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Tue, 3 Nov 2026 08:46:49 -0500 Subject: [PATCH 5/8] e2e attestation for bot/1475-prefix-speed-1016 --- .../ts/e2e-attestations/bot-1475-prefix-speed-1016.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/ts/e2e-attestations/bot-1475-prefix-speed-1016.json diff --git a/packages/ts/e2e-attestations/bot-1475-prefix-speed-1016.json b/packages/ts/e2e-attestations/bot-1475-prefix-speed-1016.json new file mode 100644 index 000000000..0d3db0999 --- /dev/null +++ b/packages/ts/e2e-attestations/bot-1475-prefix-speed-1016.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1793713609, + "exit_code": 0, + "commit": "184cd851c08ade75502c713af497523161f8e8a6", + "branch": "bot/1475-prefix-speed-1016" +} From 1404c15d5c3baf22a495ea5a700fe036f74c300d Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:07:20 -0500 Subject: [PATCH 6/8] fix: bound the worker spawn tree so no mutant can fork without end --- packages/rust/src/tree_walk.rs | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/packages/rust/src/tree_walk.rs b/packages/rust/src/tree_walk.rs index 5a54ed91f..bc79c9f15 100644 --- a/packages/rust/src/tree_walk.rs +++ b/packages/rust/src/tree_walk.rs @@ -159,7 +159,7 @@ pub(crate) fn walk_in_order( if !spawned && shared.found.load(Ordering::Relaxed) >= PARALLEL_DIRS { spawned = true; let workers = thread::available_parallelism().map_or(1, usize::from); - spawn_worker(scope, &shared, 1, workers); + spawn_workers(scope, &shared, (1..workers).collect()); } stack.push(shared.obtain(&slot).into_iter()); } @@ -177,30 +177,29 @@ fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { mutex.lock().expect("exploring a directory does not panic") } -/// Start worker `n` of `workers`, which starts its [`children`] before it -/// explores: spawning a thread takes tens of microseconds, and the -/// walk hands out nothing while it spawns. -fn spawn_worker<'scope, D: Send, L: Send>( +/// Start a worker for each of `team` as a tree: each worker starts half of +/// the rest before it explores, since spawning a thread takes tens of +/// microseconds and the walk hands out nothing while it spawns. Every worker +/// starts a strictly smaller team than it was handed, so the tree is finite. +fn spawn_workers<'scope, D: Send, L: Send>( scope: &'scope thread::Scope<'scope, '_>, shared: &'scope Shared<'_, D, L>, - n: usize, - workers: usize, + mut team: Vec, ) { - if n >= workers { + if team.pop().is_none() { return; } + let (first, second) = halves(team); scope.spawn(move || { - for child in children(n) { - spawn_worker(scope, shared, child, workers); - } + spawn_workers(scope, shared, first); + spawn_workers(scope, shared, second); shared.work(); }); } -/// The workers worker `n` starts, numbered so that, from worker 1, each -/// number below the worker count is started exactly once. -fn children(n: usize) -> std::ops::Range { - 2 * n..2 * n + 2 +fn halves(mut team: Vec) -> (Vec, Vec) { + let second = team.split_off(team.len() / 2); + (team, second) } impl Shared<'_, D, L> { @@ -482,8 +481,8 @@ mod tests { } #[test] - fn a_worker_starts_the_two_numbered_after_twice_its_own() { - assert_eq!([1, 5].map(children), [2..4, 10..12]); + fn a_team_splits_into_halves_the_second_taking_any_odd_one() { + assert_eq!(halves(vec![1, 2, 3, 4, 5]), (vec![1, 2], vec![3, 4, 5])); } #[test] From 26b351ed743d0a9352f95f176f6e3563fb862159 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:46:34 -0500 Subject: [PATCH 7/8] e2e attestation for bot/1475-prefix-speed-1016 --- .../python/e2e-attestations/bot-1475-prefix-speed-1016.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/python/e2e-attestations/bot-1475-prefix-speed-1016.json b/packages/python/e2e-attestations/bot-1475-prefix-speed-1016.json index 8a8b73ca7..7f3dab9c8 100644 --- a/packages/python/e2e-attestations/bot-1475-prefix-speed-1016.json +++ b/packages/python/e2e-attestations/bot-1475-prefix-speed-1016.json @@ -1,7 +1,7 @@ { "command": "uv run python -m pytest tests/e2e/glob_speed_depth_bound_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_gitignore_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_node_modules_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_symlink_test.py tests/e2e/stat_speed_test.py tests/e2e/frontmatter_parse_speed_test.py tests/e2e/jsonl_parse_speed_test.py tests/e2e/query_contract_test.py tests/e2e/fanout_test.py tests/e2e/repl_interactive_test.py tests/e2e/server_speed_test.py -q", - "ran_at": 1793713548, + "ran_at": 1793727994, "exit_code": 0, - "commit": "91474ddfe5a00cd0a7ae154b1ea87c1ef59e26b1", + "commit": "1404c15d5c3baf22a495ea5a700fe036f74c300d", "branch": "bot/1475-prefix-speed-1016" } From a21ed75b8c6f289d65779c34992c2f83af5036c4 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:47:24 -0500 Subject: [PATCH 8/8] e2e attestation for bot/1475-prefix-speed-1016 --- packages/ts/e2e-attestations/bot-1475-prefix-speed-1016.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ts/e2e-attestations/bot-1475-prefix-speed-1016.json b/packages/ts/e2e-attestations/bot-1475-prefix-speed-1016.json index 0d3db0999..55e41f480 100644 --- a/packages/ts/e2e-attestations/bot-1475-prefix-speed-1016.json +++ b/packages/ts/e2e-attestations/bot-1475-prefix-speed-1016.json @@ -1,7 +1,7 @@ { "command": "pnpm test:e2e", - "ran_at": 1793713609, + "ran_at": 1793728044, "exit_code": 0, - "commit": "184cd851c08ade75502c713af497523161f8e8a6", + "commit": "26b351ed743d0a9352f95f176f6e3563fb862159", "branch": "bot/1475-prefix-speed-1016" }