From cf26e1b2785be100651a759a9b9eb5f3b3edd930 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Sun, 11 Oct 2026 20:21:18 -0400 Subject: [PATCH 1/5] test: a path row keeps its scanned allocation; big row sets free off thread --- packages/rust/src/vtab.rs | 11 +++++++ packages/rust/src/vtab_scaffold.rs | 48 ++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/packages/rust/src/vtab.rs b/packages/rust/src/vtab.rs index f551b8a63..7866812dc 100644 --- a/packages/rust/src/vtab.rs +++ b/packages/rust/src/vtab.rs @@ -665,6 +665,17 @@ mod tests { FileRow::new(Path::new(prefix), PathBuf::from(rel)) } + #[cfg(unix)] + #[test] + fn a_row_without_a_prefix_keeps_the_scanned_path_allocation() { + let rel = PathBuf::from("docs/a.md"); + let scanned = rel.as_os_str().as_encoded_bytes().as_ptr(); + + let row = FileRow::new(Path::new(""), rel); + + assert_eq!(row.path.as_ptr(), scanned); + } + fn no_facts(_: &FileRow) -> StatFacts { StatFacts::default() } diff --git a/packages/rust/src/vtab_scaffold.rs b/packages/rust/src/vtab_scaffold.rs index 5bc234dae..31f1d898f 100644 --- a/packages/rust/src/vtab_scaffold.rs +++ b/packages/rust/src/vtab_scaffold.rs @@ -996,6 +996,54 @@ mod tests { } } + /// A row that reports the thread it is freed on. + struct Noted(std::sync::mpsc::Sender); + + impl Drop for Noted { + fn drop(&mut self) { + let _ = self.0.send(thread::current().id()); + } + } + + struct Many { + freed_on: Mutex>, + } + + impl TableSource for Many { + type Row = Noted; + const NAME: &'static str = "many"; + + fn connect(_args: &[&[u8]]) -> Result<(String, Self)> { + unreachable!() + } + + fn rows(&self) -> Arc> { + let freed_on = self.freed_on.lock().unwrap(); + Arc::new((0..10_000).map(|_| Noted(freed_on.clone())).collect()) + } + + fn column(&self, _row: &Noted, _ctx: &mut Context, _i: c_int) -> Result<()> { + unreachable!() + } + } + + #[test] + fn a_large_evicted_row_set_is_freed_off_the_evicting_thread() { + let (freed_on, freed) = std::sync::mpsc::channel(); + let cache = RowCache::new(Arc::new(Many { + freed_on: Mutex::new(freed_on), + })); + drop(cache.rows()); + + cache.evict(); + + let here = thread::current().id(); + let timeout = std::time::Duration::from_secs(10); + for _ in 0..10_000 { + assert_ne!(freed.recv_timeout(timeout).unwrap(), here); + } + } + fn counted() -> RowCache { RowCache::new(Arc::new(Counted { scans: std::sync::atomic::AtomicUsize::new(0), From c883de0e1c0f9ac9ee2ca2cd74432b2436557f49 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Sun, 11 Oct 2026 20:25:32 -0400 Subject: [PATCH 2/5] test: filling a text cell reuses its buffer --- packages/rust/src/db.rs | 45 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/packages/rust/src/db.rs b/packages/rust/src/db.rs index ba2fff932..aeaab7c98 100644 --- a/packages/rust/src/db.rs +++ b/packages/rust/src/db.rs @@ -1151,10 +1151,9 @@ impl Db { let mut rows = stmt.query([])?; let mut cells = Vec::with_capacity(column_names.len()); while let Some(row) = rows.next()? { - cells.clear(); - for i in 0..column_names.len() { - let val: rusqlite::types::Value = row.get(i)?; - cells.push(Value::from(val)); + cells.resize(column_names.len(), Value::Null); + for (i, cell) in cells.iter_mut().enumerate() { + fill_cell(cell, row.get_ref(i)?); } on_row(&mut state, &cells); } @@ -1428,6 +1427,11 @@ pub(crate) fn table_catalog_entry( .optional() } +/// Set `cell` to what SQLite holds in `source`. +fn fill_cell(cell: &mut Value, source: ValueRef<'_>) { + *cell = Value::from(rusqlite::types::Value::from(source)); +} + impl From for Value { fn from(v: rusqlite::types::Value) -> Self { match v { @@ -1573,6 +1577,39 @@ mod tests { .unwrap() } + #[test] + fn filling_a_text_cell_with_text_reuses_its_buffer() { + let mut cell = Value::Text(String::with_capacity(64)); + let Value::Text(held) = &cell else { + unreachable!() + }; + let buffer = held.as_ptr(); + + fill_cell(&mut cell, ValueRef::Text(b"a.md")); + + let Value::Text(filled) = &cell else { + panic!("a text source fills a text cell, got {cell:?}") + }; + assert_eq!(filled, "a.md"); + assert_eq!(filled.as_ptr(), buffer); + } + + #[test] + fn filling_a_cell_takes_each_kind_of_source() { + let mut cell = Value::Text("old".into()); + + fill_cell(&mut cell, ValueRef::Integer(7)); + assert_eq!(cell, Value::Integer(7)); + fill_cell(&mut cell, ValueRef::Real(0.5)); + assert_eq!(cell, Value::Real(0.5)); + fill_cell(&mut cell, ValueRef::Blob(&[0, 255])); + assert_eq!(cell, Value::Blob(vec![0, 255])); + fill_cell(&mut cell, ValueRef::Text(b"t")); + assert_eq!(cell, Value::Text("t".into())); + fill_cell(&mut cell, ValueRef::Null); + assert_eq!(cell, Value::Null); + } + #[test] fn query_each_folds_each_row_in_projection_order() { let db = titled_docs(&["a", "b"]); From d6653e1f4d8a1d8b4b723b6f24c0bc805e4aa7e1 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Sun, 11 Oct 2026 20:36:45 -0400 Subject: [PATCH 3/5] perf: build a path row from the scanned allocation, reuse query cells, free big row sets off thread --- .../2026-10-11-faster-path-rows.md | 1 + .../2026-10-11-faster-path-rows.md | 21 +++++++++++ packages/rust/src/db.rs | 8 ++++- packages/rust/src/vtab.rs | 36 ++++++++++++++----- packages/rust/src/vtab_scaffold.rs | 12 ++++++- 5 files changed, 68 insertions(+), 10 deletions(-) create mode 100644 packages/rust/changelog.d/2026-10-11-faster-path-rows.md create mode 100644 packages/rust/migrations.d/2026-10-11-faster-path-rows.md diff --git a/packages/rust/changelog.d/2026-10-11-faster-path-rows.md b/packages/rust/changelog.d/2026-10-11-faster-path-rows.md new file mode 100644 index 000000000..7683f0531 --- /dev/null +++ b/packages/rust/changelog.d/2026-10-11-faster-path-rows.md @@ -0,0 +1 @@ +**Changed** Path queries over large trees return sooner: a matched file's path is no longer copied into its row, query cells reuse their buffers, and a large result's rows are freed in the background. diff --git a/packages/rust/migrations.d/2026-10-11-faster-path-rows.md b/packages/rust/migrations.d/2026-10-11-faster-path-rows.md new file mode 100644 index 000000000..6b1836e46 --- /dev/null +++ b/packages/rust/migrations.d/2026-10-11-faster-path-rows.md @@ -0,0 +1,21 @@ +### Faster path rows + +#### Summary + +Path queries over large trees build, read and free their rows with fewer allocations. Results and ordering are unchanged; nothing breaks. + +#### Required changes + +_None._ + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +_None._ + +#### Verification + +Run `dirsql --help`; the CLI starts as before. diff --git a/packages/rust/src/db.rs b/packages/rust/src/db.rs index aeaab7c98..d3436a491 100644 --- a/packages/rust/src/db.rs +++ b/packages/rust/src/db.rs @@ -1429,7 +1429,13 @@ pub(crate) fn table_catalog_entry( /// Set `cell` to what SQLite holds in `source`. fn fill_cell(cell: &mut Value, source: ValueRef<'_>) { - *cell = Value::from(rusqlite::types::Value::from(source)); + match (&mut *cell, source) { + (Value::Text(held), ValueRef::Text(bytes)) => { + held.clear(); + held.push_str(&String::from_utf8_lossy(bytes)); + } + _ => *cell = Value::from(rusqlite::types::Value::from(source)), + } } impl From for Value { diff --git a/packages/rust/src/vtab.rs b/packages/rust/src/vtab.rs index 7866812dc..93e4df151 100644 --- a/packages/rust/src/vtab.rs +++ b/packages/rust/src/vtab.rs @@ -189,8 +189,8 @@ struct ScanSpec { /// path-derived text columns are slices of `path` located by [`PathSpans`], /// so a row costs two allocations however many columns a query reads. struct FileRow { - /// The path as scanned, root-relative; where the lazy content read goes. - rel: PathBuf, + /// The path as scanned, root-relative, when `path` is not already it. + rel: Option, /// The path as reported, under the table's prefix when it has one. path: String, spans: PathSpans, @@ -205,7 +205,13 @@ struct FileRow { impl FileRow { fn new(path_prefix: &Path, rel: PathBuf) -> Self { - let path = reported_path(path_prefix, &rel); + let (rel, path) = match reported_in_place(path_prefix, rel) { + Ok(path) => (None, path), + Err(rel) => { + let path = reported_path(path_prefix, &rel); + (Some(rel), path) + } + }; let spans = PathSpans::of(&path); Self { rel, @@ -216,6 +222,11 @@ impl FileRow { } } + /// The path as scanned, root-relative; where the lazy content read goes. + fn rel(&self) -> &Path { + self.rel.as_deref().unwrap_or_else(|| Path::new(&self.path)) + } + fn basename(&self) -> Option<&str> { self.spans.name_at.map(|at| &self.path[at as usize..]) } @@ -367,7 +378,7 @@ fn parse_module_args(args: &[&[u8]]) -> Result { impl ScanSpec { fn stat(&self, row: &FileRow) -> StatFacts { - fs::metadata(self.root.join(&row.rel)).map_or_else( + fs::metadata(self.root.join(row.rel())).map_or_else( |_| StatFacts::default(), |m| StatFacts::from_parts(m.len(), m.modified().ok(), m.created().ok()), ) @@ -388,7 +399,7 @@ impl ScanSpec { } fn read(&self, row: &FileRow) -> Option { - (self.reader)(&self.root.join(&row.rel), self.facts(row).size) + (self.reader)(&self.root.join(row.rel()), self.facts(row).size) } } @@ -406,6 +417,15 @@ fn reported_path(path_prefix: &Path, rel_path: &Path) -> String { format!("{prefix}/{rel_path}") } +/// The reported path taken from `rel` itself, when it is already what a +/// table with no prefix reports: handing `rel` back otherwise. +fn reported_in_place(path_prefix: &Path, rel: PathBuf) -> Result { + if !path_prefix.as_os_str().is_empty() || std::path::MAIN_SEPARATOR != '/' { + return Err(rel); + } + rel.into_os_string().into_string().map_err(PathBuf::from) +} + impl TableSource for ScanSpec { type Row = FileRow; @@ -850,7 +870,7 @@ mod tests { assert_eq!(rows.len(), 100); for (n, row) in rows.iter().enumerate() { assert_eq!(row.path, format!("/root/f{n:03}.md")); - assert_eq!(row.rel, PathBuf::from(format!("f{n:03}.md"))); + assert_eq!(row.rel(), Path::new(&format!("f{n:03}.md"))); } } @@ -1087,8 +1107,8 @@ mod tests { assert_eq!(row.dir(), Some("/var/log")); assert_eq!(row.basename(), Some("a.log")); assert_eq!( - row.rel, - PathBuf::from("a.log"), + row.rel(), + Path::new("a.log"), "content is read from the scanned path" ); } diff --git a/packages/rust/src/vtab_scaffold.rs b/packages/rust/src/vtab_scaffold.rs index 31f1d898f..e3aca5238 100644 --- a/packages/rust/src/vtab_scaffold.rs +++ b/packages/rust/src/vtab_scaffold.rs @@ -42,6 +42,11 @@ const PREFETCH_LOOKUP_COST: f64 = 100.; /// `idxNum` for a full scan; a lookup carries its column as `column + 1`. const SCAN_IDX: c_int = 0; +/// Rows a statement's row set holds before its eviction frees them on a +/// thread of their own: below it, spawning costs more than the freeing, and +/// above it the statement need not wait on the freeing. +const FREE_ELSEWHERE_ROWS: usize = 4096; + /// Set in `idxNum` when the statement names the prefetch column; the low bits /// still name the lookup. const PREFETCH_IDX: c_int = 1 << 30; @@ -217,7 +222,12 @@ impl RowCache { impl Held for RowCache { fn evict(&self) { - *self.slot.lock().unwrap() = Slot::Empty; + let evicted = std::mem::replace(&mut *self.slot.lock().unwrap(), Slot::Empty); + if let Slot::Ready(rows) = evicted + && rows.len() >= FREE_ELSEWHERE_ROWS + { + let _ = thread::Builder::new().spawn(move || drop(rows)); + } } fn warm(&self) { From 71d115d6a0a35181bff17f3c42b7b96fe5139371 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:23:51 -0400 Subject: [PATCH 4/5] e2e attestation for bot/speed-walk-rowpath-1011 --- .../e2e-attestations/bot-speed-walk-rowpath-1011.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/python/e2e-attestations/bot-speed-walk-rowpath-1011.json diff --git a/packages/python/e2e-attestations/bot-speed-walk-rowpath-1011.json b/packages/python/e2e-attestations/bot-speed-walk-rowpath-1011.json new file mode 100644 index 000000000..f689395c3 --- /dev/null +++ b/packages/python/e2e-attestations/bot-speed-walk-rowpath-1011.json @@ -0,0 +1,7 @@ +{ + "command": "uv run python -m pytest tests/e2e/glob_speed_depth_bound_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_symlink_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_node_modules_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_gitignore_test.py tests/e2e/stat_speed_test.py tests/e2e/query_contract_test.py -x -q", + "ran_at": 1791768231, + "exit_code": 0, + "commit": "d6653e1f4d8a1d8b4b723b6f24c0bc805e4aa7e1", + "branch": "bot/speed-walk-rowpath-1011" +} From ad21cc97385ebc6b461be297008a55ba45fd8383 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:25:01 -0400 Subject: [PATCH 5/5] e2e attestation for bot/speed-walk-rowpath-1011 --- .../ts/e2e-attestations/bot-speed-walk-rowpath-1011.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/ts/e2e-attestations/bot-speed-walk-rowpath-1011.json diff --git a/packages/ts/e2e-attestations/bot-speed-walk-rowpath-1011.json b/packages/ts/e2e-attestations/bot-speed-walk-rowpath-1011.json new file mode 100644 index 000000000..421e8fc7c --- /dev/null +++ b/packages/ts/e2e-attestations/bot-speed-walk-rowpath-1011.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1791768301, + "exit_code": 0, + "commit": "71d115d6a0a35181bff17f3c42b7b96fe5139371", + "branch": "bot/speed-walk-rowpath-1011" +}