From 8c8f56c6d13e74cf8c9875a9cf5090f25300210b Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:44:50 -0400 Subject: [PATCH 1/5] test(rust): assert two path-tables in one statement walk their trees at the same time --- .../rust/tests/path_table_parallel_scan.rs | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 packages/rust/tests/path_table_parallel_scan.rs diff --git a/packages/rust/tests/path_table_parallel_scan.rs b/packages/rust/tests/path_table_parallel_scan.rs new file mode 100644 index 000000000..fe431ee80 --- /dev/null +++ b/packages/rust/tests/path_table_parallel_scan.rs @@ -0,0 +1,70 @@ +//! Two path-tables named by one statement walk their trees at the same time, +//! the way a shell pipeline runs two `find`s at once. Real filesystem, real +//! SQLite, SDK public API. Its own binary, so no sibling test competes for +//! the cores the overlap needs. + +use std::fs; +use std::path::Path; +use std::time::{Duration, Instant}; + +use dirsql::DirSQL; +use tempfile::TempDir; + +const DIRS: usize = 50; +const PATTERNS: usize = 100; +const ROUNDS: usize = 4; + +/// A tree whose walk costs CPU rather than disk: every directory carries a +/// `.gitignore` the walker compiles on entry, so a scan takes the same time +/// whether or not the page cache already holds the tree. +fn tree(root: &Path, name: &str) { + let ignore: String = (0..PATTERNS) + .map(|i| format!("build-{i}/**/*.tmp\n")) + .collect(); + for i in 0..DIRS { + let dir = root.join(name).join(format!("d{i:05}")); + fs::create_dir_all(&dir).unwrap(); + fs::write(dir.join(".gitignore"), &ignore).unwrap(); + fs::write(dir.join("f.txt"), "x").unwrap(); + } +} + +fn timed(db: &DirSQL, sql: &str) -> Duration { + let started = Instant::now(); + db.query(sql).unwrap(); + started.elapsed() +} + +/// The fastest of [`ROUNDS`] interleaved runs of each statement: the floor +/// is the honest cost, and a busy box only ever adds to it. +fn floors(db: &DirSQL) -> (Duration, Duration, Duration) { + let mut a = Duration::MAX; + let mut b = Duration::MAX; + let mut both = Duration::MAX; + for _ in 0..ROUNDS { + a = a.min(timed(db, "SELECT count(*) FROM './a/**/*.txt'")); + b = b.min(timed(db, "SELECT count(*) FROM './b/**/*.txt'")); + both = both.min(timed( + db, + "SELECT count(*) FROM './a/**/*.txt' x JOIN './b/**/*.txt' y ON y.path = x.path", + )); + } + (a, b, both) +} + +#[test] +fn two_path_tables_in_one_statement_walk_their_trees_at_the_same_time() { + let root = TempDir::new().unwrap(); + tree(root.path(), "a"); + tree(root.path(), "b"); + let db = DirSQL::new(root.path(), Vec::new()).unwrap(); + + let (a, b, both) = floors(&db); + + let serial = a + b; + assert!( + both < serial.mul_f64(0.75), + "a {a:?} + b {b:?} = {serial:?} walked one after the other; \ + the statement over both took {both:?}, so the walks did not overlap" + ); +} From d7911d127a381427ad7339799691595fd603fb19 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 2 Oct 2026 14:59:53 -0400 Subject: [PATCH 2/5] perf(rust): walk every path-table a statement names at the same time Each path-table's scan ran inside its first xFilter, and SQLite opens one cursor at a time, so a statement over two tables walked the second tree after the first. The authorizer at the final prepare now names the tables the statement reads, and the statement scope starts each one's scan on a thread of its own before SQLite asks for any; the first read joins it. --- PARITY.md | 8 + .../python/tests/e2e/use_case_join_test.py | 4 - .../2026-10-02-path-table-parallel-walk.md | 1 + packages/rust/src/db.rs | 45 ++- packages/rust/src/vtab_scaffold.rs | 319 ++++++++++++++---- 5 files changed, 292 insertions(+), 85 deletions(-) create mode 100644 packages/rust/changelog.d/2026-10-02-path-table-parallel-walk.md diff --git a/PARITY.md b/PARITY.md index dd53c90e8..54ab4a575 100644 --- a/PARITY.md +++ b/PARITY.md @@ -296,6 +296,14 @@ per-binding surface. The one signature change is Rust-only: `vtab::load_module` and `parsed_vtab::load_module` take the `Arc` their tables cache under, a surface neither binding exposes. +**Path tables of one statement walked at the same time — Rust-crate-only +surface, no SDK drift.** A statement that names several path tables starts +every walk before SQLite opens the first cursor, each on a thread of its own, +so a join over two globs costs about one walk instead of one per table. The +names come from the SQLite authorizer at the final prepare, so the fallback +loop above stays the only thing that registers a table. `StatementScope::warm` +lives in the shared core; no SDK signature changes. + ## AsyncDirSQL | API | Python | Rust | diff --git a/packages/python/tests/e2e/use_case_join_test.py b/packages/python/tests/e2e/use_case_join_test.py index 345035f75..2698878b0 100644 --- a/packages/python/tests/e2e/use_case_join_test.py +++ b/packages/python/tests/e2e/use_case_join_test.py @@ -82,10 +82,6 @@ def root(tmp_path): finally: shutil.rmtree(tree, ignore_errors=True) - @pytest.mark.xfail( - strict=True, - reason="two serial tree walks run over the speed-of-light bar", - ) def it_matches_native_rows_within_the_bar(root): startup = startup_seconds() diff --git a/packages/rust/changelog.d/2026-10-02-path-table-parallel-walk.md b/packages/rust/changelog.d/2026-10-02-path-table-parallel-walk.md new file mode 100644 index 000000000..dc35dabd1 --- /dev/null +++ b/packages/rust/changelog.d/2026-10-02-path-table-parallel-walk.md @@ -0,0 +1 @@ +**Changed** A statement that names several path-tables walks their trees at the same time, each on a thread of its own, instead of one after the other as SQLite opens their cursors. A join between two globs over one corpus now costs about one walk. diff --git a/packages/rust/src/db.rs b/packages/rust/src/db.rs index ae15ff7ec..351738bc3 100644 --- a/packages/rust/src/db.rs +++ b/packages/rust/src/db.rs @@ -803,9 +803,9 @@ impl Db { // error stands. That is what bounds the loop. let mut attempted: HashSet = HashSet::new(); - let mut stmt = loop { + let (mut stmt, reads) = loop { let error = match self.prepare_guarded(sql) { - Ok(stmt) => break stmt, + Ok(prepared) => break prepared, Err(e) => e, }; let Some((name, table)) = self.path_table_for(&error)? else { @@ -820,6 +820,7 @@ impl Db { if !stmt.readonly() { return Err(DbError::WriteForbidden); } + self.scope.warm(&reads); let column_names: Vec = stmt.column_names().iter().map(|s| s.to_string()).collect(); let rows = stmt.query_map([], |row| { @@ -842,10 +843,10 @@ impl Db { } /// Prepare `sql` with the internal-table / ATTACH authorizer installed for - /// exactly that one call. The authorizer is installed and cleared around - /// every attempt, so a re-prepare after registering a path-table is gated - /// identically to the first. - fn prepare_guarded(&self, sql: &str) -> Result> { + /// exactly that one call, and name the tables it reads. The authorizer is + /// installed and cleared around every attempt, so a re-prepare after + /// registering a path-table is gated identically to the first. + fn prepare_guarded(&self, sql: &str) -> Result<(rusqlite::Statement<'_>, HashSet)> { use rusqlite::hooks::{AuthAction, AuthContext, Authorization}; use std::sync::{Arc, Mutex}; @@ -854,12 +855,18 @@ impl Db { // matching message (the closure must be `'static`, so it owns a clone). let denial: Arc>> = Arc::new(Mutex::new(None)); let denial_cb = Arc::clone(&denial); + let reads: Arc>> = Arc::default(); + let reads_cb = Arc::clone(&reads); self.conn .authorizer(Some(move |ctx: AuthContext<'_>| match ctx.action { AuthAction::Read { table_name, .. } if is_internal_table(table_name) => { *denial_cb.lock().unwrap() = Some(INTERNAL_TABLE_DENIED_MSG); Authorization::Deny } + AuthAction::Read { table_name, .. } => { + reads_cb.lock().unwrap().insert(table_name.to_owned()); + Authorization::Allow + } AuthAction::Pragma { pragma_value: Some(value), .. @@ -883,7 +890,7 @@ impl Db { .authorizer(None::) -> Authorization>); match prepared { - Ok(stmt) => Ok(stmt), + Ok(stmt) => Ok((stmt, std::mem::take(&mut *reads.lock().unwrap()))), Err(e) if e.sqlite_error_code() == Some(rusqlite::ErrorCode::AuthorizationForStatementDenied) => @@ -1609,6 +1616,30 @@ mod tests { ); } + #[test] + fn prepare_guarded_names_the_tables_a_statement_reads() { + let db = Db::new().unwrap(); + db.conn + .execute_batch("CREATE TABLE a(x); CREATE TABLE b(y)") + .unwrap(); + + let (_, reads) = db + .prepare_guarded("SELECT count(*) FROM a JOIN b ON b.y = a.x") + .unwrap(); + + assert_eq!(reads, HashSet::from(["a".to_string(), "b".to_string()])); + } + + #[test] + fn prepare_guarded_names_a_table_the_statement_takes_no_column_from() { + let db = Db::new().unwrap(); + db.conn.execute_batch("CREATE TABLE a(x)").unwrap(); + + let (_, reads) = db.prepare_guarded("SELECT count(*) FROM a").unwrap(); + + assert_eq!(reads, HashSet::from(["a".to_string()])); + } + #[test] fn internal_table_denied_message_mentions_the_namespace() { assert!(INTERNAL_TABLE_DENIED_MSG.contains("not authorized")); diff --git a/packages/rust/src/vtab_scaffold.rs b/packages/rust/src/vtab_scaffold.rs index 4a59cf2e6..5336ce01c 100644 --- a/packages/rust/src/vtab_scaffold.rs +++ b/packages/rust/src/vtab_scaffold.rs @@ -7,9 +7,10 @@ //! set, which columns answer an equality by lookup, and how a row becomes a //! cell. Everything either one would otherwise write twice lives here. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::ffi::c_int; use std::sync::{Arc, Mutex, Weak}; +use std::thread::{self, JoinHandle}; use globset::GlobSet; use rusqlite::types::ValueRef; @@ -36,7 +37,7 @@ const LOOKUP_COST: f64 = 10.; const SCAN_IDX: c_int = 0; /// What a `dirsql_*` table supplies on top of the scaffolding. -pub trait TableSource: Sized + 'static { +pub trait TableSource: Sized + Send + Sync + 'static { /// One row, as the table models it. type Row: Send + Sync + 'static; @@ -79,9 +80,14 @@ pub trait TableSource: Sized + 'static { /// to the rest. What ends a statement is [`reset`](Self::reset): the owner of /// the connection brackets each statement with it, so the next statement /// scans afresh and the rows do not outlive the statement that read them. +/// +/// SQLite also opens those cursors one at a time, so left to itself a +/// statement over two tables walks the second tree only after the first is +/// done. [`warm`](Self::warm) starts the scans of the tables a statement +/// names before SQLite asks for any of them, each on a thread of its own. #[derive(Default)] pub struct StatementScope { - caches: Mutex>>, + tables: Mutex)>>, } impl StatementScope { @@ -99,8 +105,8 @@ impl StatementScope { /// Drop every row set a table under this scope is holding, so its next /// read scans afresh. pub fn reset(&self) { - let mut caches = self.caches.lock().unwrap(); - caches.retain(|cache| { + let mut tables = self.tables.lock().unwrap(); + tables.retain(|(_, cache)| { cache.upgrade().is_some_and(|cache| { cache.evict(); true @@ -108,8 +114,21 @@ impl StatementScope { }); } - fn adopt(&self, cache: Weak) { - self.caches.lock().unwrap().push(cache); + /// Start scanning every table named in `names` now, so the statement + /// about to read them finds their rows underway. A name no table here + /// answers to is passed over; a table already holding rows keeps them. + pub fn warm(&self, names: &HashSet) { + for (name, cache) in self.tables.lock().unwrap().iter() { + if names.contains(name) + && let Some(cache) = cache.upgrade() + { + cache.warm(); + } + } + } + + fn adopt(&self, name: String, cache: Weak) { + self.tables.lock().unwrap().push((name, cache)); } } @@ -121,41 +140,69 @@ impl Drop for StatementGuard { } } -trait Evict { +trait Held { fn evict(&self); + fn warm(&self); +} + +/// Where one table's rows stand within the statement in progress. +enum Slot { + Empty, + /// A scan started ahead of the first read, on a thread of its own. + Pending(JoinHandle>>), + Ready(Arc>), } /// One table's rows for the statement in progress, empty between statements. -struct RowCache { - rows: Mutex>>>, +struct RowCache { + source: Arc, + slot: Mutex>, } -impl Default for RowCache { - fn default() -> Self { +impl RowCache { + fn new(source: Arc) -> Self { Self { - rows: Mutex::new(None), + source, + slot: Mutex::new(Slot::Empty), } } -} -impl RowCache { - /// The rows already held, or the ones `scan` produces, held from now on. - fn rows_or_else(&self, scan: impl FnOnce() -> Arc>) -> Arc> { - let mut slot = self.rows.lock().unwrap(); - match &*slot { - Some(rows) => Arc::clone(rows), - None => { - let rows = scan(); - *slot = Some(Arc::clone(&rows)); - rows - } + /// The rows held, waiting for a scan started ahead to finish; or else the + /// source's rows scanned now, held from here on. + fn rows(&self) -> Arc> { + let mut slot = self.slot.lock().unwrap(); + let rows = match std::mem::replace(&mut *slot, Slot::Empty) { + Slot::Ready(rows) => rows, + Slot::Pending(scan) => scan + .join() + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)), + Slot::Empty => self.source.rows(), + }; + *slot = Slot::Ready(Arc::clone(&rows)); + rows + } + + /// Start the scan on a thread of its own, unless rows are already held + /// or underway. + fn warm(&self) { + let mut slot = self.slot.lock().unwrap(); + if !matches!(*slot, Slot::Empty) { + return; + } + let source = Arc::clone(&self.source); + if let Ok(scan) = thread::Builder::new().spawn(move || source.rows()) { + *slot = Slot::Pending(scan); } } } -impl Evict for RowCache { +impl Held for RowCache { fn evict(&self) { - *self.rows.lock().unwrap() = None; + *self.slot.lock().unwrap() = Slot::Empty; + } + + fn warm(&self) { + self.warm(); } } @@ -174,6 +221,14 @@ pub fn user_args(args: &[&[u8]]) -> Vec { .collect() } +/// The table's name, the third argument SQLite prepends: the same spelling +/// its authorizer reports a read of the table under. +fn table_name(args: &[&[u8]]) -> String { + args.get(2) + .map(|name| String::from_utf8_lossy(name).into_owned()) + .unwrap_or_default() +} + /// The error a module raises when it was handed too few arguments. Named /// arity plus the argument list, so SQLite reports what to write against the /// offending `CREATE VIRTUAL TABLE`. @@ -319,7 +374,7 @@ pub struct ScaffoldTab { /// Base class. Must be first. base: ffi::sqlite3_vtab, source: Arc, - cache: Arc>, + cache: Arc>, } #[expect(unsafe_code, reason = "rusqlite requires an unsafe trait impl")] @@ -334,11 +389,15 @@ unsafe impl<'vtab, S: TableSource> VTab<'vtab> for ScaffoldTab { ) -> Result<(String, Self)> { let scope = aux.ok_or_else(|| no_scope_error(S::NAME))?; let (schema, source) = S::connect(args)?; - let cache = Arc::new(RowCache::default()); - scope.adopt(Arc::downgrade(&cache) as Weak); + let source = Arc::new(source); + let cache = Arc::new(RowCache::new(Arc::clone(&source))); + scope.adopt( + table_name(args), + Arc::downgrade(&cache) as Weak, + ); let vtab = Self { base: ffi::sqlite3_vtab::default(), - source: Arc::new(source), + source, cache, }; Ok((schema, vtab)) @@ -392,7 +451,7 @@ pub struct ScaffoldCursor { /// overwritten. base: ffi::sqlite3_vtab_cursor, source: Arc, - cache: Arc>, + cache: Arc>, /// Taken from the statement's cache on the first `xFilter`; `None` until /// then. rows: Option>>, @@ -405,11 +464,7 @@ pub struct ScaffoldCursor { impl ScaffoldCursor { fn rows(&mut self) -> Arc> { let cache = &self.cache; - let source = &self.source; - Arc::clone( - self.rows - .get_or_insert_with(|| cache.rows_or_else(|| source.rows())), - ) + Arc::clone(self.rows.get_or_insert_with(|| cache.rows())) } fn select_by_lookup(&mut self, column: usize, key: &str) -> Selection { @@ -784,74 +839,189 @@ mod tests { assert_eq!(rowid_of(usize::MAX), i64::MAX); } + /// A source whose rows are the number of scans it has served, so a test + /// can tell a fresh scan from rows held over. + struct Counted { + scans: std::sync::atomic::AtomicUsize, + } + + impl TableSource for Counted { + type Row = usize; + const NAME: &'static str = "counted"; + + fn connect(_args: &[&[u8]]) -> Result<(String, Self)> { + unreachable!() + } + + fn rows(&self) -> Arc> { + let scan = self + .scans + .fetch_add(1, std::sync::atomic::Ordering::Relaxed) + + 1; + Arc::new(vec![scan]) + } + + fn column(&self, _row: &usize, _ctx: &mut Context, _i: c_int) -> Result<()> { + unreachable!() + } + } + + fn counted() -> RowCache { + RowCache::new(Arc::new(Counted { + scans: std::sync::atomic::AtomicUsize::new(0), + })) + } + + fn scans(cache: &RowCache) -> usize { + cache + .source + .scans + .load(std::sync::atomic::Ordering::Relaxed) + } + + fn is_pending(cache: &RowCache) -> bool { + matches!(*cache.slot.lock().unwrap(), Slot::Pending(_)) + } + #[test] fn a_row_cache_scans_once_and_serves_the_same_rows_after() { - let cache: RowCache = RowCache::default(); - let mut scans = 0; - let first = cache.rows_or_else(|| { - scans += 1; - Arc::new(vec![1, 2]) - }); - let second = cache.rows_or_else(|| { - scans += 1; - Arc::new(Vec::new()) - }); - assert_eq!(scans, 1); + let cache = counted(); + let first = cache.rows(); + let second = cache.rows(); + assert_eq!(scans(&cache), 1); assert!(Arc::ptr_eq(&first, &second)); } #[test] fn an_evicted_row_cache_scans_again() { - let cache: RowCache = RowCache::default(); - cache.rows_or_else(|| Arc::new(vec![1])); + let cache = counted(); + cache.rows(); cache.evict(); - let rows = cache.rows_or_else(|| Arc::new(vec![2, 3])); - assert_eq!(*rows, vec![2, 3]); + assert_eq!(*cache.rows(), vec![2]); + } + + #[test] + fn a_warmed_row_cache_has_its_scan_underway_before_any_read() { + let cache = counted(); + cache.warm(); + assert!(is_pending(&cache)); + assert_eq!(*cache.rows(), vec![1]); + assert_eq!(scans(&cache), 1); + } + + #[test] + fn warming_twice_starts_one_scan() { + let cache = counted(); + cache.warm(); + cache.warm(); + assert_eq!(*cache.rows(), vec![1]); + assert_eq!(scans(&cache), 1); } - fn adopted(scope: &StatementScope) -> Arc> { - let cache = Arc::new(RowCache::default()); - scope.adopt(Arc::downgrade(&cache) as Weak); + #[test] + fn warming_a_row_cache_holding_rows_keeps_them() { + let cache = counted(); + let held = cache.rows(); + cache.warm(); + assert!(!is_pending(&cache)); + assert!(Arc::ptr_eq(&held, &cache.rows())); + assert_eq!(scans(&cache), 1); + } + + #[test] + fn a_read_after_a_warmed_scan_serves_that_scan_again() { + let cache = counted(); + cache.warm(); + let first = cache.rows(); + assert!(Arc::ptr_eq(&first, &cache.rows())); + } + + fn adopted(scope: &StatementScope, name: &str) -> Arc> { + let cache = Arc::new(counted()); + scope.adopt( + name.to_string(), + Arc::downgrade(&cache) as Weak, + ); cache } + fn names(names: &[&str]) -> HashSet { + names.iter().map(|name| name.to_string()).collect() + } + #[test] fn a_scope_reset_evicts_every_cache_it_adopted() { let scope = StatementScope::new(); - let a = adopted(&scope); - let b = adopted(&scope); - a.rows_or_else(|| Arc::new(vec![1])); - b.rows_or_else(|| Arc::new(vec![2])); + let a = adopted(&scope, "a"); + let b = adopted(&scope, "b"); + a.rows(); + b.rows(); scope.reset(); - assert_eq!(*a.rows_or_else(|| Arc::new(vec![9])), vec![9]); - assert_eq!(*b.rows_or_else(|| Arc::new(vec![9])), vec![9]); + assert_eq!(*a.rows(), vec![2]); + assert_eq!(*b.rows(), vec![2]); } #[test] fn a_scope_forgets_a_cache_whose_table_is_gone() { let scope = StatementScope::new(); - let gone = adopted(&scope); - let kept = adopted(&scope); + let gone = adopted(&scope, "gone"); + let kept = adopted(&scope, "kept"); drop(gone); scope.reset(); - assert_eq!(scope.caches.lock().unwrap().len(), 1); + assert_eq!(scope.tables.lock().unwrap().len(), 1); assert!(Arc::weak_count(&kept) == 1); } + #[test] + fn a_scope_warms_the_tables_named_and_no_other() { + let scope = StatementScope::new(); + let named = adopted(&scope, "named"); + let other = adopted(&scope, "other"); + + scope.warm(&names(&["named", "absent"])); + + assert!(is_pending(&named)); + assert!(!is_pending(&other)); + assert_eq!(scans(&other), 0); + } + + #[test] + fn a_scope_warms_a_table_by_its_own_name_only() { + let scope = StatementScope::new(); + let table = adopted(&scope, "./a"); + + scope.warm(&names(&["./b"])); + + assert!(!is_pending(&table)); + } + #[test] fn entering_a_scope_resets_on_entry_and_on_exit() { let scope = StatementScope::new(); - let cache = adopted(&scope); - cache.rows_or_else(|| Arc::new(vec![1])); + let cache = adopted(&scope, "t"); + cache.rows(); let guard = scope.enter(); - assert_eq!(*cache.rows_or_else(|| Arc::new(vec![2])), vec![2]); + assert_eq!(*cache.rows(), vec![2]); drop(guard); - assert_eq!(*cache.rows_or_else(|| Arc::new(vec![3])), vec![3]); + assert_eq!(*cache.rows(), vec![3]); + } + + #[test] + fn table_name_is_the_third_argument_sqlite_prepends() { + assert_eq!( + table_name(&[b"dirsql_path", b"temp", b"./a", b"'/root'"]), + "./a" + ); + } + + #[test] + fn table_name_is_empty_when_sqlite_prepends_too_few() { + assert_eq!(table_name(&[b"dirsql_path"]), ""); } struct FakeSource; @@ -923,14 +1093,15 @@ mod tests { } fn cursor_over(words: Vec<&'static str>) -> ScaffoldCursor { + let source = Arc::new(Words { + words, + scans: std::sync::atomic::AtomicUsize::new(0), + keys: std::sync::atomic::AtomicUsize::new(0), + }); ScaffoldCursor { base: ffi::sqlite3_vtab_cursor::default(), - source: Arc::new(Words { - words, - scans: std::sync::atomic::AtomicUsize::new(0), - keys: std::sync::atomic::AtomicUsize::new(0), - }), - cache: Arc::new(RowCache::default()), + cache: Arc::new(RowCache::new(Arc::clone(&source))), + source, rows: None, index: None, selection: Selection::Every(0), From e7987c66259acd23c0c1e23b0aba965b97201992 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:26:39 -0400 Subject: [PATCH 3/5] e2e attestation for claude/1238-parallel-walk --- .../ts/e2e-attestations/claude-1238-parallel-walk.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/ts/e2e-attestations/claude-1238-parallel-walk.json diff --git a/packages/ts/e2e-attestations/claude-1238-parallel-walk.json b/packages/ts/e2e-attestations/claude-1238-parallel-walk.json new file mode 100644 index 000000000..0eeb18842 --- /dev/null +++ b/packages/ts/e2e-attestations/claude-1238-parallel-walk.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1790969199, + "exit_code": 0, + "commit": "d7911d127a381427ad7339799691595fd603fb19", + "branch": "claude/1238-parallel-walk" +} From 61191ed4a80fcef5da9648a75290f34194dd97d5 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:44:29 -0400 Subject: [PATCH 4/5] e2e attestation for claude/1238-parallel-walk --- .../python/e2e-attestations/claude-1238-parallel-walk.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/python/e2e-attestations/claude-1238-parallel-walk.json diff --git a/packages/python/e2e-attestations/claude-1238-parallel-walk.json b/packages/python/e2e-attestations/claude-1238-parallel-walk.json new file mode 100644 index 000000000..441508ba6 --- /dev/null +++ b/packages/python/e2e-attestations/claude-1238-parallel-walk.json @@ -0,0 +1,7 @@ +{ + "command": "just test-e2e", + "ran_at": 1790970269, + "exit_code": 0, + "commit": "e7987c66259acd23c0c1e23b0aba965b97201992", + "branch": "claude/1238-parallel-walk" +} From 378aed23eb003b31f8a565ef1ae062d88c26b227 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:57:33 -0400 Subject: [PATCH 5/5] docs(rust): migrations fragment for the parallel path-table walk --- .../2026-10-02-path-table-parallel-walk.md | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 packages/rust/migrations.d/2026-10-02-path-table-parallel-walk.md diff --git a/packages/rust/migrations.d/2026-10-02-path-table-parallel-walk.md b/packages/rust/migrations.d/2026-10-02-path-table-parallel-walk.md new file mode 100644 index 000000000..4f6aba98b --- /dev/null +++ b/packages/rust/migrations.d/2026-10-02-path-table-parallel-walk.md @@ -0,0 +1,41 @@ +### Core: the path tables of one statement are walked at the same time + +#### Summary + +A statement that names several path tables used to walk their trees one +after the other, each walk running inside SQLite's first `xFilter` on that +table's cursor. The walks now start together when the statement is prepared, +each on a thread of its own, and the first cursor over a table joins its +walk. Nothing in the API changed: no signature, flag, config key, exit code +or row shape differs. Rust callers, the CLI and the Python and TypeScript +SDKs all pick this up without edits. + +#### Required changes + +_None._ + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +- Every path table a statement reads is walked as soon as the statement is + prepared, even one whose cursor SQLite ends up never opening (the inner + side of a join whose outer side turns out empty). Before, such a table cost + nothing; now it costs a walk on a background thread. The rows are the same + either way. +- A statement over several path tables uses as many threads as it names + tables while they walk, so it runs on more than one core at once. + +#### Verification + +```bash +cd "$(mktemp -d)" +mkdir -p papers/one papers/two +printf 'T1' > papers/one/title.md +printf 'A1' > papers/one/abstract.md +printf 'T2' > papers/two/title.md +dirsql query "SELECT a.dir, t.content AS title FROM './papers/*/abstract.md' a JOIN './papers/*/title.md' t ON t.dir = a.dir ORDER BY a.dir" +# expected: [{"dir":"papers/one","title":"T1"}] -- the same rows as before, one walk's worth of wall time +```