diff --git a/packages/python/e2e-attestations/bot-1475-rowpath-1016.json b/packages/python/e2e-attestations/bot-1475-rowpath-1016.json new file mode 100644 index 00000000..a5904b13 --- /dev/null +++ b/packages/python/e2e-attestations/bot-1475-rowpath-1016.json @@ -0,0 +1,7 @@ +{ + "command": "uv run python -m pytest tests/e2e/glob_speed_depth_bound_test.py tests/e2e/glob_speed_one_level_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_gitignore_test.py tests/e2e/glob_speed_node_modules_test.py tests/e2e/glob_speed_symlink_test.py tests/e2e/query_contract_test.py tests/e2e/fanout_test.py tests/e2e/use_case_join_test.py tests/e2e/stat_speed_test.py tests/e2e/release_speed_test.py -q -s", + "ran_at": 1792497279, + "exit_code": 0, + "commit": "c7452a0ddee9a0a38f73832e984d74372e8adab8", + "branch": "bot/1475-rowpath-1016" +} diff --git a/packages/python/tests/e2e/glob_speed_recursive_test.py b/packages/python/tests/e2e/glob_speed_recursive_test.py index 66a1cb29..9e47a84f 100644 --- a/packages/python/tests/e2e/glob_speed_recursive_test.py +++ b/packages/python/tests/e2e/glob_speed_recursive_test.py @@ -65,7 +65,7 @@ ] CASES = [ ("./**/*.md", "./**/*.md", ALL_NATIVE, "", FASTEST), - ("./a/b/**/*.md", "./a/b/**/*.md", PREFIX_NATIVE, "a/b", ""), + ("./a/b/**/*.md", "./a/b/**/*.md", PREFIX_NATIVE, "a/b", FASTEST), ] FANOUT = 8 diff --git a/packages/rust/changelog.d/2026-10-16-plain-scan-without-stepping.md b/packages/rust/changelog.d/2026-10-16-plain-scan-without-stepping.md new file mode 100644 index 00000000..2d0b4421 --- /dev/null +++ b/packages/rust/changelog.d/2026-10-16-plain-scan-without-stepping.md @@ -0,0 +1,3 @@ +**Changed** + +- **A plain projection of a path table, such as `SELECT path FROM './a/b/**/*.md'`, reads its rows straight from the walk instead of stepping SQLite once per row.** Any `WHERE`, `ORDER BY`, `LIMIT`, expression, join or `content` column still runs through SQLite. Query results are unchanged. diff --git a/packages/rust/migrations.d/2026-10-16-plain-scan-without-stepping.md b/packages/rust/migrations.d/2026-10-16-plain-scan-without-stepping.md new file mode 100644 index 00000000..6ee990cb --- /dev/null +++ b/packages/rust/migrations.d/2026-10-16-plain-scan-without-stepping.md @@ -0,0 +1,25 @@ +### Plain scans without stepping + +#### Summary + +A query that only projects stat columns of one path table reads its rows without stepping SQLite. Output is byte-identical; nothing breaks. + +#### Required changes + +_None._ + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +_None._ + +#### Verification + +```bash +dirsql "SELECT path FROM './**/*.md'" +``` + +Prints the same rows as before, sooner. diff --git a/packages/rust/src/db.rs b/packages/rust/src/db.rs index 19bfc6fd..ffd19af4 100644 --- a/packages/rust/src/db.rs +++ b/packages/rust/src/db.rs @@ -1341,8 +1341,11 @@ impl Db { } let _calls = self.calls.phase(); let _scope = self.scope.enter(); - let (mut stmt, column_names) = self.prepare_statement(sql)?; + let (mut stmt, column_names, reads) = self.prepare_reading(sql)?; let mut state = start(&column_names); + if self.read_directly(sql, &reads, &mut |cells| on_row(&mut state, cells)) { + return Ok(state); + } let mut rows = stmt.query([])?; let mut spare: Vec> = Vec::with_capacity(column_names.len()); while let Some(row) = rows.next()? { @@ -1382,6 +1385,16 @@ impl Db { /// Prepare one read-only statement, registering the path-tables it names /// along the way, with its column names. fn prepare_statement(&self, sql: &str) -> Result<(rusqlite::Statement<'_>, Vec)> { + self.prepare_reading(sql) + .map(|(stmt, column_names, _)| (stmt, column_names)) + } + + /// [`prepare_statement`](Self::prepare_statement), with the tables the + /// statement reads. + fn prepare_reading( + &self, + sql: &str, + ) -> Result<(rusqlite::Statement<'_>, Vec, HashSet)> { // Each iteration must register a table no earlier iteration did; a // repeat means the fallback is not making progress, so the SQLite // error stands. That is what bounds the loop. @@ -1406,7 +1419,44 @@ impl Db { } self.scope.warm(&reads); let column_names: Vec = stmt.column_names().iter().map(|s| s.to_string()).collect(); - Ok((stmt, column_names)) + Ok((stmt, column_names, reads)) + } + + /// Hand `on_row` the rows of `sql` straight from the one table it reads, + /// when its program is a plain scan of that table: SQLite would only copy + /// each cell through. False, having handed nothing, for any other + /// statement. + fn read_directly( + &self, + sql: &str, + reads: &HashSet, + on_row: &mut dyn FnMut(&[ValueRef<'_>]), + ) -> bool { + let mut tables = reads.iter(); + let (Some(table), None) = (tables.next(), tables.next()) else { + return false; + }; + self.program(sql) + .and_then(|program| scanned_columns(&program)) + .is_some_and(|columns| self.scope.each_row(table, &columns, on_row)) + } + + /// The program SQLite compiles `sql` to, as `EXPLAIN` lists it. + fn program(&self, sql: &str) -> Option> { + let mut explain = self.conn.prepare(&format!("EXPLAIN {sql}")).ok()?; + explain + .query_map([], |row| { + Ok(Op { + code: row.get(1)?, + p1: row.get(2)?, + p2: row.get(3)?, + p3: row.get(4)?, + p5: row.get(6)?, + }) + }) + .ok()? + .collect::>>() + .ok() } /// Prepare `sql` with the internal-table / ATTACH authorizer installed for @@ -1647,6 +1697,73 @@ fn value_ref(value: &Value) -> ValueRef<'_> { } } +/// One instruction of a statement's program, as `EXPLAIN` lists it; `p4` +/// is left out, since for these programs it holds only pointers and cookies. +#[derive(Debug, Clone, PartialEq)] +struct Op { + code: String, + p1: i64, + p2: i64, + p3: i64, + p5: i64, +} + +fn op(code: &str, p1: i64, p2: i64, p3: i64) -> Op { + Op { + code: code.to_string(), + p1, + p2, + p3, + p5: 0, + } +} + +/// The program SQLite compiles `SELECT` of `columns` of one virtual table to, +/// up to the transactions it opens: scan cursor 0 with idxNum 0 and no +/// arguments, and yield the columns of each row as they are. +fn scan_program(columns: &[i64]) -> Vec { + let width = i64::try_from(columns.len()).unwrap_or(i64::MAX); + let mut program = vec![ + op("Init", 0, width + 8, 0), + op("VOpen", 0, 0, 0), + op("Integer", 0, 1, 0), + op("Integer", 0, 2, 0), + op("VFilter", 0, width + 7, 1), + ]; + program.extend( + (3..) + .zip(columns) + .map(|(at, &column)| op("VColumn", 0, column, at)), + ); + program.extend([ + op("ResultRow", 3, width, 0), + op("VNext", 0, 5, 0), + op("Halt", 0, 0, 0), + ]); + program +} + +/// The columns `program` yields when it is exactly [`scan_program`]: one +/// full scan of one virtual table, with no filter, order, limit or +/// expression. `None` for any other program. +fn scanned_columns(program: &[Op]) -> Option> { + let columns: Vec = program + .iter() + .filter(|op| op.code == "VColumn") + .map(|op| op.p2) + .collect(); + let expected = scan_program(&columns); + let (body, rest) = program.split_at_checked(expected.len())?; + let (last, transactions) = rest.split_last()?; + let plain = !columns.is_empty() + && body == expected + && transactions.iter().all(|op| op.code == "Transaction") + && *last == op("Goto", 0, 1, 0); + plain + .then(|| columns.iter().map(|&c| usize::try_from(c).ok()).collect()) + .flatten() +} + /// `cells` emptied and free to borrow anew, keeping its allocation. fn recycled<'a>(mut cells: Vec>) -> Vec> { cells.clear(); @@ -1798,6 +1915,104 @@ mod tests { .unwrap() } + fn with_transactions(columns: &[i64], transactions: usize) -> Vec { + let mut program = scan_program(columns); + program.extend((0..transactions).map(|_| op("Transaction", 1, 0, 1))); + program.push(op("Goto", 0, 1, 0)); + program + } + + #[test] + fn a_scan_program_is_what_sqlite_compiles_a_plain_projection_to() { + let codes = |program: &[Op]| -> Vec<(String, i64, i64, i64)> { + program + .iter() + .map(|op| (op.code.clone(), op.p1, op.p2, op.p3)) + .collect() + }; + let expected: Vec<(String, i64, i64, i64)> = [ + ("Init", 0, 10, 0), + ("VOpen", 0, 0, 0), + ("Integer", 0, 1, 0), + ("Integer", 0, 2, 0), + ("VFilter", 0, 9, 1), + ("VColumn", 0, 4, 3), + ("VColumn", 0, 0, 4), + ("ResultRow", 3, 2, 0), + ("VNext", 0, 5, 0), + ("Halt", 0, 0, 0), + ] + .into_iter() + .map(|(code, p1, p2, p3)| (code.to_string(), p1, p2, p3)) + .collect(); + assert_eq!(codes(&scan_program(&[4, 0])), expected); + } + + #[test] + fn a_plain_scan_yields_its_columns_whatever_transactions_it_opens() { + for transactions in 0..3 { + assert_eq!( + scanned_columns(&with_transactions(&[2, 0, 2], transactions)), + Some(vec![2, 0, 2]) + ); + } + } + + #[test] + fn a_program_that_differs_from_a_plain_scan_yields_nothing() { + let plain = with_transactions(&[0, 1], 1); + let mut filtered = plain.clone(); + filtered.insert(7, op("Le", 3, 6, 4)); + let mut shifted = plain.clone(); + shifted[6].p3 = 5; + let mut flagged = plain.clone(); + flagged[5].p5 = 1; + let mut opened = plain.clone(); + opened[10] = op("OpenRead", 1, 2, 0); + let mut ended = plain.clone(); + ended[11] = op("Goto", 0, 2, 0); + for program in [ + filtered, + shifted, + flagged, + opened, + ended, + plain[..10].to_vec(), + plain[..4].to_vec(), + with_transactions(&[], 1), + with_transactions(&[-1], 1), + ] { + assert_eq!(scanned_columns(&program), None, "{program:?}"); + } + } + + #[test] + fn a_plain_scan_of_a_path_table_is_read_without_stepping() { + let mut db = Db::new().unwrap(); + db.set_path_table_root(PathBuf::from("/nonexistent-dirsql-root")); + let plain = "SELECT path FROM './*.md'"; + let limited = "SELECT path FROM './*.md' LIMIT 1"; + let (_plain_stmt, _, plain_reads) = db.prepare_reading(plain).unwrap(); + let (_limited_stmt, _, limited_reads) = db.prepare_reading(limited).unwrap(); + let mut rows = 0; + + assert!(db.read_directly(plain, &plain_reads, &mut |_| rows += 1)); + assert!(!db.read_directly(limited, &limited_reads, &mut |_| rows += 1)); + assert_eq!(rows, 0); + } + + #[test] + fn a_statement_over_an_ordinary_table_is_stepped() { + let db = Db::new().unwrap(); + db.create_table("t", "CREATE TABLE t (a TEXT)").unwrap(); + let reads: HashSet = ["t".to_string()].into(); + + assert!(db.program("SELECT a FROM t").is_some()); + assert!(db.program("SELECT a FROM").is_none()); + assert!(!db.read_directly("SELECT a FROM t", &reads, &mut |_| {})); + assert!(!db.read_directly("SELECT 1", &HashSet::new(), &mut |_| {})); + } + #[test] fn filling_a_text_cell_with_text_reuses_its_buffer() { let mut cell = Value::Text(String::with_capacity(64)); diff --git a/packages/rust/src/vtab.rs b/packages/rust/src/vtab.rs index b489b279..cb05cd05 100644 --- a/packages/rust/src/vtab.rs +++ b/packages/rust/src/vtab.rs @@ -487,6 +487,15 @@ impl TableSource for ScanSpec { } } + const HELD_COLUMNS: usize = CONTENT_COLUMN; + + fn cell<'r>(&self, row: &'r FileRow, i: usize) -> ValueRef<'r> { + c_int::try_from(i) + .ok() + .and_then(|i| cell(row, i, &|row| self.facts(row))) + .unwrap_or(ValueRef::Null) + } + const PREFETCH_COLUMN: Option = Some(CONTENT_COLUMN); fn prefetch(&self, rows: &[&FileRow]) { @@ -610,6 +619,28 @@ mod tests { all } + #[test] + fn a_held_cell_is_the_value_the_column_emits() { + let args = args_with(&[b"'/tmp/notes'", b"'**/*.md'", b"''", b"'gitignore'"]); + let spec = parse_module_args(&args).unwrap(); + let row = FileRow::new(Path::new(""), PathBuf::from("docs/a.md")); + let cells: Vec> = [PATH_COLUMN, BASENAME_COLUMN, DIR_COLUMN, EXT_COLUMN] + .into_iter() + .map(|i| spec.cell(&row, i)) + .collect(); + + assert_eq!( + cells, + [ + ValueRef::Text(b"docs/a.md"), + ValueRef::Text(b"a.md"), + ValueRef::Text(b"docs"), + ValueRef::Text(b"md"), + ] + ); + assert_eq!(spec.cell(&row, CONTENT_COLUMN), ValueRef::Null); + } + #[test] fn parse_module_args_extracts_root_and_glob() { let args = args_with(&[b"'/tmp/notes'", b"'**/*.md'", b"''", b"'gitignore'"]); diff --git a/packages/rust/src/vtab_scaffold.rs b/packages/rust/src/vtab_scaffold.rs index 605ecb2f..6bc1effe 100644 --- a/packages/rust/src/vtab_scaffold.rs +++ b/packages/rust/src/vtab_scaffold.rs @@ -94,6 +94,16 @@ pub trait TableSource: Sized + Send + Sync + 'static { /// Read [`Self::PREFETCH_COLUMN`] for `rows`. Only reached when the /// column is declared, so a source that declares none is never asked. fn prefetch(&self, _rows: &[&Self::Row]) {} + + /// How many leading columns every row holds as a [`cell`](Self::cell), + /// so a plain scan of them can be read without stepping SQLite. + const HELD_COLUMNS: usize = 0; + + /// Column `i` of `row`, the value [`column`](Self::column) emits. Only + /// reached for a column below [`HELD_COLUMNS`](Self::HELD_COLUMNS). + fn cell<'r>(&self, _row: &'r Self::Row, i: usize) -> ValueRef<'r> { + unreachable!("column {i} is not held by {}", Self::NAME) + } } /// The row sets the tables on one connection hold across a statement. @@ -152,6 +162,26 @@ impl StatementScope { } } + /// Hand `on_row` columns `columns` of every row of table `name`, in the + /// order a full scan of it yields them, without stepping SQLite. False, + /// having handed nothing, when no table here answers to `name` or its + /// rows do not hold one of `columns`. + pub fn each_row( + &self, + name: &str, + columns: &[usize], + on_row: &mut dyn FnMut(&[ValueRef<'_>]), + ) -> bool { + let cache = self + .tables + .lock() + .unwrap() + .iter() + .filter(|(table, _)| table == name) + .find_map(|(_, cache)| cache.upgrade()); + cache.is_some_and(|cache| cache.each_row(columns, on_row)) + } + fn adopt(&self, name: String, cache: Weak) { self.tables.lock().unwrap().push((name, cache)); } @@ -168,6 +198,7 @@ impl Drop for StatementGuard { trait Held { fn evict(&self); fn warm(&self); + fn each_row(&self, columns: &[usize], on_row: &mut dyn FnMut(&[ValueRef<'_>])) -> bool; } /// One table's rows for the statement in progress, empty between statements. @@ -245,6 +276,22 @@ impl Held for RowCache { fn warm(&self) { self.warm(); } + + fn each_row(&self, columns: &[usize], on_row: &mut dyn FnMut(&[ValueRef<'_>])) -> bool { + if !columns.iter().all(|&i| i < S::HELD_COLUMNS) { + return false; + } + let rows = self.rows(); + for chunk in (0..).map_while(|n| rows.chunk(n)) { + let mut cells = Vec::with_capacity(columns.len()); + for row in chunk.iter() { + cells.clear(); + cells.extend(columns.iter().map(|&i| self.source.cell(row, i))); + on_row(&cells); + } + } + true + } } /// Register `S` as a virtual-table module on `conn`, its tables caching rows @@ -1347,6 +1394,131 @@ mod tests { assert_eq!(contents(&cache.rows()), vec![3]); } + /// A source holding two columns, each cell its row times ten plus the + /// column, over rows pushed in two chunks. + struct Tens { + scans: std::sync::atomic::AtomicUsize, + } + + impl TableSource for Tens { + type Row = i64; + const NAME: &'static str = "tens"; + + fn connect(_args: &[&[u8]]) -> Result<(String, Self)> { + unreachable!() + } + + fn rows(&self, rows: &Arc>) { + self.scans + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + rows.push(Arc::new(vec![1, 2])); + rows.push(Arc::new(vec![3])); + } + + fn column(&self, _row: &i64, _ctx: &mut Context, _i: c_int) -> Result<()> { + unreachable!() + } + + const HELD_COLUMNS: usize = 2; + + fn cell<'r>(&self, row: &'r i64, i: usize) -> ValueRef<'r> { + ValueRef::Integer(row * 10 + i64::try_from(i).unwrap()) + } + } + + fn tens(scope: &StatementScope, name: &str) -> Arc> { + let cache = Arc::new(RowCache::new(Arc::new(Tens { + scans: std::sync::atomic::AtomicUsize::new(0), + }))); + scope.adopt( + name.to_string(), + Arc::downgrade(&cache) as Weak, + ); + cache + } + + fn each_row(scope: &StatementScope, name: &str, columns: &[usize]) -> Option>> { + let mut rows = Vec::new(); + let read = scope.each_row(name, columns, &mut |cells| { + rows.push(cells.iter().map(|cell| cell.as_i64().unwrap()).collect()); + }); + read.then_some(rows) + } + + #[test] + fn a_scope_hands_each_row_of_a_table_in_scan_order_by_column() { + let scope = StatementScope::new(); + let _table = tens(&scope, "t"); + + assert_eq!( + each_row(&scope, "t", &[1, 0, 1]), + Some(vec![vec![11, 10, 11], vec![21, 20, 21], vec![31, 30, 31]]) + ); + } + + #[test] + fn a_scope_reads_rows_the_statement_already_holds() { + let scope = StatementScope::new(); + let table = tens(&scope, "t"); + table.rows(); + + each_row(&scope, "t", &[0]); + + assert_eq!( + table + .source + .scans + .load(std::sync::atomic::Ordering::Relaxed), + 1 + ); + } + + #[test] + fn a_scope_hands_nothing_and_scans_nothing_for_a_column_not_held() { + let scope = StatementScope::new(); + let table = tens(&scope, "t"); + let counted = adopted(&scope, "c"); + + assert_eq!(each_row(&scope, "t", &[0, 2]), None); + assert_eq!(each_row(&scope, "c", &[0]), None); + assert!(!is_held(&counted)); + assert_eq!( + table + .source + .scans + .load(std::sync::atomic::Ordering::Relaxed), + 0 + ); + } + + #[test] + fn a_scope_hands_nothing_for_a_table_it_does_not_hold() { + let scope = StatementScope::new(); + let _table = tens(&scope, "t"); + let gone = tens(&scope, "gone"); + drop(gone); + + assert_eq!(each_row(&scope, "absent", &[0]), None); + assert_eq!(each_row(&scope, "gone", &[0]), None); + } + + #[test] + fn a_scope_reads_the_live_table_of_a_name_adopted_twice() { + let scope = StatementScope::new(); + let gone = tens(&scope, "t"); + drop(gone); + let _live = tens(&scope, "t"); + + assert_eq!(each_row(&scope, "t", &[0]).map(|rows| rows.len()), Some(3)); + } + + #[test] + #[should_panic(expected = "column 0 is not held by counted")] + fn a_source_that_holds_no_column_has_no_cell() { + let cache = counted(); + cache.source.cell(&0, 0); + } + #[test] fn table_name_is_the_third_argument_sqlite_prepends() { assert_eq!( diff --git a/packages/rust/tests/path_table_query.rs b/packages/rust/tests/path_table_query.rs index ac288cd4..5c5227cd 100644 --- a/packages/rust/tests/path_table_query.rs +++ b/packages/rust/tests/path_table_query.rs @@ -370,3 +370,49 @@ fn an_ordinary_syntax_error_carries_no_quoting_hint() { "a syntax error with no path in it must stay unhinted, got: {err}" ); } + +/// Each row of `sql` as `query_each` folds it, beside the same rows as +/// `query_ordered` steps them, both sorted. +fn folded_and_stepped(db: &DirSQL, sql: &str) -> (Vec>, Vec>) { + let (columns, mut folded) = db + .query_each( + sql, + |columns| (columns.to_vec(), Vec::new()), + |(_, rows), cells| rows.push(cells.to_vec()), + ) + .unwrap(); + let result = db.query_ordered(sql).unwrap(); + assert_eq!(columns, result.columns, "{sql}"); + let mut stepped: Vec> = result + .rows + .iter() + .map(|row| columns.iter().map(|c| row[c].clone()).collect()) + .collect(); + let key = |row: &Vec| format!("{row:?}"); + folded.sort_by_key(key); + stepped.sort_by_key(key); + (folded, stepped) +} + +#[test] +fn rows_folded_from_a_path_table_are_the_rows_sqlite_steps() { + let root = fixture(); + fs::write(root.path().join(".hidden"), "").unwrap(); + let db = open(&root); + + for sql in [ + "SELECT path FROM './**'", + "SELECT path AS p, ext, dir, basename, path FROM './**'", + "SELECT * FROM './**/*.md'", + "SELECT size, mtime, ctime FROM './docs/*'", + "SELECT path FROM './docs/*.rst'", + "SELECT path, content FROM './docs/*.md'", + "SELECT path FROM './**' WHERE ext = 'md' LIMIT 2", + "SELECT path, cells FROM rows_csv", + "SELECT path FROM './docs/*.md';", + ] { + let (folded, stepped) = folded_and_stepped(&db, sql); + assert!(!folded.is_empty() || sql.contains(".rst"), "{sql}"); + assert_eq!(folded, stepped, "{sql}"); + } +} diff --git a/packages/ts/e2e-attestations/bot-1475-rowpath-1016.json b/packages/ts/e2e-attestations/bot-1475-rowpath-1016.json new file mode 100644 index 00000000..93470df8 --- /dev/null +++ b/packages/ts/e2e-attestations/bot-1475-rowpath-1016.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1792497328, + "exit_code": 0, + "commit": "2637cbd19758e32de351f68960a841c891e480b6", + "branch": "bot/1475-rowpath-1016" +}