From 9399d4737057130d9cab9f96139fae7585639db3 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Wed, 9 Sep 2026 09:52:55 -0400 Subject: [PATCH 1/8] refactor(rust): share the vtab scaffolding between the two path tables `vtab.rs` and `parsed_vtab.rs` are two `dirsql_*` virtual tables built the same way, and each carried its own copy of the way: module-argument decoding through `sql_literal::unquote`, the arity error, glob and ignore-pattern compilation, the gitignore switch, cursor arithmetic, and the whole `VTab` / `CreateVTab` / `VTabCursor` shape. That shape now lives once in `vtab_scaffold`, as a generic `ScaffoldTab` / `ScaffoldCursor` over a `TableSource` trait. Each table supplies only the three things that actually differ: how its module arguments decode, how it produces a row set, and how a row becomes a cell. One behavior-preserving consequence worth naming: the parsed table used to clone its whole row vector into every cursor at `xOpen`; the scaffold hands out an `Arc` per `xFilter` instead, so a join over a parsed table no longer copies the rows per outer row. Public surface is unchanged -- `MODULE_NAME`, `STAT_COLUMNS`, `declared_schema` and both `load_module` functions keep their paths and signatures, and the scaffold module is crate-private. --- .../2026-09-09-shared-vtab-scaffold.md | 7 + packages/rust/src/lib.rs | 1 + packages/rust/src/parsed_vtab.rs | 184 ++-------- packages/rust/src/vtab.rs | 242 +++---------- packages/rust/src/vtab_scaffold.rs | 331 ++++++++++++++++++ 5 files changed, 415 insertions(+), 350 deletions(-) create mode 100644 packages/rust/changelog.d/2026-09-09-shared-vtab-scaffold.md create mode 100644 packages/rust/src/vtab_scaffold.rs diff --git a/packages/rust/changelog.d/2026-09-09-shared-vtab-scaffold.md b/packages/rust/changelog.d/2026-09-09-shared-vtab-scaffold.md new file mode 100644 index 000000000..5eeceac34 --- /dev/null +++ b/packages/rust/changelog.d/2026-09-09-shared-vtab-scaffold.md @@ -0,0 +1,7 @@ +**Changed** — the `dirsql_path` and `dirsql_parsed` virtual tables now share one +piece of scaffolding instead of carrying parallel copies of it. Module-argument +decoding, glob and ignore-pattern compilation, the gitignore switch, and the +read-only table and cursor (`xBestIndex` / `xFilter` / `xNext` / `xEof` / +`xColumn` / `xRowid`) live once in an internal module; each table now supplies +only what differs — its own arguments, how it produces a row set, and how a row +becomes a cell. No public API, SQL surface, or observable behavior changes. diff --git a/packages/rust/src/lib.rs b/packages/rust/src/lib.rs index 6df900248..45ca65443 100644 --- a/packages/rust/src/lib.rs +++ b/packages/rust/src/lib.rs @@ -37,6 +37,7 @@ pub mod scanner; pub mod sql_literal; #[doc(hidden)] pub mod vtab; +mod vtab_scaffold; #[doc(hidden)] pub mod watcher; diff --git a/packages/rust/src/parsed_vtab.rs b/packages/rust/src/parsed_vtab.rs index e8cdbb8a5..97b3029c2 100644 --- a/packages/rust/src/parsed_vtab.rs +++ b/packages/rust/src/parsed_vtab.rs @@ -23,13 +23,11 @@ use std::collections::{HashMap, HashSet}; use std::ffi::c_int; use std::path::{Path, PathBuf}; +use std::sync::Arc; use globset::GlobSet; -use rusqlite::vtab::{ - Context, CreateVTab, IndexInfo, VTab, VTabConnection, VTabCursor, VTabKind, Values, - read_only_module, -}; -use rusqlite::{Connection, Error, Result, ffi}; +use rusqlite::vtab::Context; +use rusqlite::{Connection, Error, Result}; use crate::Value; use crate::command::{Placeholder, run_command}; @@ -38,16 +36,15 @@ use crate::matcher::TableMatcher; use crate::parsed_cache::{self, CachedParse, Entry, RowCache, SqliteRowCache}; use crate::path_table; use crate::persist::{FileStat, hash_file, now_ns}; -use crate::scanner::{self, scan_glob}; -use crate::sql_literal::unquote; +use crate::scanner::scan_glob; +use crate::vtab_scaffold::{self, TableSource}; /// SQL module name a parsed path-table is created with. pub const MODULE_NAME: &str = "dirsql_parsed"; /// Register the parsed path-table module on `conn`. pub fn load_module(conn: &Connection) -> Result<()> { - let aux: Option<()> = None; - conn.create_module(MODULE_NAME, read_only_module::(), aux) + vtab_scaffold::load_module::(conn) } /// Number of module arguments that are not ignore patterns. @@ -67,43 +64,32 @@ struct ModuleArgs { ignore: TableMatcher, } -/// Compile the ignore patterns a parsed path-table scan applies. -fn compile_ignore(patterns: &[String]) -> Result { - let refs: Vec<&str> = patterns.iter().map(String::as_str).collect(); - TableMatcher::new(&[], &refs).map_err(|e| Error::ModuleError(e.to_string())) -} - /// Parse a parsed path-table's `CREATE VIRTUAL TABLE` arguments. `args[0..3]` /// are the module, database and table names; the module's own follow — root, /// glob, parser, the gitignore switch, the cache path, then any ignore /// patterns. fn parse_module_args(args: &[&[u8]]) -> Result { - let user_args: Vec = args - .iter() - .skip(3) - .map(|a| unquote(&String::from_utf8_lossy(a)).to_string()) - .collect(); + let user_args = vtab_scaffold::user_args(args); let [root, pattern, command, gitignore, cache, ignore @ ..] = user_args.as_slice() else { - return Err(Error::ModuleError(format!( - "{MODULE_NAME} takes at least {FIXED_ARGS} arguments \ - (root, glob, parser, gitignore switch, cache path), got {}", - user_args.len() - ))); + return Err(vtab_scaffold::arity_error( + MODULE_NAME, + FIXED_ARGS, + "root, glob, parser, gitignore switch, cache path", + user_args.len(), + )); }; - let glob = scanner::compile_glob(pattern).map_err(|e| Error::ModuleError(e.to_string()))?; - Ok(ModuleArgs { root: PathBuf::from(root), pattern: pattern.clone(), - glob, + glob: vtab_scaffold::compile_glob(pattern)?, command: command.clone(), - gitignore: scanner::parse_gitignore_arg(gitignore).map_err(Error::ModuleError)?, + gitignore: vtab_scaffold::parse_gitignore(gitignore)?, // The empty string is how "no cache" is spelled: a module argument // cannot be absent without shifting every argument after it. cache: Some(PathBuf::from(cache)).filter(|p| !p.as_os_str().is_empty()), - ignore: compile_ignore(ignore)?, + ignore: vtab_scaffold::compile_ignore(ignore)?, }) } @@ -321,37 +307,21 @@ fn column_name(names: &[String], i: c_int) -> Option<&str> { names.get(index).map(String::as_str) } -/// Whether the cursor has run past the last row. -fn at_eof(index: usize, row_count: usize) -> bool { - index >= row_count -} - -/// Rowid for a cursor position, saturating rather than wrapping on a row count -/// no filesystem will produce. -fn rowid_of(index: usize) -> i64 { - i64::try_from(index).unwrap_or(i64::MAX) -} - -#[repr(C)] -struct ParsedTab { - /// Base class. Must be first. - base: ffi::sqlite3_vtab, +/// The table a `CREATE` produced: the inferred column names and the rows the +/// parsers emitted, both fixed for the life of the table. +struct ParsedTable { /// Only the names survive registration: the types live in the schema /// SQLite already holds, and a cursor addresses columns by index. column_names: Vec, - rows: Vec, + rows: Arc>, } -#[expect(unsafe_code, reason = "rusqlite requires VTab to be an unsafe trait")] -unsafe impl<'vtab> VTab<'vtab> for ParsedTab { - type Aux = (); - type Cursor = ParsedTabCursor; +impl TableSource for ParsedTable { + type Row = JsonRow; + + const NAME: &'static str = MODULE_NAME; - fn connect( - _db: &mut VTabConnection, - _aux: Option<&()>, - args: &[&[u8]], - ) -> Result<(String, Self)> { + fn connect(args: &[&[u8]]) -> Result<(String, Self)> { let ModuleArgs { root, pattern, @@ -384,81 +354,25 @@ unsafe impl<'vtab> VTab<'vtab> for ParsedTab { } let schema = declared_schema(&columns); - let vtab = Self { - base: ffi::sqlite3_vtab::default(), + let table = Self { column_names: columns.into_iter().map(|c| c.name).collect(), - rows, + rows: Arc::new(rows), }; - Ok((schema, vtab)) + Ok((schema, table)) } - fn best_index(&self, info: &mut IndexInfo) -> Result<()> { - info.set_estimated_cost(1000.); - Ok(()) + fn rows(&self) -> Arc> { + // Materialized once at CREATE, so every statement reads the same rows + // rather than re-spawning a parser per file. + Arc::clone(&self.rows) } - fn open(&'vtab mut self) -> Result { - Ok(ParsedTabCursor { - base: ffi::sqlite3_vtab_cursor::default(), - column_names: self.column_names.clone(), - rows: self.rows.clone(), - index: 0, - }) - } -} - -impl<'vtab> CreateVTab<'vtab> for ParsedTab { - const KIND: VTabKind = VTabKind::Default; -} - -#[repr(C)] -struct ParsedTabCursor { - /// Base class. Must be first: `rust_open` hands this pointer straight to - /// SQLite as a `sqlite3_vtab_cursor`, so anything ahead of it gets - /// overwritten. - base: ffi::sqlite3_vtab_cursor, - column_names: Vec, - rows: Vec, - index: usize, -} - -#[expect( - unsafe_code, - reason = "rusqlite requires VTabCursor to be an unsafe trait" -)] -unsafe impl VTabCursor for ParsedTabCursor { - fn filter( - &mut self, - _idx_num: c_int, - _idx_str: Option<&str>, - _args: &Values<'_>, - ) -> Result<()> { - self.index = 0; - Ok(()) - } - - fn next(&mut self) -> Result<()> { - self.index += 1; - Ok(()) - } - - fn eof(&self) -> bool { - at_eof(self.index, self.rows.len()) - } - - fn column(&self, ctx: &mut Context, i: c_int) -> Result<()> { - let value = self - .rows - .get(self.index) - .zip(column_name(&self.column_names, i)) - .map(|(row, name)| cell(row, name)) + fn column(&self, row: &JsonRow, ctx: &mut Context, i: c_int) -> Result<()> { + let value = column_name(&self.column_names, i) + .map(|name| cell(row, name)) .unwrap_or(Value::Null); ctx.set_result(&value) } - - fn rowid(&self) -> Result { - Ok(rowid_of(self.index)) - } } #[cfg(test)] @@ -729,34 +643,6 @@ mod tests { assert_eq!(column_name(&names(), -1), None); } - #[test] - fn at_eof_is_false_while_rows_remain() { - assert!(!at_eof(0, 2)); - assert!(!at_eof(1, 2)); - } - - #[test] - fn at_eof_is_true_once_past_the_last_row() { - assert!(at_eof(2, 2)); - assert!(at_eof(3, 2)); - } - - #[test] - fn at_eof_is_true_for_an_empty_row_set() { - assert!(at_eof(0, 0)); - } - - #[test] - fn rowid_of_tracks_the_cursor_position() { - assert_eq!(rowid_of(0), 0); - assert_eq!(rowid_of(7), 7); - } - - #[test] - fn rowid_of_saturates_rather_than_wrapping() { - assert_eq!(rowid_of(usize::MAX), i64::MAX); - } - // The run_parser tests spawn a real `sh`. Their test code statically // references only `super::` items (plus pure std), matching the pattern // `command.rs` uses for the runner underneath. diff --git a/packages/rust/src/vtab.rs b/packages/rust/src/vtab.rs index c0c3572ea..abbdcbfc4 100644 --- a/packages/rust/src/vtab.rs +++ b/packages/rust/src/vtab.rs @@ -3,17 +3,14 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use globset::GlobSet; -use rusqlite::vtab::{ - Context, CreateVTab, IndexInfo, VTab, VTabConnection, VTabCursor, VTabKind, Values, - read_only_module, -}; -use rusqlite::{Connection, Error, Result, ffi}; +use rusqlite::vtab::Context; +use rusqlite::{Connection, Result}; use crate::compute_stat_virtuals; use crate::matcher::TableMatcher; use crate::path_table; -use crate::scanner::{self, scan_glob}; -use crate::sql_literal::unquote; +use crate::scanner::scan_glob; +use crate::vtab_scaffold::{self, TableSource}; use crate::{Row, Value}; /// SQL module name a path-table is created with: @@ -53,12 +50,6 @@ fn column_type(column: &str) -> &'static str { } } -/// Compile a path-table's glob, surfacing a bad pattern as a module error so -/// SQLite reports it against the `CREATE VIRTUAL TABLE` statement. -fn compile_glob(pattern: &str) -> Result { - scanner::compile_glob(pattern).map_err(|e| Error::ModuleError(e.to_string())) -} - /// Read `path` as text, yielding `None` when it is unreadable or not valid /// UTF-8. A file that cannot be read is a NULL cell, never a failed row: the /// filesystem is allowed to be messy and a query over it should still return. @@ -84,10 +75,11 @@ struct ScanSpec { gitignore: bool, } -/// Compile the ignore patterns a path-table scan applies. -fn compile_ignore(patterns: &[String]) -> Result { - let refs: Vec<&str> = patterns.iter().map(String::as_str).collect(); - TableMatcher::new(&[], &refs).map_err(|e| Error::ModuleError(e.to_string())) +/// One matched file: its absolute path, kept for the lazy content read, and +/// its already-computed stat columns. +struct FileRow { + abs_path: PathBuf, + stats: Row, } /// Parse a path-table's own `CREATE VIRTUAL TABLE` arguments into its scan @@ -95,27 +87,24 @@ fn compile_ignore(patterns: &[String]) -> Result { /// own arguments follow — root, glob, path prefix, the gitignore switch, then /// any ignore patterns. fn parse_module_args(args: &[&[u8]]) -> Result { - let user_args: Vec = args - .iter() - .skip(3) - .map(|a| unquote(&String::from_utf8_lossy(a)).to_string()) - .collect(); + let user_args = vtab_scaffold::user_args(args); let [root, pattern, path_prefix, gitignore, ignore @ ..] = user_args.as_slice() else { - return Err(Error::ModuleError(format!( - "{MODULE_NAME} takes at least {FIXED_ARGS} arguments \ - (root, glob, path prefix, gitignore switch), got {}", - user_args.len() - ))); + return Err(vtab_scaffold::arity_error( + MODULE_NAME, + FIXED_ARGS, + "root, glob, path prefix, gitignore switch", + user_args.len(), + )); }; Ok(ScanSpec { root: PathBuf::from(root), - glob: compile_glob(pattern)?, + glob: vtab_scaffold::compile_glob(pattern)?, path_prefix: PathBuf::from(path_prefix), - ignore: compile_ignore(ignore)?, + ignore: vtab_scaffold::compile_ignore(ignore)?, ignore_base: path_table::ignore_base(pattern), - gitignore: scanner::parse_gitignore_arg(gitignore).map_err(Error::ModuleError)?, + gitignore: vtab_scaffold::parse_gitignore(gitignore)?, }) } @@ -143,17 +132,6 @@ fn stat_cell(stats: &Row, column: usize) -> Value { .unwrap_or(Value::Null) } -/// Whether the cursor has run past the last row. -fn at_eof(index: usize, row_count: usize) -> bool { - index >= row_count -} - -/// Rowid for a cursor position, saturating rather than wrapping on a row count -/// no filesystem will produce. -fn rowid_of(index: usize) -> i64 { - i64::try_from(index).unwrap_or(i64::MAX) -} - /// Build the row set for a scan. `stat` is injected so unit tests can supply /// deterministic facts without touching the filesystem; production passes /// [`compute_stat_virtuals`]. @@ -173,118 +151,34 @@ fn build_rows( .collect() } -/// Register the path-table module on `conn`. -pub fn load_module(conn: &Connection) -> Result<()> { - let aux: Option<()> = None; - conn.create_module(MODULE_NAME, read_only_module::(), aux) -} +impl TableSource for ScanSpec { + type Row = FileRow; -#[repr(C)] -struct PathTab { - /// Base class. Must be first. - base: ffi::sqlite3_vtab, - spec: Arc, -} + const NAME: &'static str = MODULE_NAME; -#[expect(unsafe_code, reason = "rusqlite requires VTab to be an unsafe trait")] -unsafe impl<'vtab> VTab<'vtab> for PathTab { - type Aux = (); - type Cursor = PathTabCursor; - - fn connect( - _db: &mut VTabConnection, - _aux: Option<&()>, - args: &[&[u8]], - ) -> Result<(String, Self)> { - let vtab = Self { - base: ffi::sqlite3_vtab::default(), - spec: Arc::new(parse_module_args(args)?), - }; - Ok((declared_schema(), vtab)) - } - - fn best_index(&self, info: &mut IndexInfo) -> Result<()> { - info.set_estimated_cost(1000.); - Ok(()) - } - - fn open(&'vtab mut self) -> Result { - Ok(PathTabCursor { - base: ffi::sqlite3_vtab_cursor::default(), - spec: Arc::clone(&self.spec), - rows: Vec::new(), - index: 0, - }) + fn connect(args: &[&[u8]]) -> Result<(String, Self)> { + Ok((declared_schema(), parse_module_args(args)?)) } -} -impl<'vtab> CreateVTab<'vtab> for PathTab { - const KIND: VTabKind = VTabKind::Default; -} - -/// One matched file: its absolute path, kept for the lazy content read, and -/// its already-computed stat columns. -struct FileRow { - abs_path: PathBuf, - stats: Row, -} - -#[repr(C)] -struct PathTabCursor { - /// Base class. Must be first: `rust_open` hands this pointer straight to - /// SQLite as a `sqlite3_vtab_cursor`, so anything ahead of it gets - /// overwritten. - base: ffi::sqlite3_vtab_cursor, - spec: Arc, - rows: Vec, - index: usize, -} - -#[expect( - unsafe_code, - reason = "rusqlite requires VTabCursor to be an unsafe trait" -)] -unsafe impl VTabCursor for PathTabCursor { - fn filter( - &mut self, - _idx_num: c_int, - _idx_str: Option<&str>, - _args: &Values<'_>, - ) -> Result<()> { - // The scan runs here rather than at CREATE, which is what makes reads - // live: each statement sees the filesystem as it is now. - let spec = &self.spec; + fn rows(&self) -> Arc> { + // The scan runs per statement rather than at CREATE, which is what + // makes reads live: each statement sees the filesystem as it is now. let rel_paths = scan_glob( - &spec.root, - &spec.glob, - &spec.ignore, - &spec.ignore_base, - spec.gitignore, + &self.root, + &self.glob, + &self.ignore, + &self.ignore_base, + self.gitignore, ); - self.rows = build_rows( - &spec.root, - &spec.path_prefix, + Arc::new(build_rows( + &self.root, + &self.path_prefix, rel_paths, &compute_stat_virtuals, - ); - self.index = 0; - Ok(()) - } - - fn next(&mut self) -> Result<()> { - self.index += 1; - Ok(()) - } - - fn eof(&self) -> bool { - at_eof(self.index, self.rows.len()) + )) } - fn column(&self, ctx: &mut Context, i: c_int) -> Result<()> { - let Some(row) = self.rows.get(self.index) else { - return ctx.set_result(&Value::Null); - }; - + fn column(&self, row: &FileRow, ctx: &mut Context, i: c_int) -> Result<()> { let column = usize::try_from(i).unwrap_or(usize::MAX); if is_content_column(column) { @@ -298,10 +192,11 @@ unsafe impl VTabCursor for PathTabCursor { ctx.set_result(&stat_cell(&row.stats, column)) } +} - fn rowid(&self) -> Result { - Ok(rowid_of(self.index)) - } +/// Register the path-table module on `conn`. +pub fn load_module(conn: &Connection) -> Result<()> { + vtab_scaffold::load_module::(conn) } #[cfg(test)] @@ -365,20 +260,6 @@ mod tests { assert_eq!(MODULE_NAME, "dirsql_path"); } - #[test] - fn compile_glob_accepts_a_valid_pattern() { - assert!(compile_glob("**/*.md").is_ok()); - } - - #[test] - fn compile_glob_rejects_an_invalid_pattern() { - let err = compile_glob("[").unwrap_err(); - assert!( - matches!(err, Error::ModuleError(_)), - "invalid globs surface as module errors, got {err:?}" - ); - } - /// The three fixed arguments SQLite prepends before the module's own. fn args_with<'a>(user: &[&'a [u8]]) -> Vec<&'a [u8]> { let mut all: Vec<&'a [u8]> = vec![ @@ -491,19 +372,6 @@ mod tests { assert!(parse_module_args(&args).is_err()); } - #[test] - fn compile_ignore_accepts_an_empty_pattern_list() { - assert!(compile_ignore(&[]).is_ok()); - } - - #[test] - fn compile_ignore_rejects_an_invalid_pattern() { - let err = compile_ignore(&["[".to_string()]) - .err() - .expect("an invalid pattern must be rejected"); - assert!(matches!(err, Error::ModuleError(_)), "got {err:?}"); - } - #[test] fn reported_path_is_the_relative_path_without_a_prefix() { assert_eq!( @@ -552,34 +420,6 @@ mod tests { assert_eq!(stat_cell(&stats, 99), Value::Null); } - #[test] - fn at_eof_is_false_while_rows_remain() { - assert!(!at_eof(0, 2)); - assert!(!at_eof(1, 2)); - } - - #[test] - fn at_eof_is_true_once_past_the_last_row() { - assert!(at_eof(2, 2)); - assert!(at_eof(3, 2)); - } - - #[test] - fn at_eof_is_true_for_an_empty_scan() { - assert!(at_eof(0, 0)); - } - - #[test] - fn rowid_of_tracks_the_cursor_position() { - assert_eq!(rowid_of(0), 0); - assert_eq!(rowid_of(7), 7); - } - - #[test] - fn rowid_of_saturates_rather_than_wrapping() { - assert_eq!(rowid_of(usize::MAX), i64::MAX); - } - #[test] fn build_rows_joins_each_relative_path_onto_the_root() { let rows = build_rows( diff --git a/packages/rust/src/vtab_scaffold.rs b/packages/rust/src/vtab_scaffold.rs new file mode 100644 index 000000000..3a95b93f5 --- /dev/null +++ b/packages/rust/src/vtab_scaffold.rs @@ -0,0 +1,331 @@ +//! The scaffolding both `dirsql_*` virtual tables are built from: decoding the +//! arguments SQLite hands `CREATE VIRTUAL TABLE`, and the read-only table and +//! cursor those arguments drive. +//! +//! A table implements [`TableSource`] and keeps only what actually differs +//! between the two — which module arguments it takes, how it produces a row +//! set, and how a row becomes a cell. Everything either one would otherwise +//! write twice lives here. + +use std::ffi::c_int; +use std::sync::Arc; + +use globset::GlobSet; +use rusqlite::vtab::{ + Context, CreateVTab, IndexInfo, VTab, VTabConnection, VTabCursor, VTabKind, Values, + read_only_module, +}; +use rusqlite::{Connection, Error, Result, ffi}; + +use crate::Value; +use crate::matcher::TableMatcher; +use crate::scanner; +use crate::sql_literal::unquote; + +/// Cost a full scan is declared to carry. Both tables answer `xBestIndex` the +/// same way: there is no index to choose, so the only job is to stop SQLite +/// assuming the near-infinite default and reordering a join around it. +const SCAN_COST: f64 = 1000.; + +/// What a `dirsql_*` table supplies on top of the scaffolding. +pub trait TableSource: Sized + 'static { + /// One row, as the table models it. + type Row; + + /// SQL module name the table is created with. + const NAME: &'static str; + + /// Decode the module arguments into the table's source and the schema it + /// declares to SQLite. + fn connect(args: &[&[u8]]) -> Result<(String, Self)>; + + /// The row set one statement reads, taken fresh on every `xFilter`. A + /// table whose reads are live rescans here; one that materialized at + /// `CREATE` hands back the `Arc` it already holds. + fn rows(&self) -> Arc>; + + /// Emit column `i` of `row`. + fn column(&self, row: &Self::Row, ctx: &mut Context, i: c_int) -> Result<()>; +} + +/// Register `S` as a virtual-table module on `conn`. +pub fn load_module(conn: &Connection) -> Result<()> { + let aux: Option<()> = None; + conn.create_module(S::NAME, read_only_module::>(), aux) +} + +/// The module's own arguments: the three SQLite prepends (module, database and +/// table name) dropped, and every SQL literal unquoted. +pub fn user_args(args: &[&[u8]]) -> Vec { + args.iter() + .skip(3) + .map(|a| unquote(&String::from_utf8_lossy(a)).to_string()) + .collect() +} + +/// The error a module raises when it was handed too few arguments. Named +/// arity plus the argument list, so SQLite reports what to write against the +/// offending `CREATE VIRTUAL TABLE`. +pub fn arity_error(module: &str, wanted: usize, names: &str, got: usize) -> Error { + Error::ModuleError(format!( + "{module} takes at least {wanted} arguments ({names}), got {got}" + )) +} + +/// Compile a table's glob, surfacing a bad pattern as a module error so SQLite +/// reports it against the `CREATE VIRTUAL TABLE` statement. +pub fn compile_glob(pattern: &str) -> Result { + scanner::compile_glob(pattern).map_err(|e| Error::ModuleError(e.to_string())) +} + +/// Compile the ignore patterns a scan applies. +pub fn compile_ignore(patterns: &[String]) -> Result { + let refs: Vec<&str> = patterns.iter().map(String::as_str).collect(); + TableMatcher::new(&[], &refs).map_err(|e| Error::ModuleError(e.to_string())) +} + +/// Whether the scan respects `.gitignore` files, from the module's switch +/// argument. +pub fn parse_gitignore(arg: &str) -> Result { + scanner::parse_gitignore_arg(arg).map_err(Error::ModuleError) +} + +/// Whether the cursor has run past the last row. +fn at_eof(index: usize, row_count: usize) -> bool { + index >= row_count +} + +/// Rowid for a cursor position, saturating rather than wrapping on a row count +/// no filesystem will produce. +fn rowid_of(index: usize) -> i64 { + i64::try_from(index).unwrap_or(i64::MAX) +} + +#[repr(C)] +pub struct ScaffoldTab { + /// Base class. Must be first. + base: ffi::sqlite3_vtab, + source: Arc, +} + +#[expect(unsafe_code, reason = "rusqlite requires an unsafe trait impl")] +unsafe impl<'vtab, S: TableSource> VTab<'vtab> for ScaffoldTab { + type Aux = (); + type Cursor = ScaffoldCursor; + + fn connect( + _db: &mut VTabConnection, + _aux: Option<&()>, + args: &[&[u8]], + ) -> Result<(String, Self)> { + let (schema, source) = S::connect(args)?; + let vtab = Self { + base: ffi::sqlite3_vtab::default(), + source: Arc::new(source), + }; + Ok((schema, vtab)) + } + + fn best_index(&self, info: &mut IndexInfo) -> Result<()> { + info.set_estimated_cost(SCAN_COST); + Ok(()) + } + + fn open(&'vtab mut self) -> Result> { + Ok(ScaffoldCursor { + base: ffi::sqlite3_vtab_cursor::default(), + source: Arc::clone(&self.source), + rows: Arc::new(Vec::new()), + index: 0, + }) + } +} + +impl<'vtab, S: TableSource> CreateVTab<'vtab> for ScaffoldTab { + const KIND: VTabKind = VTabKind::Default; +} + +#[repr(C)] +pub struct ScaffoldCursor { + /// Base class. Must be first: `rust_open` hands this pointer straight to + /// SQLite as a `sqlite3_vtab_cursor`, so anything ahead of it gets + /// overwritten. + base: ffi::sqlite3_vtab_cursor, + source: Arc, + rows: Arc>, + index: usize, +} + +#[expect(unsafe_code, reason = "rusqlite requires an unsafe trait impl")] +unsafe impl VTabCursor for ScaffoldCursor { + fn filter( + &mut self, + _idx_num: c_int, + _idx_str: Option<&str>, + _args: &Values<'_>, + ) -> Result<()> { + self.rows = self.source.rows(); + self.index = 0; + Ok(()) + } + + fn next(&mut self) -> Result<()> { + self.index += 1; + Ok(()) + } + + fn eof(&self) -> bool { + at_eof(self.index, self.rows.len()) + } + + fn column(&self, ctx: &mut Context, i: c_int) -> Result<()> { + match self.rows.get(self.index) { + Some(row) => self.source.column(row, ctx, i), + None => ctx.set_result(&Value::Null), + } + } + + fn rowid(&self) -> Result { + Ok(rowid_of(self.index)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + + // The `VTab`/`VTabCursor` impls need a real `rusqlite::Connection`, which + // `unit lint`'s isolation rule forbids here; they are covered over real + // directories by `tests/vtab.rs` and `tests/schema_inference.rs`. + + /// The three fixed arguments SQLite prepends before the module's own. + fn args_with<'a>(user: &[&'a [u8]]) -> Vec<&'a [u8]> { + let mut all: Vec<&'a [u8]> = vec![ + b"dirsql_path".as_slice(), + b"main".as_slice(), + b"t".as_slice(), + ]; + all.extend_from_slice(user); + all + } + + #[test] + fn user_args_drops_the_three_sqlite_prepends() { + let args = args_with(&[b"'/tmp'", b"'**/*'"]); + assert_eq!( + user_args(&args), + vec!["/tmp".to_string(), "**/*".to_string()] + ); + } + + #[test] + fn user_args_unquotes_each_sql_literal() { + let args = args_with(&[b"'a''b'"]); + assert_eq!(user_args(&args), vec!["a'b".to_string()]); + } + + #[test] + fn user_args_accepts_unquoted_arguments() { + let args = args_with(&[b"/tmp/notes"]); + assert_eq!(user_args(&args), vec!["/tmp/notes".to_string()]); + } + + #[test] + fn user_args_is_empty_when_the_module_took_none() { + assert_eq!(user_args(&args_with(&[])), Vec::::new()); + } + + #[test] + fn arity_error_names_the_module_the_arity_and_the_arguments() { + let err = arity_error("dirsql_path", 4, "root, glob", 2); + assert!(matches!(err, Error::ModuleError(_)), "got {err:?}"); + let message = err.to_string(); + assert!(message.contains("dirsql_path"), "got: {message}"); + assert!(message.contains("at least 4 arguments"), "got: {message}"); + assert!(message.contains("(root, glob)"), "got: {message}"); + assert!(message.contains("got 2"), "got: {message}"); + } + + #[test] + fn compile_glob_accepts_a_valid_pattern() { + assert!(compile_glob("**/*.md").unwrap().is_match(Path::new("a.md"))); + } + + #[test] + fn compile_glob_rejects_an_invalid_pattern() { + let err = compile_glob("[").unwrap_err(); + assert!( + matches!(err, Error::ModuleError(_)), + "invalid globs surface as module errors, got {err:?}" + ); + } + + #[test] + fn compile_ignore_accepts_an_empty_pattern_list() { + assert!(!compile_ignore(&[]).unwrap().is_ignored(Path::new("a.md"))); + } + + #[test] + fn compile_ignore_applies_each_pattern() { + let matcher = compile_ignore(&["node_modules/**".to_string()]).unwrap(); + assert!(matcher.is_ignored(Path::new("node_modules/a.js"))); + assert!(!matcher.is_ignored(Path::new("docs/a.md"))); + } + + #[test] + fn compile_ignore_rejects_an_invalid_pattern() { + let err = compile_ignore(&["[".to_string()]) + .err() + .expect("an invalid pattern must be rejected"); + assert!(matches!(err, Error::ModuleError(_)), "got {err:?}"); + } + + #[test] + fn parse_gitignore_reads_both_switches() { + assert!(parse_gitignore("gitignore").unwrap()); + assert!(!parse_gitignore("no-gitignore").unwrap()); + } + + #[test] + fn parse_gitignore_rejects_an_unknown_switch() { + let err = parse_gitignore("sometimes") + .err() + .expect("an unknown switch must be rejected"); + assert!(matches!(err, Error::ModuleError(_)), "got {err:?}"); + assert!(err.to_string().contains("no-gitignore"), "got: {err}"); + } + + #[test] + fn scan_cost_is_the_declared_flat_cost() { + assert_eq!(SCAN_COST, 1000.); + } + + #[test] + fn at_eof_is_false_while_rows_remain() { + assert!(!at_eof(0, 2)); + assert!(!at_eof(1, 2)); + } + + #[test] + fn at_eof_is_true_once_past_the_last_row() { + assert!(at_eof(2, 2)); + assert!(at_eof(3, 2)); + } + + #[test] + fn at_eof_is_true_for_an_empty_scan() { + assert!(at_eof(0, 0)); + } + + #[test] + fn rowid_of_tracks_the_cursor_position() { + assert_eq!(rowid_of(0), 0); + assert_eq!(rowid_of(7), 7); + } + + #[test] + fn rowid_of_saturates_rather_than_wrapping() { + assert_eq!(rowid_of(usize::MAX), i64::MAX); + } +} From 993b01132bbe77478eb29811e8e47f47c4a7cc3d Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Wed, 9 Sep 2026 14:45:15 -0400 Subject: [PATCH 2/8] test(rust): assert the scan cost and rowids the vtab scaffold declares Two integration assertions over the shared scaffold: rowids count up from zero in scan order, and a join against a small ordinary table plans as a plain nested loop rather than an automatic index. The second is what the declared flat scan cost buys; left at SQLite's near-infinite default every join path costs the same and the planner reshapes the join around it. --- packages/rust/tests/vtab.rs | 53 +++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/packages/rust/tests/vtab.rs b/packages/rust/tests/vtab.rs index 4ec4e2245..19d957c53 100644 --- a/packages/rust/tests/vtab.rs +++ b/packages/rust/tests/vtab.rs @@ -356,3 +356,56 @@ fn a_pattern_naming_an_ignored_directory_still_scans_it() { .unwrap(); assert_eq!(count, 1, "skip rules apply below the path you name"); } + +#[test] +fn rowids_count_up_from_zero_in_scan_order() { + let dir = TempDir::new().unwrap(); + fs::write(dir.path().join("a.md"), "x").unwrap(); + fs::write(dir.path().join("b.md"), "x").unwrap(); + fs::write(dir.path().join("c.md"), "x").unwrap(); + let conn = open_over(&dir, "**/*"); + + let mut stmt = conn + .prepare("SELECT rowid FROM t ORDER BY basename") + .unwrap(); + let rowids: Vec = stmt + .query_map([], |r| r.get(0)) + .unwrap() + .map(Result::unwrap) + .collect(); + assert_eq!(rowids, vec![0, 1, 2]); +} + +/// The declared scan cost is what keeps SQLite from treating the table as +/// near-infinitely expensive. Left at the default, every join path costs the +/// same and the planner falls back to building an automatic index over the +/// small ordinary table instead of a plain nested loop. +#[test] +fn a_join_against_a_small_table_plans_as_a_plain_scan() { + let dir = TempDir::new().unwrap(); + fs::write(dir.path().join("a.md"), "x").unwrap(); + fs::write(dir.path().join("b.md"), "x").unwrap(); + let conn = open_over(&dir, "**/*"); + conn.execute_batch( + "CREATE TABLE tags(name TEXT, tag TEXT); + INSERT INTO tags VALUES ('a.md', 'keep');", + ) + .unwrap(); + + let mut stmt = conn + .prepare("EXPLAIN QUERY PLAN SELECT tags.tag FROM t JOIN tags ON tags.name = t.basename") + .unwrap(); + let plan: Vec = stmt + .query_map([], |r| r.get(3)) + .unwrap() + .map(Result::unwrap) + .collect(); + assert!( + plan.iter().any(|step| step == "SCAN tags"), + "expected a plain scan of the ordinary table, got {plan:?}" + ); + assert!( + !plan.iter().any(|step| step.contains("AUTOMATIC")), + "the declared cost must keep SQLite from building an automatic index, got {plan:?}" + ); +} From d44790acfbbcc31cd6e499ea61535abbd0f3d275 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:02:15 -0400 Subject: [PATCH 3/8] e2e attestation for claude/1080-share-vtab-scaffolding --- .../claude-1080-share-vtab-scaffolding.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/python/e2e-attestations/claude-1080-share-vtab-scaffolding.json diff --git a/packages/python/e2e-attestations/claude-1080-share-vtab-scaffolding.json b/packages/python/e2e-attestations/claude-1080-share-vtab-scaffolding.json new file mode 100644 index 000000000..e155de34d --- /dev/null +++ b/packages/python/e2e-attestations/claude-1080-share-vtab-scaffolding.json @@ -0,0 +1,7 @@ +{ + "command": "just test-e2e", + "ran_at": 1788980535, + "exit_code": 0, + "commit": "993b01132bbe77478eb29811e8e47f47c4a7cc3d", + "branch": "claude/1080-share-vtab-scaffolding" +} From a096a6d44077c872fe81f0f46c0e689378213bc6 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:03:15 -0400 Subject: [PATCH 4/8] e2e attestation for claude/1080-share-vtab-scaffolding --- .../claude-1080-share-vtab-scaffolding.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/ts/e2e-attestations/claude-1080-share-vtab-scaffolding.json diff --git a/packages/ts/e2e-attestations/claude-1080-share-vtab-scaffolding.json b/packages/ts/e2e-attestations/claude-1080-share-vtab-scaffolding.json new file mode 100644 index 000000000..8a1b8608c --- /dev/null +++ b/packages/ts/e2e-attestations/claude-1080-share-vtab-scaffolding.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1788980595, + "exit_code": 0, + "commit": "d44790acfbbcc31cd6e499ea61535abbd0f3d275", + "branch": "claude/1080-share-vtab-scaffolding" +} From 51555d8d0ad35095d69c1568e4478f34feb36e09 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:33:30 -0400 Subject: [PATCH 5/8] test(rust): step the vtab scaffold cursor directly and bound the empty-root queries Five cargo-mutants mutants in vtab_scaffold.rs (at_eof -> false, >= -> <, next -> Ok(()), += -> *=, eof -> false) make the cursor never reach EOF. Nothing killed them except the OOM killer: the lib tests that run a real SELECT through Db::query collected an infinite row stream until the runner died, which cargo-mutants counted as caught. Those four lib tests query an empty root, so only the eof mutants looped there; the next mutants passed the lib binary and hung in the integration binaries instead. A direct ScaffoldCursor test over a two-row fake source now fails all five in the lib binary in microseconds, and cargo stops before the integration binaries run. LIMIT 1 on the empty-root queries bounds them under the eof mutants either way. Refs #1113 --- packages/rust/src/db.rs | 4 +-- packages/rust/src/lib.rs | 4 +-- packages/rust/src/vtab_scaffold.rs | 47 ++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 4 deletions(-) diff --git a/packages/rust/src/db.rs b/packages/rust/src/db.rs index 16d3bea1f..7cf7308c8 100644 --- a/packages/rust/src/db.rs +++ b/packages/rust/src/db.rs @@ -2340,7 +2340,7 @@ mod tests { let mut db = Db::new().unwrap(); db.set_path_table_root(PathBuf::from("/nonexistent-dirsql-root")); - let rows = db.query("SELECT path FROM './'").unwrap(); + let rows = db.query("SELECT path FROM './' LIMIT 1").unwrap(); assert!(rows.is_empty(), "an empty root yields no rows: {rows:?}"); } @@ -2361,7 +2361,7 @@ mod tests { db.set_path_table_root(PathBuf::from("/nonexistent-dirsql-root")); let rows = db - .query("SELECT path FROM '/nonexistent-dirsql-dir/*.log'") + .query("SELECT path FROM '/nonexistent-dirsql-dir/*.log' LIMIT 1") .unwrap(); assert!( diff --git a/packages/rust/src/lib.rs b/packages/rust/src/lib.rs index 45ca65443..90ec0af22 100644 --- a/packages/rust/src/lib.rs +++ b/packages/rust/src/lib.rs @@ -3290,7 +3290,7 @@ mod internal_tests { let db = DirSQL::builder().root(dir.path()).build().unwrap(); assert!( - db.query("SELECT path FROM './'").is_ok(), + db.query("SELECT path FROM './' LIMIT 1").is_ok(), "the path-table fallback must be armed on an ephemeral db" ); } @@ -3331,7 +3331,7 @@ mod internal_tests { .unwrap(); assert!( - db.query("SELECT path FROM './'").is_ok(), + db.query("SELECT path FROM './' LIMIT 1").is_ok(), "the persist branch must arm the fallback too" ); } diff --git a/packages/rust/src/vtab_scaffold.rs b/packages/rust/src/vtab_scaffold.rs index 3a95b93f5..f83d8b558 100644 --- a/packages/rust/src/vtab_scaffold.rs +++ b/packages/rust/src/vtab_scaffold.rs @@ -328,4 +328,51 @@ mod tests { fn rowid_of_saturates_rather_than_wrapping() { assert_eq!(rowid_of(usize::MAX), i64::MAX); } + + struct FakeSource; + + impl TableSource for FakeSource { + type Row = (); + const NAME: &'static str = "fake"; + + fn connect(_args: &[&[u8]]) -> Result<(String, Self)> { + unreachable!() + } + + fn rows(&self) -> Arc> { + unreachable!() + } + + fn column(&self, _row: &(), _ctx: &mut Context, _i: c_int) -> Result<()> { + unreachable!() + } + } + + fn cursor_over(rows: Vec<()>) -> ScaffoldCursor { + ScaffoldCursor { + base: ffi::sqlite3_vtab_cursor::default(), + source: Arc::new(FakeSource), + rows: Arc::new(rows), + index: 0, + } + } + + #[test] + fn cursor_steps_each_row_then_reaches_eof() { + let mut cursor = cursor_over(vec![(), ()]); + + assert!(!cursor.eof()); + assert_eq!(cursor.rowid().unwrap(), 0); + cursor.next().unwrap(); + assert!(!cursor.eof()); + assert_eq!(cursor.rowid().unwrap(), 1); + cursor.next().unwrap(); + assert!(cursor.eof()); + assert_eq!(cursor.rowid().unwrap(), 2); + } + + #[test] + fn cursor_is_at_eof_over_no_rows() { + assert!(cursor_over(Vec::new()).eof()); + } } From 5d6948ec4659a06a4eebc830dbe0649511e5002f Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:38:14 -0400 Subject: [PATCH 6/8] e2e attestation for claude/1080-share-vtab-scaffolding --- .../e2e-attestations/claude-1080-share-vtab-scaffolding.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/python/e2e-attestations/claude-1080-share-vtab-scaffolding.json b/packages/python/e2e-attestations/claude-1080-share-vtab-scaffolding.json index e155de34d..bca05c70a 100644 --- a/packages/python/e2e-attestations/claude-1080-share-vtab-scaffolding.json +++ b/packages/python/e2e-attestations/claude-1080-share-vtab-scaffolding.json @@ -1,7 +1,7 @@ { "command": "just test-e2e", - "ran_at": 1788980535, + "ran_at": 1789515494, "exit_code": 0, - "commit": "993b01132bbe77478eb29811e8e47f47c4a7cc3d", + "commit": "51555d8d0ad35095d69c1568e4478f34feb36e09", "branch": "claude/1080-share-vtab-scaffolding" } From 0f12a5a3ddf16cd30ddd2dd767694b5a0bbd5369 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:38:49 -0400 Subject: [PATCH 7/8] e2e attestation for claude/1080-share-vtab-scaffolding --- .../e2e-attestations/claude-1080-share-vtab-scaffolding.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ts/e2e-attestations/claude-1080-share-vtab-scaffolding.json b/packages/ts/e2e-attestations/claude-1080-share-vtab-scaffolding.json index 8a1b8608c..08c49618d 100644 --- a/packages/ts/e2e-attestations/claude-1080-share-vtab-scaffolding.json +++ b/packages/ts/e2e-attestations/claude-1080-share-vtab-scaffolding.json @@ -1,7 +1,7 @@ { "command": "pnpm test:e2e", - "ran_at": 1788980595, + "ran_at": 1789515529, "exit_code": 0, - "commit": "d44790acfbbcc31cd6e499ea61535abbd0f3d275", + "commit": "5d6948ec4659a06a4eebc830dbe0649511e5002f", "branch": "claude/1080-share-vtab-scaffolding" } From e5514132e18daf974a340f1c9f713dfc90cb6fc2 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:35:40 -0400 Subject: [PATCH 8/8] chore: retrigger CI on a fresh head sha The stale release-ci check runs left on the previous sha carry version numbers willfire no longer predicts, so CI Gate reports them as unpredicted checks and cannot confirm the set is complete.