From e3fe0e10bf5247366d8287c4f31ef4c2d28a1118 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:33:32 -0400 Subject: [PATCH 01/11] test(rust): content named by a statement is read ahead for every row A statement naming content has every selected row's file read before the first row is delivered, not one file at a time as SQLite steps the cursor. Stepping a statement with the tree deleted after its first row shows which: the rows that follow still carry the text only if it was read ahead. Covers a projection, a WHERE on content, and the side of a join that names it. --- .../rust/tests/path_table_content_prefetch.rs | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 packages/rust/tests/path_table_content_prefetch.rs diff --git a/packages/rust/tests/path_table_content_prefetch.rs b/packages/rust/tests/path_table_content_prefetch.rs new file mode 100644 index 000000000..36477ba9c --- /dev/null +++ b/packages/rust/tests/path_table_content_prefetch.rs @@ -0,0 +1,125 @@ +//! A statement that names `content` has the column read for every row it +//! selects before SQLite delivers the first of them, rather than one file at +//! a time as the cursor is stepped. Real temp trees, real SQLite, a raw +//! connection so the rows can be stepped one at a time with the tree changed +//! in between: a file deleted after the first row still has its content in +//! the rows that follow, which a per-row read could not supply. + +use std::fs; +use std::path::{Path, PathBuf}; + +use dirsql::vtab::{StatementScope, load_module}; +use rusqlite::{Connection, Rows}; +use tempfile::TempDir; + +const PAPERS: usize = 8; + +/// `PAPERS` directories, each holding a `title.md` and an `abstract.md`; +/// the paths of the titles, in scan order. +fn papers(root: &Path) -> Vec { + (0..PAPERS) + .map(|i| { + let dir = root.join(format!("p{i:02}")); + fs::create_dir(&dir).unwrap(); + fs::write(dir.join("abstract.md"), format!("abstract {i}")).unwrap(); + let title = dir.join("title.md"); + fs::write(&title, format!("title {i}")).unwrap(); + title + }) + .collect() +} + +fn titles() -> Vec> { + (0..PAPERS).map(|i| Some(format!("title {i}"))).collect() +} + +/// A connection with path-tables `a` over the abstracts and `t` over the +/// titles under `root`. +fn open(root: &Path) -> Connection { + let conn = Connection::open_in_memory().unwrap(); + load_module(&conn, StatementScope::new()).unwrap(); + for (name, glob) in [("a", "*/abstract.md"), ("t", "*/title.md")] { + conn.execute_batch(&format!( + "CREATE VIRTUAL TABLE {name} USING dirsql_path('{}', '{glob}', '', 'gitignore')", + root.display() + )) + .unwrap(); + } + conn +} + +fn first(rows: &mut Rows<'_>) -> Option { + rows.next().unwrap().expect("a first row").get(0).unwrap() +} + +fn rest(rows: &mut Rows<'_>) -> Vec> { + let mut out = Vec::new(); + while let Some(row) = rows.next().unwrap() { + out.push(row.get(0).unwrap()); + } + out +} + +fn delete(paths: &[PathBuf]) { + for path in paths { + fs::remove_file(path).unwrap(); + } +} + +#[test] +fn content_is_read_for_every_row_before_the_first_is_delivered() { + let root = TempDir::new().unwrap(); + let paths = papers(root.path()); + let conn = open(root.path()); + let mut stmt = conn.prepare("SELECT content FROM t").unwrap(); + let mut rows = stmt.query([]).unwrap(); + + assert_eq!(first(&mut rows).as_deref(), Some("title 0")); + delete(&paths); + + assert_eq!( + rest(&mut rows), + titles()[1..], + "the titles were deleted after the first row; a row read when stepped finds nothing" + ); +} + +#[test] +fn a_where_on_content_has_the_column_read_ahead_too() { + let root = TempDir::new().unwrap(); + let paths = papers(root.path()); + let conn = open(root.path()); + let mut stmt = conn + .prepare("SELECT basename FROM t WHERE content LIKE 'title%'") + .unwrap(); + let mut rows = stmt.query([]).unwrap(); + + assert_eq!(first(&mut rows).as_deref(), Some("title.md")); + delete(&paths); + + assert_eq!( + rest(&mut rows).len(), + PAPERS - 1, + "every title matched before the deletion; a row read when stepped matches nothing" + ); +} + +#[test] +fn a_join_has_the_content_it_names_read_ahead() { + let root = TempDir::new().unwrap(); + let paths = papers(root.path()); + let conn = open(root.path()); + let mut stmt = conn + .prepare("SELECT t.content FROM a JOIN t ON t.dir = a.dir") + .unwrap(); + let mut rows = stmt.query([]).unwrap(); + + assert_eq!(first(&mut rows).as_deref(), Some("title 0")); + delete(&paths); + + assert_eq!( + rest(&mut rows), + titles()[1..], + "the titles were deleted after the first joined row; a side read as it is probed finds nothing" + ); +} From 8e6dc1dcedcd90dc67e80a7189141469031da8e5 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:43:12 -0400 Subject: [PATCH 02/11] perf(rust): read a path table's content ahead, in parallel, when a statement names it A statement that names content had each row's file read on the SQLite cursor thread as the row was emitted: five syscalls per file, one file at a time, 28% of the join speed-of-light query. Now xBestIndex reads colUsed, and a filter whose statement names content reads the file of every row it selected before SQLite steps the first, eight files at a time; a statement that does not name it reads nothing, as before. The read is also sized from the size the scan already stat'ed, so it opens, reads and closes without stat'ing again. A lookup on a table whose content is named is declared dearer, so a join scans that side and probes the other: read ahead, the column covers the whole table at once rather than one probe at a time. --- PARITY.md | 8 + docs/reference/path-tables.md | 14 +- .../2026-10-02-path-table-content-prefetch.md | 1 + .../2026-10-02-path-table-content-prefetch.md | 37 +++ packages/rust/src/vtab.rs | 233 +++++++++++++++++- packages/rust/src/vtab_scaffold.rs | 223 ++++++++++++++++- testing-conventions.toml | 2 +- 7 files changed, 493 insertions(+), 25 deletions(-) create mode 100644 packages/rust/changelog.d/2026-10-02-path-table-content-prefetch.md create mode 100644 packages/rust/migrations.d/2026-10-02-path-table-content-prefetch.md diff --git a/PARITY.md b/PARITY.md index 30d7e3f3a..edfcf546f 100644 --- a/PARITY.md +++ b/PARITY.md @@ -305,6 +305,14 @@ names come from the SQLite authorizer at the final prepare, so the fallback loop above stays the only thing that registers a table. `StatementScope::warm` lives in the shared core; no SDK signature changes. +**Content read ahead in parallel (#1241) — Rust-crate-only surface, no SDK +drift.** A statement that names a path table's `content` has the column read +for every row it selects at once, eight files at a time, before SQLite steps +the first row; a statement that does not name it reads no file, as before. The +read-ahead and the planner cost that keeps a content-bearing table on the +scanned side of a join live in the shared core, so all three SDKs and the CLI +inherit it with no per-binding surface and no signature changes. + ## AsyncDirSQL | API | Python | Rust | diff --git a/docs/reference/path-tables.md b/docs/reference/path-tables.md index 0a73bfa85..fef0810bd 100644 --- a/docs/reference/path-tables.md +++ b/docs/reference/path-tables.md @@ -180,12 +180,14 @@ A path-table is scanned when the statement runs, so it always reflects the filesystem as it is *now* — unlike declared tables, which are indexed on build and updated by the watcher. A file created a moment ago shows up immediately. -The scan is live all the way down to `content`: a file's body is read when the -query names the `content` column, not when the row is discovered. A file -deleted *after* the scan finds it but *before* its `content` is read yields -`NULL` content — the same NULL an unreadable or non-UTF-8 file gives — rather -than failing the query. This is an accepted consequence of reading live, not a -bug to design around. +The scan is live all the way down to `content`: a file's body is read only +when the query names the `content` column, not when the row is discovered. A +query that names it has the bodies of the rows it selects read all at once, +several files at a time, before the first row comes back. A file deleted +*after* the scan finds it but *before* its `content` is read yields `NULL` +content — the same NULL an unreadable or non-UTF-8 file gives — rather than +failing the query. This is an accepted consequence of reading live, not a bug +to design around. The table itself is per-connection: it lives in `temp`, so it cannot leak into `sqlite_master` or survive a restart. Under `--persist` a *parsed* table's rows diff --git a/packages/rust/changelog.d/2026-10-02-path-table-content-prefetch.md b/packages/rust/changelog.d/2026-10-02-path-table-content-prefetch.md new file mode 100644 index 000000000..03c849eed --- /dev/null +++ b/packages/rust/changelog.d/2026-10-02-path-table-content-prefetch.md @@ -0,0 +1 @@ +**Changed** A statement that names a path table's `content` column -- in its projection, its `WHERE`, or across a join -- has the file bodies of the rows it selects read ahead, eight files at a time, before SQLite steps the first row, instead of one file at a time as each row is emitted. Each read is also one stat cheaper: the buffer is sized from the size the scan already recorded. A statement that does not name `content` still reads no file, `SELECT *` still excludes it, and an unreadable or non-UTF-8 file is still a `NULL` cell. In a join, the planner now scans the table whose `content` is named and probes the other, so the read-ahead covers the whole column rather than one probe at a time. diff --git a/packages/rust/migrations.d/2026-10-02-path-table-content-prefetch.md b/packages/rust/migrations.d/2026-10-02-path-table-content-prefetch.md new file mode 100644 index 000000000..c8fcd199d --- /dev/null +++ b/packages/rust/migrations.d/2026-10-02-path-table-content-prefetch.md @@ -0,0 +1,37 @@ +### Core: a path table's `content` is read ahead when a statement names it + +#### Summary + +No API changes. A statement that names `content` has the bodies of the rows it +selects read together, several files at a time, before its first row comes +back; one that does not name it reads no file, as before. + +#### Required changes + +_None._ + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +- A file deleted after a statement's scan found it is still readable to that + statement if the read-ahead reached it first; it used to yield `NULL` + whenever the deletion came before the row was emitted. Either way the next + statement sees the file gone. +- A statement that names `content` holds the text of every row it selects + for its own duration, where before it held one row's text at a time. +- In a join, the planner scans the path table whose `content` is named and + probes the other, where before it probed whichever came second. + +#### Verification + +```bash +cd "$(mktemp -d)" +mkdir -p p1 p2 +printf 'one\n' > p1/title.md +printf 'two\n' > p2/title.md +dirsql query "SELECT content FROM './*/title.md' ORDER BY path" +# expected: [{"content":"one\n"},{"content":"two\n"}] -- both bodies, read together +``` diff --git a/packages/rust/src/vtab.rs b/packages/rust/src/vtab.rs index a135bac95..a767d91ca 100644 --- a/packages/rust/src/vtab.rs +++ b/packages/rust/src/vtab.rs @@ -1,7 +1,9 @@ use std::ffi::c_int; -use std::fs; +use std::fs::{self, File}; +use std::io::{self, Read}; use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, OnceLock}; use std::thread; use std::time::{SystemTime, UNIX_EPOCH}; @@ -65,13 +67,71 @@ fn column_type(column: &str) -> &'static str { } } +/// Files read at once when `content` is read ahead. Each read is a few +/// syscalls the kernel answers independently of the last, so they overlap +/// almost perfectly; past eight the box, not the count, sets the pace. +const READERS: usize = 8; + /// Read `path` as text, yielding `None` when it is unreadable or not valid /// UTF-8. A file that cannot be read is a NULL cell, never a failed row: the /// filesystem is allowed to be messy and a query over it should still return. -fn read_text(path: &Path) -> Option { - std::fs::read(path) - .ok() - .and_then(|b| String::from_utf8(b).ok()) +/// `size` is what the scan's stat reported, so the buffer is sized without +/// stat'ing again; a file that has grown since is still read whole. +fn read_text(path: &Path, size: Option) -> Option { + let mut file = File::open(path).ok()?; + let bytes = read_all(&mut file, presize(size)).ok()?; + String::from_utf8(bytes).ok() +} + +/// One past the size the scan saw: a file unchanged since fills the buffer in +/// one read and reports its end on the next, with no stat in between. +fn presize(size: Option) -> usize { + usize::try_from(size.unwrap_or(0)) + .unwrap_or(0) + .saturating_add(1) +} + +/// Every byte `reader` yields, into a buffer of `expected` bytes that grows +/// if the reader has more. +fn read_all(reader: &mut impl Read, expected: usize) -> io::Result> { + let mut buf = Vec::new(); + buf.try_reserve_exact(expected.max(1))?; + buf.resize(expected.max(1), 0); + let mut filled = 0; + loop { + if filled == buf.len() { + buf.resize(buf.len() * 2, 0); + } + match reader.read(&mut buf[filled..]) { + Ok(0) => break, + Ok(n) => filled += n, + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + buf.truncate(filled); + Ok(buf) +} + +/// Read `content` for every row in `rows` that has none yet, [`READERS`] +/// files at a time, each reader taking the next unread row. `read` is +/// injected so the sharing-out is testable without a filesystem. +fn read_contents(rows: &[&FileRow], read: &(dyn Fn(&FileRow) -> Option + Sync)) { + let unread: Vec<&FileRow> = rows + .iter() + .copied() + .filter(|row| row.content.get().is_none()) + .collect(); + let next = AtomicUsize::new(0); + thread::scope(|scope| { + for _ in 0..READERS.min(unread.len()) { + scope.spawn(|| { + while let Some(row) = unread.get(next.fetch_add(1, Ordering::Relaxed)) { + row.content.get_or_init(|| read(row)); + } + }); + } + }); } /// Everything a scan needs: where to walk, what to match, what to call the @@ -97,6 +157,10 @@ struct FileRow { path: String, spans: PathSpans, facts: StatFacts, + /// The file's text once a statement has asked for it, read at most once + /// per statement: ahead for every row when the statement names the + /// column, or on demand for the row being emitted. + content: OnceLock>, } impl FileRow { @@ -108,6 +172,7 @@ impl FileRow { path, spans, facts, + content: OnceLock::new(), } } @@ -217,8 +282,8 @@ fn integer(n: Option) -> ValueRef<'static> { n.map_or(ValueRef::Null, ValueRef::Integer) } -/// The cell a row holds for column `i`, `None` for the one it does not hold: -/// `content` is read from the file when asked for, never stored. +/// The cell a row holds for column `i`, `None` for the one it does not hold +/// until asked: `content` is read from the file, not stat'ed with the rest. fn cell(row: &FileRow, i: c_int) -> Option> { Some(match usize::try_from(i).unwrap_or(usize::MAX) { PATH_COLUMN => ValueRef::from(row.path.as_str()), @@ -258,6 +323,12 @@ fn parse_module_args(args: &[&[u8]]) -> Result { }) } +impl ScanSpec { + fn read(&self, row: &FileRow) -> Option { + read_text(&self.root.join(&row.rel), row.facts.size) + } +} + /// The string a matched file is reported under: the relative path as scanned, /// under the table's path prefix when it has one. fn reported_path(path_prefix: &Path, rel_path: &Path) -> String { @@ -301,10 +372,16 @@ impl TableSource for ScanSpec { Some(value) => ctx.set_result(&ToSqlOutput::Borrowed(value)), // The one effectful read, reached only when a query names the // column: this is where laziness actually lives. - None => ctx.set_result(&read_text(&self.root.join(&row.rel))), + None => ctx.set_result(row.content.get_or_init(|| self.read(row))), } } + const PREFETCH_COLUMN: Option = Some(CONTENT_COLUMN); + + fn prefetch(&self, rows: &[&FileRow]) { + read_contents(rows, &|row| self.read(row)); + } + fn lookup_key<'r>(&self, row: &'r FileRow, column: usize) -> Option<&'r str> { match column { PATH_COLUMN => Some(&row.path), @@ -629,6 +706,144 @@ mod tests { assert!(rows.is_empty()); } + #[test] + fn presize_is_one_past_the_size_the_scan_saw() { + assert_eq!(presize(Some(0)), 1); + assert_eq!(presize(Some(41)), 42); + } + + #[test] + fn presize_is_one_byte_without_a_usable_size() { + assert_eq!(presize(None), 1); + assert_eq!(presize(Some(-1)), 1); + } + + /// A reader that yields `chunks` one call at a time, `interrupted` times + /// interrupted first. + struct Chunked { + chunks: Vec>, + interrupted: usize, + } + + impl Read for Chunked { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.interrupted > 0 { + self.interrupted -= 1; + return Err(io::Error::from(io::ErrorKind::Interrupted)); + } + if self.chunks.is_empty() { + return Ok(0); + } + let chunk = self.chunks.remove(0); + let n = chunk.len().min(buf.len()); + buf[..n].copy_from_slice(&chunk[..n]); + if n < chunk.len() { + self.chunks.insert(0, chunk[n..].to_vec()); + } + Ok(n) + } + } + + #[test] + fn read_all_takes_every_byte_a_presized_read_yields() { + let mut reader = Chunked { + chunks: vec![b"hello".to_vec()], + interrupted: 0, + }; + assert_eq!(read_all(&mut reader, 6).unwrap(), b"hello"); + } + + #[test] + fn read_all_grows_past_a_size_the_file_has_outgrown() { + let mut reader = Chunked { + chunks: vec![b"hello".to_vec(), b" world".to_vec()], + interrupted: 0, + }; + assert_eq!(read_all(&mut reader, 2).unwrap(), b"hello world"); + let mut reader = Chunked { + chunks: vec![b"hello".to_vec()], + interrupted: 0, + }; + assert_eq!(read_all(&mut reader, 0).unwrap(), b"hello"); + } + + #[test] + fn read_all_retries_an_interrupted_read() { + let mut reader = Chunked { + chunks: vec![b"hi".to_vec()], + interrupted: 2, + }; + assert_eq!(read_all(&mut reader, 3).unwrap(), b"hi"); + } + + #[test] + fn read_all_surfaces_a_failed_read() { + struct Broken; + impl Read for Broken { + fn read(&mut self, _buf: &mut [u8]) -> io::Result { + Err(io::Error::from(io::ErrorKind::PermissionDenied)) + } + } + let err = read_all(&mut Broken, 4).unwrap_err(); + assert_eq!(err.kind(), io::ErrorKind::PermissionDenied); + } + + #[test] + fn read_all_yields_nothing_for_an_empty_reader() { + let mut reader = Chunked { + chunks: Vec::new(), + interrupted: 0, + }; + assert_eq!(read_all(&mut reader, 1).unwrap(), b""); + } + + #[test] + fn read_contents_reads_each_unread_row_once() { + let rows: Vec = (0..40).map(|n| row_for("", &format!("f{n}.md"))).collect(); + rows[3].content.set(Some("already".to_string())).unwrap(); + let reads = std::sync::Mutex::new(Vec::new()); + let read = |row: &FileRow| { + reads.lock().unwrap().push(row.path.clone()); + Some(format!("body of {}", row.path)) + }; + + read_contents(&rows.iter().collect::>(), &read); + + let mut reads = reads.into_inner().unwrap(); + reads.sort(); + let mut expected: Vec = (0..40) + .filter(|&n| n != 3) + .map(|n| format!("f{n}.md")) + .collect(); + expected.sort(); + assert_eq!( + reads, expected, + "every row but the one already read, once each" + ); + assert_eq!(rows[3].content.get().unwrap().as_deref(), Some("already")); + assert_eq!( + rows[7].content.get().unwrap().as_deref(), + Some("body of f7.md") + ); + } + + #[test] + fn read_contents_keeps_a_null_read() { + let rows = [row_for("", "a.md")]; + read_contents(&[&rows[0]], &|_| None); + assert_eq!(rows[0].content.get(), Some(&None)); + } + + #[test] + fn read_contents_reads_nothing_when_every_row_is_read() { + let rows = [row_for("", "a.md")]; + rows[0].content.set(None).unwrap(); + read_contents(&[&rows[0]], &|_| { + panic!("a row read already is not read again") + }); + read_contents(&[], &|_| panic!("nothing to read")); + } + #[test] fn a_nested_path_splits_into_dir_basename_and_ext() { let row = row_for("", "nested/sub.txt"); diff --git a/packages/rust/src/vtab_scaffold.rs b/packages/rust/src/vtab_scaffold.rs index 5336ce01c..209e8fd86 100644 --- a/packages/rust/src/vtab_scaffold.rs +++ b/packages/rust/src/vtab_scaffold.rs @@ -33,9 +33,20 @@ const SCAN_COST: f64 = 1000.; /// that SQLite puts a table it can probe on the inner side of a join. const LOOKUP_COST: f64 = 10.; +/// Cost of a lookup when the statement names the source's prefetch column. +/// Dearer than [`LOOKUP_COST`] so that, of two tables a join could probe, +/// the planner probes the one whose cells are all held and scans this one, +/// where the column is read ahead for every row at once; probed, it would be +/// read one row at a time. +const PREFETCH_LOOKUP_COST: f64 = 100.; + /// `idxNum` for a full scan; a lookup carries its column as `column + 1`. const SCAN_IDX: c_int = 0; +/// Set in `idxNum` when the statement names the prefetch column; the low bits +/// still name the lookup. +const PREFETCH_IDX: c_int = 1 << 30; + /// What a `dirsql_*` table supplies on top of the scaffolding. pub trait TableSource: Sized + Send + Sync + 'static { /// One row, as the table models it. @@ -69,6 +80,17 @@ pub trait TableSource: Sized + Send + Sync + 'static { /// Emit column `i` of `row`. fn column(&self, row: &Self::Row, ctx: &mut Context, i: c_int) -> Result<()>; + + /// A column a row does not hold but reads on demand, which a statement + /// naming it has read ahead for every row a filter selects, all at once + /// rather than one row at a time as SQLite steps the cursor. + const PREFETCH_COLUMN: Option = None; + + /// Read [`Self::PREFETCH_COLUMN`] for `rows`. Only reached when the + /// column is declared, so a source that declares none is never asked. + fn prefetch(&self, rows: &[&Self::Row]) { + let _ = rows; + } } /// The row sets the tables on one connection hold across a statement. @@ -297,9 +319,45 @@ fn lookup_idx_num(column: usize) -> c_int { .unwrap_or(SCAN_IDX) } +/// The cost a lookup is declared to carry, given whether the statement names +/// the prefetch column. +fn lookup_cost(prefetch: bool) -> f64 { + if prefetch { + PREFETCH_LOOKUP_COST + } else { + LOOKUP_COST + } +} + +/// The `idxNum` for a plan: the lookup it probes, if any, and whether the +/// statement names the prefetch column. +fn plan_idx_num(lookup: Option, prefetch: bool) -> c_int { + let lookup = lookup.map_or(SCAN_IDX, lookup_idx_num); + if prefetch { + lookup | PREFETCH_IDX + } else { + lookup + } +} + /// The lookup column an `idxNum` names; `None` for a full scan. fn lookup_column(idx_num: c_int) -> Option { - usize::try_from(idx_num).ok()?.checked_sub(1) + usize::try_from(idx_num & !PREFETCH_IDX) + .ok()? + .checked_sub(1) +} + +/// Whether an `idxNum` asks for the prefetch column to be read ahead. +fn prefetches(idx_num: c_int) -> bool { + idx_num & PREFETCH_IDX != 0 +} + +/// Whether a statement's `colUsed` mask names `column`. +fn names_column(col_used: u64, column: usize) -> bool { + u32::try_from(column) + .ok() + .and_then(|column| 1u64.checked_shl(column)) + .is_some_and(|bit| col_used & bit != 0) } /// The text of a filter argument; `None` when it is not text, in which case @@ -413,14 +471,16 @@ unsafe impl<'vtab, S: TableSource> VTab<'vtab> for ScaffoldTab { }) .collect(); + let prefetch = + S::PREFETCH_COLUMN.is_some_and(|column| names_column(info.col_used(), column)); match choose_lookup(S::LOOKUP_COLUMNS, &constraints) { Some(lookup) => { info.constraint_usage(lookup.constraint).set_argv_index(1); - info.set_idx_num(lookup_idx_num(lookup.column)); - info.set_estimated_cost(LOOKUP_COST); + info.set_idx_num(plan_idx_num(Some(lookup.column), prefetch)); + info.set_estimated_cost(lookup_cost(prefetch)); } None => { - info.set_idx_num(SCAN_IDX); + info.set_idx_num(plan_idx_num(None, prefetch)); info.set_estimated_cost(SCAN_COST); } } @@ -486,6 +546,16 @@ impl ScaffoldCursor { let row = self.rows.as_ref()?.get(position)?; Some((position, row)) } + + /// Read the prefetch column ahead for every row the filter selected. + fn prefetch(&self) { + let Some(rows) = &self.rows else { return }; + let selected: Vec<&S::Row> = (0..self.selection.len()) + .filter_map(|step| self.selection.position(step)) + .filter_map(|position| rows.get(position)) + .collect(); + self.source.prefetch(&selected); + } } #[expect(unsafe_code, reason = "rusqlite requires an unsafe trait impl")] @@ -497,6 +567,9 @@ unsafe impl VTabCursor for ScaffoldCursor { _ => Selection::Every(self.rows().len()), }; self.step = 0; + if prefetches(idx_num) { + self.prefetch(); + } Ok(()) } @@ -753,6 +826,53 @@ mod tests { assert_eq!(lookup_column(-1), None); } + #[test] + fn a_plan_without_prefetch_is_the_bare_lookup_or_scan() { + assert_eq!(plan_idx_num(None, false), SCAN_IDX); + assert_eq!(plan_idx_num(Some(2), false), lookup_idx_num(2)); + assert!(!prefetches(plan_idx_num(Some(2), false))); + assert!(!prefetches(SCAN_IDX)); + } + + #[test] + fn a_plan_with_prefetch_keeps_its_lookup_and_says_so() { + for lookup in [None, Some(0), Some(2), Some(7)] { + let idx_num = plan_idx_num(lookup, true); + assert!(prefetches(idx_num), "{lookup:?}"); + assert_eq!(lookup_column(idx_num), lookup); + assert_ne!(idx_num, plan_idx_num(lookup, false)); + } + assert_eq!(plan_idx_num(None, true), PREFETCH_IDX); + } + + #[test] + fn names_column_reads_the_bit_of_the_column() { + assert!(names_column(1 << 7, 7)); + assert!(names_column(u64::MAX, 7)); + assert!(!names_column(1 << 6, 7)); + assert!(!names_column(0, 7)); + assert!(!names_column(0, 0)); + assert!(names_column(1, 0)); + } + + #[test] + fn names_column_is_false_past_the_mask() { + assert!(!names_column(u64::MAX, 64)); + assert!(!names_column(u64::MAX, usize::MAX)); + } + + #[test] + fn a_lookup_costs_more_when_the_statement_prefetches() { + assert_eq!(lookup_cost(false), LOOKUP_COST); + assert_eq!(lookup_cost(true), PREFETCH_LOOKUP_COST); + } + + #[test] + fn a_source_declares_no_prefetch_column_by_default() { + assert_eq!(FakeSource::PREFETCH_COLUMN, None); + FakeSource.prefetch(&[&(), &()]); + } + #[test] fn text_key_reads_a_text_argument() { assert_eq!(text_key(ValueRef::Text(b"docs")), Some("docs")); @@ -1056,6 +1176,12 @@ mod tests { /// Reads of the `initial` column through SQLite on this thread: one /// per row the cursor hands back, since SQLite re-checks the equality. static INITIAL_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// The rows whose `shout` column was read ahead on this thread. + static PREFETCHED: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; + } + + fn prefetched() -> Vec<&'static str> { + PREFETCHED.with(|rows| rows.borrow().clone()) } impl TableSource for Words { @@ -1069,7 +1195,10 @@ mod tests { scans: std::sync::atomic::AtomicUsize::new(0), keys: std::sync::atomic::AtomicUsize::new(0), }; - Ok(("CREATE TABLE x(word TEXT, initial TEXT)".to_string(), words)) + Ok(( + "CREATE TABLE x(word TEXT, initial TEXT, shout TEXT)".to_string(), + words, + )) } fn rows(&self) -> Arc> { @@ -1079,11 +1208,20 @@ mod tests { } fn column(&self, row: &&'static str, ctx: &mut Context, i: c_int) -> Result<()> { - if i == 0 { - return ctx.set_result(&Value::Text(row.to_string())); + match i { + 0 => ctx.set_result(&Value::Text(row.to_string())), + 1 => { + INITIAL_READS.with(|reads| reads.set(reads.get() + 1)); + ctx.set_result(&Value::Text(row[..1].to_string())) + } + _ => ctx.set_result(&Value::Text(row.to_uppercase())), } - INITIAL_READS.with(|reads| reads.set(reads.get() + 1)); - ctx.set_result(&Value::Text(row[..1].to_string())) + } + + const PREFETCH_COLUMN: Option = Some(2); + + fn prefetch(&self, rows: &[&&'static str]) { + PREFETCHED.with(|seen| seen.borrow_mut().extend(rows.iter().map(|row| **row))); } fn lookup_key<'r>(&self, row: &'r &'static str, column: usize) -> Option<&'r str> { @@ -1306,4 +1444,71 @@ mod tests { assert_eq!(INITIAL_READS.with(std::cell::Cell::get), 2); } + + #[test] + fn a_statement_naming_the_prefetch_column_has_it_read_ahead_for_every_scanned_row() { + let conn = words_table(); + PREFETCHED.with(|seen| seen.borrow_mut().clear()); + + assert_eq!( + strings(&conn, "SELECT shout FROM w"), + vec!["ANT", "BAT", "BEE"] + ); + assert_eq!(prefetched(), vec!["ant", "bat", "bee"]); + } + + #[test] + fn a_lookup_has_the_prefetch_column_read_ahead_for_the_rows_it_selects() { + let conn = words_table(); + PREFETCHED.with(|seen| seen.borrow_mut().clear()); + + assert_eq!( + strings(&conn, "SELECT shout FROM w WHERE initial = 'b'"), + vec!["BAT", "BEE"] + ); + assert_eq!(prefetched(), vec!["bat", "bee"]); + assert_eq!( + plan(&conn, "SELECT shout FROM w WHERE initial = 'b'"), + vec![format!("SCAN w VIRTUAL TABLE INDEX {}:", 2 | PREFETCH_IDX)] + ); + } + + #[test] + fn a_where_on_the_prefetch_column_names_it_too() { + let conn = words_table(); + PREFETCHED.with(|seen| seen.borrow_mut().clear()); + + assert_eq!( + strings(&conn, "SELECT word FROM w WHERE shout = 'BAT'"), + vec!["bat"] + ); + assert_eq!(prefetched(), vec!["ant", "bat", "bee"]); + } + + #[test] + fn a_statement_not_naming_the_prefetch_column_reads_nothing_ahead() { + let conn = words_table(); + PREFETCHED.with(|seen| seen.borrow_mut().clear()); + + strings(&conn, "SELECT word FROM w WHERE initial = 'b'"); + strings(&conn, "SELECT word || initial FROM w"); + + assert_eq!(prefetched(), Vec::<&str>::new()); + } + + #[test] + fn sqlite_scans_the_side_of_a_join_that_prefetches_and_probes_the_other() { + let conn = words_table(); + + assert_eq!( + plan( + &conn, + "SELECT y.shout FROM w x JOIN w y ON y.initial = x.initial" + ), + vec![ + format!("SCAN y VIRTUAL TABLE INDEX {PREFETCH_IDX}:"), + "SCAN x VIRTUAL TABLE INDEX 2:".to_string() + ] + ); + } } diff --git a/testing-conventions.toml b/testing-conventions.toml index 79376aa46..6fad9221f 100644 --- a/testing-conventions.toml +++ b/testing-conventions.toml @@ -125,7 +125,7 @@ reason = "a child's depth decides the reserved top-level .dirsql rule for a real [[rust.exempt]] path = "src/vtab.rs" rules = ["mutation"] -lines = [289, 302] +lines = [360, 371] reason = "rows walks and stats a real tree and column hands a cell to SQLite; the cell choice is unit-tested in `cell`, and tests/vtab.rs stat_columns_carry_real_values and content_is_excluded_from_star_but_selectable_by_name assert both through a real connection" [[rust.exempt]] From d1787b49c870a3291198bafc0c03eea7fe77492a Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:51:13 -0400 Subject: [PATCH 03/11] test(rust): kill the read-ahead mutants the unit tier can reach, waive the rest read_all now has to grow geometrically for an outgrown file, and the default prefetch is an empty body with nothing to mutate. The file reads and the read-ahead itself are integration-tier behavior, and the bitwise-or in plan_idx_num cannot differ from xor on any column index. --- packages/rust/src/vtab.rs | 20 ++++++++++++++++++++ packages/rust/src/vtab_scaffold.rs | 4 +--- testing-conventions.toml | 12 ++++++++++++ 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/packages/rust/src/vtab.rs b/packages/rust/src/vtab.rs index a767d91ca..e3cbfad28 100644 --- a/packages/rust/src/vtab.rs +++ b/packages/rust/src/vtab.rs @@ -767,6 +767,26 @@ mod tests { assert_eq!(read_all(&mut reader, 0).unwrap(), b"hello"); } + struct Counting(R, usize); + + impl Read for Counting { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.1 += 1; + self.0.read(buf) + } + } + + #[test] + fn read_all_grows_geometrically_when_a_file_has_outgrown_its_size() { + let chunked = Chunked { + chunks: vec![vec![b'x'; 4096]], + interrupted: 0, + }; + let mut reader = Counting(chunked, 0); + assert_eq!(read_all(&mut reader, 1).unwrap().len(), 4096); + assert!(reader.1 <= 16, "{} reads", reader.1); + } + #[test] fn read_all_retries_an_interrupted_read() { let mut reader = Chunked { diff --git a/packages/rust/src/vtab_scaffold.rs b/packages/rust/src/vtab_scaffold.rs index 209e8fd86..e0ed3307c 100644 --- a/packages/rust/src/vtab_scaffold.rs +++ b/packages/rust/src/vtab_scaffold.rs @@ -88,9 +88,7 @@ pub trait TableSource: Sized + Send + Sync + 'static { /// Read [`Self::PREFETCH_COLUMN`] for `rows`. Only reached when the /// column is declared, so a source that declares none is never asked. - fn prefetch(&self, rows: &[&Self::Row]) { - let _ = rows; - } + fn prefetch(&self, _rows: &[&Self::Row]) {} } /// The row sets the tables on one connection hold across a statement. diff --git a/testing-conventions.toml b/testing-conventions.toml index 6fad9221f..a7f66fcfe 100644 --- a/testing-conventions.toml +++ b/testing-conventions.toml @@ -134,6 +134,18 @@ rules = ["mutation"] lines = [351, 392] reason = "run_default and run_query build a real index and drive stdout, stderr and the process exit; tests/on_file_failure_e2e.rs (a_clean_scan_exits_zero, a_failed_command_fails_the_run) and tests/repl_pipe_e2e.rs assert their codes through the built binary" +[[rust.exempt]] +path = "src/vtab.rs" +rules = ["mutation"] +lines = [82, 333, 393] +reason = "read_text and ScanSpec::read open a real file and prefetch reads a real tree ahead of the cursor; the buffer logic is unit-tested in `read_all` and `presize`, tests/vtab.rs content_is_excluded_from_star_but_selectable_by_name, content_is_null_for_non_utf8_files and unreadable_file_yields_null_content_without_erroring_the_row assert the cell, and tests/path_table_content_prefetch.rs content_is_read_for_every_row_before_the_first_is_delivered and a_join_has_the_content_it_names_read_ahead assert the read-ahead" + +[[rust.exempt]] +path = "src/vtab_scaffold.rs" +rules = ["mutation"] +lines = [280] +reason = "`|` -> `^` is equivalent: the lookup idx_num is a column index plus one, and SQLite caps a table at 32767 columns, so it never carries the PREFETCH_IDX bit and the two operators agree on every input" + # Unit-only coverage floor: measured by `cargo llvm-cov --lib --features cli` # (the `--lib` scoping keeps the integration tier under packages/rust/tests/ out # of the number -- testing-conventions #269). `branch` runs under `--branch`, From 7ba22717c1fef22c5516bb8c8036db7a4ffced88 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:59:30 -0400 Subject: [PATCH 04/11] e2e attestation for claude/1241-content-prefetch --- .../e2e-attestations/claude-1241-content-prefetch.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/python/e2e-attestations/claude-1241-content-prefetch.json diff --git a/packages/python/e2e-attestations/claude-1241-content-prefetch.json b/packages/python/e2e-attestations/claude-1241-content-prefetch.json new file mode 100644 index 000000000..962b902ad --- /dev/null +++ b/packages/python/e2e-attestations/claude-1241-content-prefetch.json @@ -0,0 +1,7 @@ +{ + "command": "just test-e2e", + "ran_at": 1790974770, + "exit_code": 0, + "commit": "1d300a2a5ab623b5e3dd5e44f585cf3f4089b046", + "branch": "claude/1241-content-prefetch" +} From 609faa694a85bfb77abd64b51152a16a5618ebeb Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:59:43 -0400 Subject: [PATCH 05/11] e2e attestation for claude/1241-content-prefetch --- .../ts/e2e-attestations/claude-1241-content-prefetch.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/ts/e2e-attestations/claude-1241-content-prefetch.json diff --git a/packages/ts/e2e-attestations/claude-1241-content-prefetch.json b/packages/ts/e2e-attestations/claude-1241-content-prefetch.json new file mode 100644 index 000000000..dc7edbf37 --- /dev/null +++ b/packages/ts/e2e-attestations/claude-1241-content-prefetch.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1790974783, + "exit_code": 0, + "commit": "e0e89a65ede94e3b1e4212281f390275b57f4bbd", + "branch": "claude/1241-content-prefetch" +} From fe1da905066567ba5fd085842d75d4fbe7d95885 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:27:48 -0400 Subject: [PATCH 06/11] test(rust): re-pin the read-ahead mutation waivers after the parallel walk --- testing-conventions.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing-conventions.toml b/testing-conventions.toml index a7f66fcfe..b5e5e52e8 100644 --- a/testing-conventions.toml +++ b/testing-conventions.toml @@ -137,13 +137,13 @@ reason = "run_default and run_query build a real index and drive stdout, stderr [[rust.exempt]] path = "src/vtab.rs" rules = ["mutation"] -lines = [82, 333, 393] +lines = [81, 328, 382] reason = "read_text and ScanSpec::read open a real file and prefetch reads a real tree ahead of the cursor; the buffer logic is unit-tested in `read_all` and `presize`, tests/vtab.rs content_is_excluded_from_star_but_selectable_by_name, content_is_null_for_non_utf8_files and unreadable_file_yields_null_content_without_erroring_the_row assert the cell, and tests/path_table_content_prefetch.rs content_is_read_for_every_row_before_the_first_is_delivered and a_join_has_the_content_it_names_read_ahead assert the read-ahead" [[rust.exempt]] path = "src/vtab_scaffold.rs" rules = ["mutation"] -lines = [280] +lines = [337] reason = "`|` -> `^` is equivalent: the lookup idx_num is a column index plus one, and SQLite caps a table at 32767 columns, so it never carries the PREFETCH_IDX bit and the two operators agree on every input" # Unit-only coverage floor: measured by `cargo llvm-cov --lib --features cli` From 0ffd63fb9f800a7068670739394ead95c83706c0 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:29:55 -0400 Subject: [PATCH 07/11] e2e attestation for claude/1241-content-prefetch --- .../python/e2e-attestations/claude-1241-content-prefetch.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/python/e2e-attestations/claude-1241-content-prefetch.json b/packages/python/e2e-attestations/claude-1241-content-prefetch.json index 962b902ad..c79f688f3 100644 --- a/packages/python/e2e-attestations/claude-1241-content-prefetch.json +++ b/packages/python/e2e-attestations/claude-1241-content-prefetch.json @@ -1,7 +1,7 @@ { "command": "just test-e2e", - "ran_at": 1790974770, + "ran_at": 1791160195, "exit_code": 0, - "commit": "1d300a2a5ab623b5e3dd5e44f585cf3f4089b046", + "commit": "d899839e25c8663fcfdc33c9f83c68dcd10913c2", "branch": "claude/1241-content-prefetch" } From df0d59aab9ddf48441af428fc6c0b5afe9e1cd18 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:30:40 -0400 Subject: [PATCH 08/11] e2e attestation for claude/1241-content-prefetch --- .../ts/e2e-attestations/claude-1241-content-prefetch.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ts/e2e-attestations/claude-1241-content-prefetch.json b/packages/ts/e2e-attestations/claude-1241-content-prefetch.json index dc7edbf37..a393377ee 100644 --- a/packages/ts/e2e-attestations/claude-1241-content-prefetch.json +++ b/packages/ts/e2e-attestations/claude-1241-content-prefetch.json @@ -1,7 +1,7 @@ { "command": "pnpm test:e2e", - "ran_at": 1790974783, + "ran_at": 1791160240, "exit_code": 0, - "commit": "e0e89a65ede94e3b1e4212281f390275b57f4bbd", + "commit": "20578e8281b4baa02c4bea8148f4bb9d2801f255", "branch": "claude/1241-content-prefetch" } From 4c645eb195b0c6a04d18c2f1064157adb275c8a5 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:44:00 -0400 Subject: [PATCH 09/11] test(rust): pin the plan_idx_num waiver to the line the mutation gate reports --- testing-conventions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing-conventions.toml b/testing-conventions.toml index b5e5e52e8..02e065532 100644 --- a/testing-conventions.toml +++ b/testing-conventions.toml @@ -143,7 +143,7 @@ reason = "read_text and ScanSpec::read open a real file and prefetch reads a rea [[rust.exempt]] path = "src/vtab_scaffold.rs" rules = ["mutation"] -lines = [337] +lines = [335] reason = "`|` -> `^` is equivalent: the lookup idx_num is a column index plus one, and SQLite caps a table at 32767 columns, so it never carries the PREFETCH_IDX bit and the two operators agree on every input" # Unit-only coverage floor: measured by `cargo llvm-cov --lib --features cli` From 6f1ae99e79ae4033eb5760d1120528fe82d5c25b Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Sun, 4 Oct 2026 21:04:01 -0400 Subject: [PATCH 10/11] e2e attestation for claude/1241-content-prefetch --- .../python/e2e-attestations/claude-1241-content-prefetch.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/python/e2e-attestations/claude-1241-content-prefetch.json b/packages/python/e2e-attestations/claude-1241-content-prefetch.json index c79f688f3..053870157 100644 --- a/packages/python/e2e-attestations/claude-1241-content-prefetch.json +++ b/packages/python/e2e-attestations/claude-1241-content-prefetch.json @@ -1,7 +1,7 @@ { "command": "just test-e2e", - "ran_at": 1791160195, + "ran_at": 1791162240, "exit_code": 0, - "commit": "d899839e25c8663fcfdc33c9f83c68dcd10913c2", + "commit": "4c645eb195b0c6a04d18c2f1064157adb275c8a5", "branch": "claude/1241-content-prefetch" } From f87ec678e29f6d6140022de5fff2053ad13d9994 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Sun, 4 Oct 2026 21:04:37 -0400 Subject: [PATCH 11/11] e2e attestation for claude/1241-content-prefetch --- .../ts/e2e-attestations/claude-1241-content-prefetch.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ts/e2e-attestations/claude-1241-content-prefetch.json b/packages/ts/e2e-attestations/claude-1241-content-prefetch.json index a393377ee..4f9e0f62f 100644 --- a/packages/ts/e2e-attestations/claude-1241-content-prefetch.json +++ b/packages/ts/e2e-attestations/claude-1241-content-prefetch.json @@ -1,7 +1,7 @@ { "command": "pnpm test:e2e", - "ran_at": 1791160240, + "ran_at": 1791162277, "exit_code": 0, - "commit": "20578e8281b4baa02c4bea8148f4bb9d2801f255", + "commit": "6f1ae99e79ae4033eb5760d1120528fe82d5c25b", "branch": "claude/1241-content-prefetch" }