From fe211d95faa7bcae8234f897ce0d9c657072ecca Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:09:33 -0400 Subject: [PATCH 1/4] perf: build scan paths on walk threads and insert hook rows beside grouping The config build's startup scan now carries each match's relative path from the walk worker that found it, instead of recomputing it serially, and streams early per-table hook rows into SQLite while the scanned files are grouped on a second thread. --- ...config-scan-bookkeeping-off-main-thread.md | 3 + ...config-scan-bookkeeping-off-main-thread.md | 25 ++ packages/rust/src/lib.rs | 229 +++++++++++------- packages/rust/src/scanner.rs | 100 ++++++-- 4 files changed, 258 insertions(+), 99 deletions(-) create mode 100644 packages/rust/changelog.d/2026-10-15-config-scan-bookkeeping-off-main-thread.md create mode 100644 packages/rust/migrations.d/2026-10-15-config-scan-bookkeeping-off-main-thread.md diff --git a/packages/rust/changelog.d/2026-10-15-config-scan-bookkeeping-off-main-thread.md b/packages/rust/changelog.d/2026-10-15-config-scan-bookkeeping-off-main-thread.md new file mode 100644 index 000000000..49fae5f02 --- /dev/null +++ b/packages/rust/changelog.d/2026-10-15-config-scan-bookkeeping-off-main-thread.md @@ -0,0 +1,3 @@ +**Changed** + +- **A config table's startup scan builds each file's relative path on the walk thread that found it, and inserts an `on-file` command's rows while the scanned files are still being grouped.** A config over many files, such as a notes vault with a frontmatter parser, starts faster. Query results are unchanged. diff --git a/packages/rust/migrations.d/2026-10-15-config-scan-bookkeeping-off-main-thread.md b/packages/rust/migrations.d/2026-10-15-config-scan-bookkeeping-off-main-thread.md new file mode 100644 index 000000000..840f91cd9 --- /dev/null +++ b/packages/rust/migrations.d/2026-10-15-config-scan-bookkeeping-off-main-thread.md @@ -0,0 +1,25 @@ +### Faster config-table startup + +#### Summary + +A config table's startup scan does less per-file work on one thread: the walk threads build each file's relative path, and an `on-file` command's rows are inserted while the scanned files are grouped. Query results are unchanged; nothing breaks. + +#### Required changes + +_None._ + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +_None._ + +#### Verification + +```bash +dirsql query "SELECT count(*) FROM notes" -c .dirsql.toml +``` + +Prints the same count as before, sooner. diff --git a/packages/rust/src/lib.rs b/packages/rust/src/lib.rs index 860e35ea0..4626f1c5d 100644 --- a/packages/rust/src/lib.rs +++ b/packages/rust/src/lib.rs @@ -65,10 +65,10 @@ use crate::persist::{ resolve_persist_path, upsert_file, write_meta, }; use crate::progress::Progress; -use crate::scanner::scan_directory_reporting; +use crate::scanner::{Found, scan_directory_reporting}; use crate::watcher::{FileEvent, WatchScope, Watcher}; use futures_channel::mpsc::{UnboundedReceiver, UnboundedSender, unbounded}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::error::Error as StdError; use std::path::{Path, PathBuf}; use std::sync::{ @@ -492,7 +492,7 @@ type ScanFn<'a> = &'a dyn Fn( &TableMatcher, &mut dyn FnMut(u64), &mut dyn FnMut(&Path, &str), -) -> Vec<(PathBuf, String)>; +) -> Vec; impl DirSQL { /// Start building a `DirSQL`. See [`DirSQLBuilder`] for the available @@ -1286,19 +1286,19 @@ impl DirSQL { // re-parsed. let (scanned_files, deleted) = match &persist_ctx { None => { - let mut files = Vec::with_capacity(scanned.len()); - for (path, table_name) in scanned { - files.push(ScannedFile { - rel_path: relative_path(anchors.of(&table_name), &path), - abs_path: path, - table_name, + let files = scanned + .into_iter() + .map(|found| ScannedFile { + abs_path: found.path, + rel_path: found.rel_path, + table_name: found.table_name, stat: None, trusted: false, - }); - } + }) + .collect(); (files, Vec::new()) } - Some(ctx) => reconcile_scan(&anchors, scanned, ctx, &RealFs)?, + Some(ctx) => reconcile_scan(scanned, ctx, &RealFs)?, }; Ok(PreparedBuild { @@ -1368,7 +1368,7 @@ impl DirSQL { anchors, ignore, scanned_files, - mut early, + early, persist, poll_interval, hint_legacy_files_table, @@ -1478,36 +1478,18 @@ impl DirSQL { let mut on_file_failures: Vec = Vec::new(); let mut batch_files: HashMap> = HashMap::new(); - // Per-file hooks run file by file; a per-table hook's files are - // gathered, in scan order, for one run after the per-file pass. - let mut per_file: Vec<(ScannedFile, Arc)> = Vec::new(); - let mut batches: Vec<(String, Arc, Vec)> = Vec::new(); - for file in scanned_files { - match on_file_map.get(&file.table_name) { - Some(Hook::PerFile(hook)) => { - if !file.trusted { - per_file.push((file, Arc::clone(hook))); - } - } - Some(Hook::PerTable(hook)) => { - match batches - .iter_mut() - .find(|(name, _, _)| *name == file.table_name) - { - Some((_, _, files)) => files.push(file), - None => { - batches.push((file.table_name.clone(), Arc::clone(hook), vec![file])) - } - } - } - None => { - return Err(DirSqlError::Ddl(format!( - "missing on-file function for table {}", - file.table_name - ))); - } - } - } + // Grouping walks every scanned file, so it runs beside the early + // hooks' inserts instead of ahead of them. + let (grouped, streamed) = std::thread::scope(|scope| { + let grouping = scope.spawn(|| group_scanned(scanned_files, &on_file_map)); + let streamed = stream_early(&db, early, &strict_map); + ( + grouping.join().expect("grouping scanned files panicked"), + streamed, + ) + }); + let streamed = streamed?; + let (per_file, batches) = grouped?; // The phase a user actually waits on: one `on_file` round trip per // file or per table, plus whatever the table's DDL triggers on insert. @@ -1595,11 +1577,9 @@ impl DirSQL { let current = persist_ready.as_ref().is_some_and(|(deleted, _, _)| { files.iter().all(|f| f.trusted) && !deleted.iter().any(|(_, t)| *t == table_name) }); - if !current { + if !current && !streamed.contains(&table_name) { let table_root = anchors.of(&table_name); - let production = early - .remove(&table_name) - .unwrap_or_else(|| start_hook(hook, abs_paths.into_iter())); + let production = start_hook(hook, abs_paths.into_iter()); let outcome = stream_table_rows(&db, &table_name, production, strict) .map_err(map_db_error)?; match outcome { @@ -2088,7 +2068,7 @@ fn scan_groups( scan: ScanFn<'_>, progress: &mut Progress, on_match: &mut dyn FnMut(&Path, &str), -) -> (Vec<(PathBuf, String)>, u64) { +) -> (Vec, u64) { let mut seen = 0; let mut scanned = Vec::new(); for group in groups { @@ -2183,12 +2163,12 @@ fn compile_groups( fn empty_table_warnings( tables: &[Table], anchors: &TableAnchors, - scanned: &[(PathBuf, String)], + scanned: &[Found], ) -> Vec { tables .iter() .filter(|table| table.warn_when_empty) - .filter(|table| !scanned.iter().any(|(_, name)| *name == table.name)) + .filter(|table| !scanned.iter().any(|found| found.table_name == table.name)) .map(|table| { format!( "dirsql: table '{}': glob '{}' matched no files under {}", @@ -2299,8 +2279,7 @@ impl FileSystem for RealFs { /// and which were removed since the last cache write. #[allow(clippy::type_complexity)] fn reconcile_scan( - anchors: &TableAnchors, - scanned: Vec<(PathBuf, String)>, + scanned: Vec, ctx: &PersistContext, fs: &dyn FileSystem, ) -> Result<(Vec, Vec<(String, String)>)> { @@ -2310,8 +2289,12 @@ fn reconcile_scan( let mut seen: std::collections::HashSet<(String, String)> = std::collections::HashSet::with_capacity(scanned.len()); - for (path, table_name) in scanned { - let rel_path = relative_path(anchors.of(&table_name), &path); + for Found { + path, + rel_path, + table_name, + } in scanned + { seen.insert((rel_path.clone(), table_name.clone())); let stat = fs.stat(&path)?; @@ -2558,6 +2541,76 @@ fn run_channel_loop(db: DirSQL, tx: UnboundedSender) { /// The absolute path is the one the scan found. Relative paths are cloned /// when `keep_rel` (the files are still read afterwards) and moved out /// otherwise. +/// Each per-file hook's file paired with its hook, then each per-table +/// hook's table with its files, in scan order. +#[allow(clippy::type_complexity)] +fn group_scanned( + scanned_files: Vec, + on_file_map: &HashMap, +) -> Result<( + Vec<(ScannedFile, Arc)>, + Vec<(String, Arc, Vec)>, +)> { + let mut per_file: Vec<(ScannedFile, Arc)> = Vec::new(); + let mut batches: Vec<(String, Arc, Vec)> = Vec::new(); + for file in scanned_files { + // Files arrive in walk order, so a table's files mostly follow + // one another: the last batch is checked before the map. + if let Some((name, _, files)) = batches.last_mut() + && *name == file.table_name + { + files.push(file); + continue; + } + match on_file_map.get(&file.table_name) { + Some(Hook::PerFile(hook)) => { + if !file.trusted { + per_file.push((file, Arc::clone(hook))); + } + } + Some(Hook::PerTable(hook)) => { + match batches + .iter_mut() + .find(|(name, _, _)| *name == file.table_name) + { + Some((_, _, files)) => files.push(file), + None => batches.push((file.table_name.clone(), Arc::clone(hook), vec![file])), + } + } + None => { + return Err(DirSqlError::Ddl(format!( + "missing on-file function for table {}", + file.table_name + ))); + } + } + } + Ok((per_file, batches)) +} + +/// Insert the rows of every hook started before the scan, returning the +/// tables streamed. +fn stream_early( + db: &Db, + early: HashMap, + strict_map: &HashMap, +) -> Result> { + let mut streamed = HashSet::new(); + for (table_name, production) in early { + let strict = *strict_map.get(&table_name).unwrap_or(&false); + if let Err(message) = + stream_table_rows(db, &table_name, production, strict).map_err(map_db_error)? + { + return Err(DirSqlError::TableCommand { + name: table_name, + message, + }); + } + streamed.insert(table_name); + } + Ok(streamed) +} + fn batch_paths(files: &mut [ScannedFile], keep_rel: bool) -> (Vec, Vec) { let mut rel = Vec::with_capacity(files.len()); let mut abs = Vec::with_capacity(files.len()); @@ -2572,10 +2625,6 @@ fn batch_paths(files: &mut [ScannedFile], keep_rel: bool) -> (Vec, Vec

String { - scanner::to_slash(path.strip_prefix(root).unwrap_or(path)) -} - /// Build [`Table`] objects from a parsed config. /// /// A config-defined table runs its `on-file` command once over every matched @@ -3027,11 +3076,11 @@ mod internal_tests { /// tree on disk. struct RecordingScan { calls: Cell, - found: Vec<(PathBuf, String)>, + found: Vec, } impl RecordingScan { - fn new(found: Vec<(PathBuf, String)>) -> Self { + fn new(found: Vec) -> Self { Self { calls: Cell::new(0), found, @@ -3046,7 +3095,7 @@ mod internal_tests { _matcher: &TableMatcher, on_file: &mut dyn FnMut(u64), _on_match: &mut dyn FnMut(&Path, &str), - ) -> Vec<(PathBuf, String)> { + ) -> Vec { self.calls.set(self.calls.get() + 1); for (index, _) in self.found.iter().enumerate() { on_file(index as u64 + 1); @@ -3085,7 +3134,11 @@ mod internal_tests { #[test] fn prepare_does_not_walk_the_tree_when_no_tables_are_declared() { let dir = TempDir::new().unwrap(); - let scan = RecordingScan::new(vec![(dir.path().join("a.txt"), "items".into())]); + let scan = RecordingScan::new(vec![Found { + path: dir.path().join("a.txt"), + rel_path: "a.txt".into(), + table_name: "items".into(), + }]); let prepared = DirSQL::prepare_resolved_with( resolved_over(dir.path(), Vec::new(), false), @@ -3106,7 +3159,11 @@ mod internal_tests { #[test] fn prepare_walks_the_tree_when_a_table_is_declared() { let dir = TempDir::new().unwrap(); - let scan = RecordingScan::new(vec![(dir.path().join("a.txt"), "items".into())]); + let scan = RecordingScan::new(vec![Found { + path: dir.path().join("a.txt"), + rel_path: "a.txt".into(), + table_name: "items".into(), + }]); let prepared = DirSQL::prepare_resolved_with( resolved_over(dir.path(), vec![txt_table()], false), @@ -3150,7 +3207,11 @@ mod internal_tests { .recv_timeout(std::time::Duration::from_secs(5)) .is_ok(), ); - vec![(found.clone(), "items".to_string())] + vec![Found { + path: found.clone(), + rel_path: "a.txt".into(), + table_name: "items".into(), + }] }) .unwrap(); assert!(hook_saw_it_mid_walk.get()); @@ -3559,6 +3620,14 @@ mod internal_tests { assert_eq!(pending.0, vec![("held".to_string(), "moved".to_string())]); } + fn found_at(root: &Path, path: &Path, table: &str) -> Found { + Found { + path: path.to_path_buf(), + rel_path: scanner::to_slash(path.strip_prefix(root).unwrap()), + table_name: table.to_string(), + } + } + #[test] fn scan_groups_counts_across_groups_and_collects_every_match() { let tables = vec![ @@ -3571,7 +3640,7 @@ mod internal_tests { report: &mut dyn FnMut(u64), _: &mut dyn FnMut(&Path, &str)| { report(3); - vec![(anchor.join("f.txt"), "t".to_string())] + vec![found_at(anchor, &anchor.join("f.txt"), "t")] }; let (scanned, seen) = @@ -3786,9 +3855,8 @@ mod internal_tests { meta_current: false, needs_sweep: false, }; - let scanned = vec![(abs.clone(), "t".to_string())]; - let (files, deleted) = - reconcile_scan(&TableAnchors::at_root(dir.path()), scanned, &ctx, &fake).unwrap(); + let scanned = vec![found_at(dir.path(), &abs, "t")]; + let (files, deleted) = reconcile_scan(scanned, &ctx, &fake).unwrap(); assert_eq!(files.len(), 1); assert!(files[0].trusted); assert_eq!(files[0].rel_path, "a.txt"); @@ -3823,9 +3891,8 @@ mod internal_tests { meta_current: false, needs_sweep: false, }; - let scanned = vec![(abs.clone(), "t".to_string())]; - let (files, _deleted) = - reconcile_scan(&TableAnchors::at_root(dir.path()), scanned, &ctx, &fake).unwrap(); + let scanned = vec![found_at(dir.path(), &abs, "t")]; + let (files, _deleted) = reconcile_scan(scanned, &ctx, &fake).unwrap(); assert_eq!(files.len(), 1); assert!(!files[0].trusted); } @@ -3841,9 +3908,9 @@ mod internal_tests { needs_sweep: false, }; let missing = dir.path().join("ghost.txt"); - let scanned = vec![(missing, "t".to_string())]; + let scanned = vec![found_at(dir.path(), &missing, "t")]; let fake = FakeFs::default(); - assert!(reconcile_scan(&TableAnchors::at_root(dir.path()), scanned, &ctx, &fake).is_err()); + assert!(reconcile_scan(scanned, &ctx, &fake).is_err()); } #[test] @@ -4709,9 +4776,8 @@ mod internal_tests { meta_current: false, needs_sweep: false, }; - let scanned = vec![(abs, "t".to_string())]; - let (files, deleted) = - reconcile_scan(&TableAnchors::at_root(dir.path()), scanned, &ctx, &fake).unwrap(); + let scanned = vec![found_at(dir.path(), &abs, "t")]; + let (files, deleted) = reconcile_scan(scanned, &ctx, &fake).unwrap(); assert_eq!(files.len(), 1); assert!(files[0].trusted); assert!(deleted.is_empty()); @@ -4743,16 +4809,14 @@ mod internal_tests { meta_current: false, needs_sweep: false, }; - let scanned = vec![(abs, "t".to_string())]; - let (files, _deleted) = - reconcile_scan(&TableAnchors::at_root(dir.path()), scanned, &ctx, &fake).unwrap(); + let scanned = vec![found_at(dir.path(), &abs, "t")]; + let (files, _deleted) = reconcile_scan(scanned, &ctx, &fake).unwrap(); assert_eq!(files.len(), 1); assert!(!files[0].trusted); } #[test] fn reconcile_scan_reports_deleted_cached_files() { - let dir = TempDir::new().unwrap(); let mut cached = HashMap::new(); cached.insert( ("gone.txt".to_string(), "t".to_string()), @@ -4772,8 +4836,7 @@ mod internal_tests { needs_sweep: false, }; let fake = FakeFs::default(); - let (files, deleted) = - reconcile_scan(&TableAnchors::at_root(dir.path()), Vec::new(), &ctx, &fake).unwrap(); + let (files, deleted) = reconcile_scan(Vec::new(), &ctx, &fake).unwrap(); assert!(files.is_empty()); assert_eq!(deleted, vec![("gone.txt".to_string(), "t".to_string())]); } @@ -6135,7 +6198,7 @@ mod internal_tests { fn a_table_with_a_matched_file_does_not_warn() { let tables = vec![anchored_table("a", "*.txt", "/one").warning_when_empty()]; let anchors = compile_groups(&tables, &[], Path::new("/idx")).unwrap().1; - let scanned = vec![(PathBuf::from("/one/x.txt"), "a".to_string())]; + let scanned = vec![found_at(Path::new("/one"), Path::new("/one/x.txt"), "a")]; assert!(empty_table_warnings(&tables, &anchors, &scanned).is_empty()); } diff --git a/packages/rust/src/scanner.rs b/packages/rust/src/scanner.rs index 46b0c4bca..31495015a 100644 --- a/packages/rust/src/scanner.rs +++ b/packages/rust/src/scanner.rs @@ -34,7 +34,28 @@ fn with_slashes(path: &str, native: char) -> String { /// Walk a directory tree and return all file paths paired with their matching table name. /// Ignored paths and directories are skipped. Only files (not directories) are returned. pub fn scan_directory(root: &Path, matcher: &TableMatcher) -> Vec<(PathBuf, String)> { - scan_directory_reporting(root, matcher, &mut |_| {}, &mut |_, _| {}) + paired(scan_directory_reporting( + root, + matcher, + &mut |_| {}, + &mut |_, _| {}, + )) +} + +/// A file a scan matched to a table: its path, the same path relative to the +/// scan root as `/`-separated text, and the table. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Found { + pub path: PathBuf, + pub rel_path: String, + pub table_name: String, +} + +fn paired(found: Vec) -> Vec<(PathBuf, String)> { + found + .into_iter() + .map(|found| (found.path, found.table_name)) + .collect() } /// [`scan_directory`], calling `on_file` with the running count of files the @@ -54,7 +75,7 @@ pub fn scan_directory_reporting( matcher: &TableMatcher, on_file: &mut dyn FnMut(u64), on_match: &mut dyn FnMut(&Path, &str), -) -> Vec<(PathBuf, String)> { +) -> Vec { scan_below(root, root, matcher, on_file, on_match) } @@ -66,7 +87,26 @@ pub fn scan_directory_reporting( /// `mkdir` or a rename into the tree — since the OS reports one event for the /// directory and none for the files already inside it. pub fn scan_subtree(root: &Path, dir: &Path, matcher: &TableMatcher) -> Vec<(PathBuf, String)> { - scan_below(root, dir, matcher, &mut |_| {}, &mut |_, _| {}) + paired(scan_below(root, dir, matcher, &mut |_| {}, &mut |_, _| {})) +} + +/// One [`Found`] per table whose glob matches `rel_path`, in declaration +/// order, for the file of that name in `dir`. +fn found_in(matcher: &TableMatcher, rel_path: &Path, dir: &Path) -> Vec { + let matches = matcher.match_all(rel_path); + if matches.is_empty() { + return Vec::new(); + } + let path = dir.join(rel_path.file_name().unwrap_or_default()); + let rel = to_slash(rel_path); + matches + .into_iter() + .map(|m| Found { + path: path.clone(), + rel_path: rel.clone(), + table_name: m.table_name, + }) + .collect() } /// `start` and every directory below it that [`scan_subtree`] would enter, @@ -99,7 +139,7 @@ fn scan_below( matcher: &TableMatcher, on_file: &mut dyn FnMut(u64), on_match: &mut dyn FnMut(&Path, &str), -) -> Vec<(PathBuf, String)> { +) -> Vec { let mut results = Vec::new(); let mut seen: u64 = 0; let gitignore = matcher.respects_gitignore(); @@ -126,20 +166,17 @@ fn scan_below( start, walker, &|rel_path| !matcher.is_ignored(rel_path), - &|rel_path, dir| { - let path = dir.join(rel_path.file_name().unwrap_or_default()); - (rel_path, path) - }, - &mut |(rel_path, path)| { + // Matching and the relative path are made on the worker that found + // the file: the calling thread hands out every file in turn, so any + // per-file work left to it is serial. + &|rel_path, dir| found_in(matcher, &rel_path, dir), + &mut |found| { seen += 1; on_file(seen); - - // Fan-out: a file matching N tables' globs yields N (path, table) - // pairs, one per matching table, in declaration order. - for m in matcher.match_all(&rel_path) { - on_match(&path, &m.table_name); - results.push((path.clone(), m.table_name)); + for found in &found { + on_match(&found.path, &found.table_name); } + results.extend(found); }, ); @@ -956,11 +993,42 @@ mod tests { &mut |_, _| {}, ); - assert_eq!(rows, vec![(root.join("scanner.rs"), "source".to_owned())]); + assert_eq!( + rows, + vec![Found { + path: root.join("scanner.rs"), + rel_path: "scanner.rs".to_owned(), + table_name: "source".to_owned(), + }] + ); assert_eq!(counts.first(), Some(&1)); assert!(counts.windows(2).all(|pair| pair[1] == pair[0] + 1)); } + #[test] + fn found_in_fans_a_file_out_to_every_matching_table_in_order() { + let matcher = + TableMatcher::new(&[("docs/**/*.md", "notes"), ("**/*.md", "all")], &[]).unwrap(); + let rel = Path::new("docs").join("a").join("b.md"); + let found = found_in(&matcher, &rel, Path::new("/r/docs/a")); + let path = Path::new("/r/docs/a").join("b.md"); + let expected: Vec = ["notes", "all"] + .into_iter() + .map(|table| Found { + path: path.clone(), + rel_path: "docs/a/b.md".to_owned(), + table_name: table.to_owned(), + }) + .collect(); + assert_eq!(found, expected); + } + + #[test] + fn found_in_is_empty_for_a_file_no_table_matches() { + let matcher = TableMatcher::new(&[("*.md", "notes")], &[]).unwrap(); + assert!(found_in(&matcher, Path::new("a.txt"), Path::new("/r")).is_empty()); + } + // Real directory-walk behavior is covered by `tests/scanner.rs` // (unit-lint isolation); only the pure predicate is tested here. From 70a2e2b0af98f794420a1c38accee599bf658532 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:15:09 -0400 Subject: [PATCH 2/4] e2e attestation for bot/1483-fm-speed-1015 --- .../python/e2e-attestations/bot-1483-fm-speed-1015.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/python/e2e-attestations/bot-1483-fm-speed-1015.json diff --git a/packages/python/e2e-attestations/bot-1483-fm-speed-1015.json b/packages/python/e2e-attestations/bot-1483-fm-speed-1015.json new file mode 100644 index 000000000..1f86ec532 --- /dev/null +++ b/packages/python/e2e-attestations/bot-1483-fm-speed-1015.json @@ -0,0 +1,7 @@ +{ + "command": "uv run python -m pytest tests/e2e/frontmatter_parse_speed_test.py tests/e2e/jsonl_parse_speed_test.py tests/e2e/on_file_query_test.py tests/e2e/fanout_test.py tests/e2e/declared_table_name_test.py tests/e2e/query_contract_test.py tests/e2e/use_case_notebook_test.py tests/e2e/use_case_transcripts_test.py tests/e2e/sigint_interrupt_test.py -x -q -s", + "ran_at": 1792073709, + "exit_code": 0, + "commit": "fe211d95faa7bcae8234f897ce0d9c657072ecca", + "branch": "bot/1483-fm-speed-1015" +} From 1c57ef268d4a4f427e1a2581b914890ceead5ddf Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:16:36 -0400 Subject: [PATCH 3/4] e2e attestation for bot/1483-fm-speed-1015 --- packages/ts/e2e-attestations/bot-1483-fm-speed-1015.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 packages/ts/e2e-attestations/bot-1483-fm-speed-1015.json diff --git a/packages/ts/e2e-attestations/bot-1483-fm-speed-1015.json b/packages/ts/e2e-attestations/bot-1483-fm-speed-1015.json new file mode 100644 index 000000000..9f70dfeca --- /dev/null +++ b/packages/ts/e2e-attestations/bot-1483-fm-speed-1015.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1792073795, + "exit_code": 0, + "commit": "70a2e2b0af98f794420a1c38accee599bf658532", + "branch": "bot/1483-fm-speed-1015" +} From 3b1ef507c7b478d0998c84299345bdb5f5ca2224 Mon Sep 17 00:00:00 2001 From: thekevinbot <248217931+thekevinbot@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:38:58 -0400 Subject: [PATCH 4/4] test: startup runs a per-table hook once over every file --- packages/rust/src/lib.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/rust/src/lib.rs b/packages/rust/src/lib.rs index 4626f1c5d..705bce0bb 100644 --- a/packages/rust/src/lib.rs +++ b/packages/rust/src/lib.rs @@ -5846,6 +5846,28 @@ mod internal_tests { assert_eq!(row_names(&db), vec!["a.txt", "b.txt"]); } + #[test] + fn startup_runs_a_per_table_hook_once_over_every_file() { + let dir = TempDir::new().unwrap(); + let root = dir.path().to_path_buf(); + std::fs::write(root.join("a.txt"), "").unwrap(); + std::fs::write(root.join("b.txt"), "").unwrap(); + let (items, calls) = recording_per_table("items", "*.txt"); + + let db = + DirSQL::with_ignore_and_fs(&root, vec![items], Vec::::new(), Arc::new(RealFs)) + .unwrap(); + + let calls = calls.lock().unwrap(); + assert_eq!(calls.len(), 1, "{calls:?}"); + let mut seen = calls[0].clone(); + seen.sort(); + assert_eq!(seen, vec![root.join("a.txt"), root.join("b.txt")]); + let mut names = row_names(&db); + names.sort(); + assert_eq!(names, vec!["a.txt", "b.txt"]); + } + #[test] fn events_arriving_while_a_batch_is_routed_share_its_per_table_rerun() { let dir = TempDir::new().unwrap();