diff --git a/packages/python/e2e-attestations/claude-1232-anchor-plumbing.json b/packages/python/e2e-attestations/claude-1232-anchor-plumbing.json new file mode 100644 index 000000000..e911300e7 --- /dev/null +++ b/packages/python/e2e-attestations/claude-1232-anchor-plumbing.json @@ -0,0 +1,7 @@ +{ + "command": "uv run python -m pytest tests/e2e/capture_collision_test.py tests/e2e/declared_table_name_test.py tests/e2e/extension_package_test.py tests/e2e/query_contract_test.py tests/e2e/plugin_discovery_test.py tests/e2e/sigint_interrupt_test.py tests/e2e/fanout_test.py tests/e2e/use_case_transcripts_test.py tests/e2e/plugin_extension_discovery_test.py tests/e2e/on_file_query_test.py -x -q", + "ran_at": 1791302031, + "exit_code": 0, + "commit": "1746a7286f9b11c9ecb0078ec263e69ecce29d7f", + "branch": "claude/1232-anchor-plumbing" +} diff --git a/packages/rust/src/lib.rs b/packages/rust/src/lib.rs index f82c1999c..1f4d2b42e 100644 --- a/packages/rust/src/lib.rs +++ b/packages/rust/src/lib.rs @@ -284,9 +284,22 @@ pub struct Table { pub glob: String, pub strict: bool, hook: Hook, + anchor: Option, } impl Table { + pub(crate) fn anchor_key(&self) -> Option { + self.anchor + .as_ref() + .map(|anchor| anchor.to_string_lossy().into_owned()) + } + + #[doc(hidden)] + pub fn anchored(mut self, anchor: impl Into) -> Self { + self.anchor = Some(anchor.into()); + self + } + pub fn new( name: impl Into, ddl: impl Into, @@ -330,6 +343,7 @@ impl Table { glob: glob.into(), hook: Hook::PerFile(Arc::new(on_file)), strict: false, + anchor: None, } } @@ -374,6 +388,7 @@ impl Table { glob: glob.into(), hook: Hook::PerTable(Arc::new(on_files)), strict: false, + anchor: None, } } } @@ -388,7 +403,9 @@ struct DirSqlInner { /// not-yet-created root); the user's `root` — and therefore the initial /// scan and the `path` column — stays byte-for-byte unchanged. watch_root: PathBuf, - matcher: TableMatcher, + /// One entry per distinct table anchor; each holds the matcher for the + /// tables anchored there. + groups: Vec, on_file_map: HashMap, strict_map: HashMap, /// The root-relative paths each per-table hook was last run over. Rows @@ -409,6 +426,37 @@ struct DirSqlInner { fs: Arc, } +/// The tables sharing one anchor, matched against paths relative to it. +struct AnchorGroup { + anchor: PathBuf, + /// `anchor` canonicalized for the live watcher; see + /// [`DirSqlInner::watch_root`]. + watch_anchor: PathBuf, + matcher: TableMatcher, + tables: Vec, +} + +/// Where each table's paths are anchored. A table without an entry anchors at +/// the index root. +#[derive(Clone)] +struct TableAnchors { + index_root: PathBuf, + by_table: HashMap, +} + +impl TableAnchors { + fn at_root(index_root: &Path) -> Self { + Self { + index_root: index_root.to_path_buf(), + by_table: HashMap::new(), + } + } + + fn of(&self, table: &str) -> &Path { + self.by_table.get(table).unwrap_or(&self.index_root) + } +} + #[derive(Clone)] pub struct DirSQL { inner: Arc, @@ -636,63 +684,85 @@ impl DirSQL { let abs_path = match &event { FileEvent::Created(p) | FileEvent::Modified(p) | FileEvent::Deleted(p) => p.clone(), }; - // Events arrive under the canonical `watch_root`, so strip that - // first; fall back to the user-supplied `root` (the already-canonical - // /absolute-root case), then to the raw absolute path. - let rel_path_buf = abs_path - .strip_prefix(&self.inner.watch_root) - .or_else(|_| abs_path.strip_prefix(&self.inner.root)) - .unwrap_or(&abs_path) - .to_path_buf(); - - if self.inner.matcher.is_ignored(&rel_path_buf) { - return Vec::new(); - } - - // A directory that appears whole (`mkdir`, or a populated directory - // renamed into the tree) arrives as one event naming the directory and - // none for the files already inside it, so the subtree is walked here. - // Only on creation: a metadata change on a directory says nothing - // about its files, and walking on every one would re-run `on_file` - // across the subtree for a `chmod`. - if matches!(event, FileEvent::Created(_)) - && self.inner.fs.is_dir(&abs_path).unwrap_or(false) - { - return self.index_subtree(&abs_path, pending); - } - - // Fan-out: dispatch the event to every table whose glob matches, and - // concatenate the resulting row events. Cross-table event order is - // unspecified. An `on_file` failure produces an error event for that - // table only; the other matching tables still process the event. - let matches = self.inner.matcher.match_all(&rel_path_buf); - let rel_path = scanner::to_slash(&rel_path_buf); - let mut events = Vec::new(); - for m in matches { - if self.is_per_table(&m.table_name) { - pending.mark(&m.table_name, &rel_path); + for (group, rel_path_buf) in self.groups_containing(&abs_path) { + if group.matcher.is_ignored(&rel_path_buf) { + continue; + } + + // A directory that appears whole (`mkdir`, or a populated directory + // renamed into the tree) arrives as one event naming the directory and + // none for the files already inside it, so the subtree is walked here. + // Only on creation: a metadata change on a directory says nothing + // about its files, and walking on every one would re-run `on_file` + // across the subtree for a `chmod`. + if matches!(event, FileEvent::Created(_)) + && self.inner.fs.is_dir(&abs_path).unwrap_or(false) + { + events.extend(self.index_subtree(group, &abs_path, pending)); continue; } - match &event { - FileEvent::Deleted(_) => { - events.extend(self.handle_delete(&m.table_name, &rel_path)); + + // Fan-out: dispatch the event to every table whose glob matches, and + // concatenate the resulting row events. Cross-table event order is + // unspecified. An `on_file` failure produces an error event for that + // table only; the other matching tables still process the event. + let matches = group.matcher.match_all(&rel_path_buf); + let rel_path = scanner::to_slash(&rel_path_buf); + + for m in matches { + if self.is_per_table(&m.table_name) { + pending.mark(&m.table_name, &rel_path); + continue; } - FileEvent::Created(_) | FileEvent::Modified(_) => { - events.extend(self.handle_upsert(&m.table_name, &abs_path, &rel_path)); + match &event { + FileEvent::Deleted(_) => { + events.extend(self.handle_delete(&m.table_name, &rel_path)); + } + FileEvent::Created(_) | FileEvent::Modified(_) => { + events.extend(self.handle_upsert(&m.table_name, &abs_path, &rel_path)); + } } } - } - // The mirror of the walk above: a directory that leaves the tree - // (`rm -r`, or a rename out of or within the root) arrives as one - // event naming the directory and none for the files beneath it. It is - // already gone, so the rows are the only record of what it held. - if matches!(event, FileEvent::Deleted(_)) { - events.extend(self.delete_subtree(&rel_path, pending)); + // The mirror of the walk above: a directory that leaves the tree + // (`rm -r`, or a rename out of or within the root) arrives as one + // event naming the directory and none for the files beneath it. It is + // already gone, so the rows are the only record of what it held. + if matches!(event, FileEvent::Deleted(_)) { + events.extend(self.delete_subtree(group, &rel_path, pending)); + } } events } + /// The anchor groups an event path falls under, each with the path + /// relative to its anchor. Events arrive under the canonical + /// `watch_anchor`, so that is stripped first; the user-supplied anchor + /// (the already-canonical /absolute case) is the fallback. A path under + /// no anchor is matched raw against every group. + fn groups_containing(&self, abs_path: &Path) -> Vec<(&AnchorGroup, PathBuf)> { + let under: Vec<(&AnchorGroup, PathBuf)> = self + .inner + .groups + .iter() + .filter_map(|group| { + abs_path + .strip_prefix(&group.watch_anchor) + .or_else(|_| abs_path.strip_prefix(&group.anchor)) + .ok() + .map(|rel| (group, rel.to_path_buf())) + }) + .collect(); + if !under.is_empty() { + return under; + } + self.inner + .groups + .iter() + .map(|group| (group, abs_path.to_path_buf())) + .collect() + } + fn is_per_table(&self, table: &str) -> bool { matches!(self.inner.on_file_map.get(table), Some(Hook::PerTable(_))) } @@ -711,10 +781,18 @@ impl DirSQL { let Some(Hook::PerTable(hook)) = self.inner.on_file_map.get(table) else { return Vec::new(); }; - let root = &self.inner.root; + let Some(group) = self + .inner + .groups + .iter() + .find(|group| group.tables.iter().any(|name| name == table)) + else { + return Vec::new(); + }; + let root = &group.anchor; let mut rel_paths = Vec::new(); let mut abs_paths = Vec::new(); - for (path, matched) in self.inner.fs.scan_subtree(root, root, &self.inner.matcher) { + for (path, matched) in self.inner.fs.scan_subtree(root, root, &group.matcher) { if matched == table { rel_paths.push(relative_path(root, &path)); abs_paths.push(path); @@ -760,11 +838,17 @@ impl DirSQL { /// Delete the rows of every file recorded beneath `rel_dir`, in every /// table, as if each file had its own delete event. - fn delete_subtree(&self, rel_dir: &str, pending: &mut PendingRefresh) -> Vec { + fn delete_subtree( + &self, + group: &AnchorGroup, + rel_dir: &str, + pending: &mut PendingRefresh, + ) -> Vec { let prefix = format!("{rel_dir}/"); + let in_group = |table: &str| group.tables.iter().any(|name| name == table); if let Ok(batch_files) = self.inner.batch_files.lock() { for (table, files) in batch_files.iter() { - if files.iter().any(|file| file.starts_with(&prefix)) { + if in_group(table) && files.iter().any(|file| file.starts_with(&prefix)) { pending.mark(table, rel_dir); } } @@ -781,6 +865,9 @@ impl DirSQL { }; let mut events = Vec::new(); for (table, file_path) in files { + if !in_group(&table) { + continue; + } events.extend(self.handle_delete(&table, &file_path)); } events @@ -789,14 +876,19 @@ impl DirSQL { /// Index every file beneath `dir` as if each had its own create event, /// using the scanner's walk so the watch path and the initial scan share /// one definition of what counts as a row. - fn index_subtree(&self, dir: &Path, pending: &mut PendingRefresh) -> Vec { - let base = if dir.starts_with(&self.inner.watch_root) { - &self.inner.watch_root + fn index_subtree( + &self, + group: &AnchorGroup, + dir: &Path, + pending: &mut PendingRefresh, + ) -> Vec { + let base = if dir.starts_with(&group.watch_anchor) { + &group.watch_anchor } else { - &self.inner.root + &group.anchor }; let mut events = Vec::new(); - for (path, table) in self.inner.fs.scan_subtree(base, dir, &self.inner.matcher) { + for (path, table) in self.inner.fs.scan_subtree(base, dir, &group.matcher) { let rel_path = scanner::to_slash(path.strip_prefix(base).unwrap_or(&path)); if self.is_per_table(&table) { pending.mark(&table, &rel_path); @@ -974,7 +1066,7 @@ impl DirSQL { no_ignore, } = resolved; - let (matcher, table_names) = compile_matcher(&tables, &ignore)?; + let (groups, anchors, table_names) = compile_groups(&tables, &ignore, &root)?; // Resolve the persistent context before scanning, so the scan can // consult the cached file index. @@ -999,11 +1091,7 @@ impl DirSQL { // The walk reports a running count rather than a fraction: it does // not know how many files there are until it has found them all. let mut progress = Progress::scanning(); - let mut seen = 0; - let scanned = scan(&root, &matcher, &mut |count| { - seen = count; - progress.update(count, None); - }); + let (scanned, seen) = scan_groups(&groups, scan, &mut progress); progress.finish(seen); scanned }; @@ -1016,7 +1104,7 @@ impl DirSQL { let mut files = Vec::with_capacity(scanned.len()); for (path, table_name) in scanned { files.push(ScannedFile { - rel_path: relative_path(&root, &path), + rel_path: relative_path(anchors.of(&table_name), &path), table_name, stat: None, trusted: false, @@ -1024,7 +1112,7 @@ impl DirSQL { } (files, Vec::new()) } - Some(ctx) => reconcile_scan(&root, scanned, ctx, &RealFs)?, + Some(ctx) => reconcile_scan(&anchors, scanned, ctx, &RealFs)?, }; Ok(PreparedBuild { @@ -1032,7 +1120,8 @@ impl DirSQL { tables, extensions, functions, - matcher, + groups, + anchors, ignore, scanned_files, hint_legacy_files_table, @@ -1088,7 +1177,8 @@ impl DirSQL { tables, extensions, functions, - matcher, + groups, + anchors, ignore, scanned_files, persist, @@ -1265,7 +1355,7 @@ impl DirSQL { progress.update(done, Some(total_files)); done += 1; let strict = *strict_map.get(&table_name).unwrap_or(&false); - let abs_path = root.join(&rel_path); + let abs_path = anchors.of(&table_name).join(&rel_path); // A hook failure is this file's problem, not the scan's: record it // and keep going, so one unreadable file cannot hide the state of // every file after it. @@ -1305,7 +1395,7 @@ impl DirSQL { if persist_ready.is_some() && let Some(stat) = stat.as_ref() { - let hash = hash_file(&root.join(&rel_path)).ok(); + let hash = hash_file(&anchors.of(&table_name).join(&rel_path)).ok(); upsert_file( db.conn(), &rel_path, @@ -1330,7 +1420,9 @@ impl DirSQL { files.iter().all(|f| f.trusted) && !deleted.iter().any(|(_, t)| *t == table_name) }); if !current { - let abs_paths: Vec = rel_paths.iter().map(|r| root.join(r)).collect(); + let table_root = anchors.of(&table_name); + let abs_paths: Vec = + rel_paths.iter().map(|r| table_root.join(r)).collect(); let outcome = stream_table_rows(&db, &table_name, hook.as_ref(), &abs_paths, strict) .map_err(map_db_error)?; @@ -1341,7 +1433,7 @@ impl DirSQL { let Some(stat) = file.stat.as_ref() else { continue; }; - let hash = hash_file(&root.join(&file.rel_path)).ok(); + let hash = hash_file(&table_root.join(&file.rel_path)).ok(); upsert_file( db.conn(), &file.rel_path, @@ -1390,13 +1482,17 @@ impl DirSQL { // Canonicalize the watch root so the live watcher never sees a // relative path; `root` itself is left untouched. let watch_root = PathBuf::from(fs.canonical_root(&root)); + let mut groups = groups; + for group in &mut groups { + group.watch_anchor = PathBuf::from(fs.canonical_root(&group.anchor)); + } Ok(Self { inner: Arc::new(DirSqlInner { db: Mutex::new(db), root, watch_root, - matcher, + groups, on_file_map, strict_map, batch_files: Mutex::new(batch_files), @@ -1757,11 +1853,12 @@ pub struct ScannedFile { pub struct PreparedBuild { root: PathBuf, tables: Vec, + groups: Vec, + anchors: TableAnchors, /// SQLite extensions to load onto the connection before any table DDL. extensions: Vec, /// Worker-backed SQL scalar functions to register after extensions load. functions: Vec, - matcher: TableMatcher, /// The configured skip rules, carried through so path-table scans apply /// the same ones declared tables do. ignore: Vec, @@ -1805,13 +1902,34 @@ struct PersistContext { needs_sweep: bool, } -fn compile_matcher( +/// Walk every group's anchor in turn, reporting one running count across all of +/// them. Returns the matches and the total count. +fn scan_groups( + groups: &[AnchorGroup], + scan: ScanFn<'_>, + progress: &mut Progress, +) -> (Vec<(PathBuf, String)>, u64) { + let mut seen = 0; + let mut scanned = Vec::new(); + for group in groups { + let before = seen; + scanned.extend(scan(&group.anchor, &group.matcher, &mut |count| { + seen = before + count; + progress.update(seen, None); + })); + } + (scanned, seen) +} + +fn compile_groups( tables: &[Table], ignore_patterns: &[String], -) -> Result<(TableMatcher, Vec)> { + root: &Path, +) -> Result<(Vec, TableAnchors, Vec)> { let mut seen: HashMap = HashMap::with_capacity(tables.len()); - let mut mappings: Vec<(String, String)> = Vec::with_capacity(tables.len()); let mut names = Vec::with_capacity(tables.len()); + let mut anchors = TableAnchors::at_root(root); + let mut grouped: Vec<(PathBuf, Vec<(String, String)>)> = Vec::new(); for table in tables { let table_name = table.name.clone(); // Validate up front so a poisoned name from a stored cache or a @@ -1821,17 +1939,38 @@ fn compile_matcher( if seen.insert(table_name.clone(), ()).is_some() { return Err(DirSqlError::DuplicateTable(table_name)); } - mappings.push((table.glob.clone(), table_name.clone())); + let anchor = match &table.anchor { + Some(anchor) => { + anchors.by_table.insert(table_name.clone(), anchor.clone()); + anchor.clone() + } + None => root.to_path_buf(), + }; + let mapping = (table.glob.clone(), table_name.clone()); + match grouped.iter_mut().find(|(a, _)| *a == anchor) { + Some((_, mappings)) => mappings.push(mapping), + None => grouped.push((anchor, vec![mapping])), + } names.push(table_name); } - let mapping_refs: Vec<(&str, &str)> = mappings - .iter() - .map(|(g, n)| (g.as_str(), n.as_str())) - .collect(); let ignore_refs: Vec<&str> = ignore_patterns.iter().map(String::as_str).collect(); - let matcher = TableMatcher::new(&mapping_refs, &ignore_refs).map_err(DirSqlError::matcher)?; - Ok((matcher, names)) + let mut groups = Vec::with_capacity(grouped.len()); + for (anchor, mappings) in grouped { + let mapping_refs: Vec<(&str, &str)> = mappings + .iter() + .map(|(g, n)| (g.as_str(), n.as_str())) + .collect(); + let matcher = + TableMatcher::new(&mapping_refs, &ignore_refs).map_err(DirSqlError::matcher)?; + groups.push(AnchorGroup { + watch_anchor: anchor.clone(), + anchor, + matcher, + tables: mappings.into_iter().map(|(_, name)| name).collect(), + }); + } + Ok((groups, anchors, names)) } /// Open (or create) the persistent SQLite cache and read its meta. If the @@ -1933,7 +2072,7 @@ impl FileSystem for RealFs { /// and which were removed since the last cache write. #[allow(clippy::type_complexity)] fn reconcile_scan( - root: &Path, + anchors: &TableAnchors, scanned: Vec<(PathBuf, String)>, ctx: &PersistContext, fs: &dyn FileSystem, @@ -1945,7 +2084,7 @@ fn reconcile_scan( std::collections::HashSet::with_capacity(scanned.len()); for (path, table_name) in scanned { - let rel_path = relative_path(root, &path); + let rel_path = relative_path(anchors.of(&table_name), &path); seen.insert((rel_path.clone(), table_name.clone())); let stat = fs.stat(&path)?; @@ -2758,14 +2897,14 @@ mod internal_tests { #[test] fn finish_build_errors_on_ghost_scanned_file() { let dir = TempDir::new().unwrap(); - let matcher = TableMatcher::new(&[], &[]).unwrap(); let prepared = PreparedBuild { ignore: Vec::new(), root: dir.path().to_path_buf(), tables: Vec::new(), extensions: Vec::new(), functions: Vec::new(), - matcher, + groups: Vec::new(), + anchors: TableAnchors::at_root(dir.path()), scanned_files: vec![ScannedFile { rel_path: "ghost.txt".into(), table_name: "ghost".into(), @@ -3070,11 +3209,52 @@ mod internal_tests { ); } + #[test] + fn delete_subtree_marks_only_this_groups_tables_holding_files_beneath_it() { + let (_dir, db, _abs, _rel) = upsert_fixture(); + let tables = vec![ + anchored_table("held", "*.txt", "/one"), + anchored_table("bare", "*.txt", "/one"), + ]; + let (groups, _, _) = compile_groups(&tables, &[], Path::new("/idx")).unwrap(); + let group = &groups[0]; + { + let mut files = db.inner.batch_files.lock().unwrap(); + files.insert("held".into(), vec!["moved/x".into()]); + files.insert("bare".into(), vec!["elsewhere/y".into()]); + files.insert("foreign".into(), vec!["moved/z".into()]); + } + let mut pending = PendingRefresh::default(); + + db.delete_subtree(group, "moved", &mut pending); + + assert_eq!(pending.0, vec![("held".to_string(), "moved".to_string())]); + } + + #[test] + fn scan_groups_counts_across_groups_and_collects_every_match() { + let tables = vec![ + anchored_table("a", "*.txt", "/one"), + anchored_table("b", "*.txt", "/two"), + ]; + let (groups, _, _) = compile_groups(&tables, &[], Path::new("/idx")).unwrap(); + let scan = |anchor: &Path, _: &TableMatcher, report: &mut dyn FnMut(u64)| { + report(3); + vec![(anchor.join("f.txt"), "t".to_string())] + }; + + let (scanned, seen) = scan_groups(&groups, &scan, &mut Progress::scanning()); + + assert_eq!(seen, 6); + assert_eq!(scanned.len(), 2); + } + #[test] fn delete_subtree_surfaces_db_poison() { let (_dir, db, _abs, _rel) = upsert_fixture(); poison(&db.inner.db); - let events = db.delete_subtree("moved", &mut PendingRefresh::default()); + let events = + db.delete_subtree(&db.inner.groups[0], "moved", &mut PendingRefresh::default()); assert_single_lock_error(&events); } @@ -3088,7 +3268,8 @@ mod internal_tests { .conn() .execute("DROP TABLE _dirsql_internal_rows", []) .unwrap(); - let events = db.delete_subtree("moved", &mut PendingRefresh::default()); + let events = + db.delete_subtree(&db.inner.groups[0], "moved", &mut PendingRefresh::default()); assert_eq!(events.len(), 1, "expected one error event: {events:?}"); assert!( matches!(&events[0], RowEvent::Error { table: None, file_path, error } @@ -3194,7 +3375,7 @@ mod internal_tests { .unwrap(); // Repoint watch_root to a non-prefix sibling so the first strip misses. - Arc::get_mut(&mut db.inner).unwrap().watch_root = root.join("does-not-prefix"); + Arc::get_mut(&mut db.inner).unwrap().groups[0].watch_anchor = root.join("does-not-prefix"); let events = db.process_file_event(FileEvent::Created(abs)); assert_eq!(events.len(), 1, "expected one insert: {events:?}"); @@ -3274,7 +3455,8 @@ mod internal_tests { needs_sweep: false, }; let scanned = vec![(abs.clone(), "t".to_string())]; - let (files, deleted) = reconcile_scan(dir.path(), scanned, &ctx, &fake).unwrap(); + let (files, deleted) = + reconcile_scan(&TableAnchors::at_root(dir.path()), scanned, &ctx, &fake).unwrap(); assert_eq!(files.len(), 1); assert!(files[0].trusted); assert_eq!(files[0].rel_path, "a.txt"); @@ -3310,7 +3492,8 @@ mod internal_tests { needs_sweep: false, }; let scanned = vec![(abs.clone(), "t".to_string())]; - let (files, _deleted) = reconcile_scan(dir.path(), scanned, &ctx, &fake).unwrap(); + let (files, _deleted) = + reconcile_scan(&TableAnchors::at_root(dir.path()), scanned, &ctx, &fake).unwrap(); assert_eq!(files.len(), 1); assert!(!files[0].trusted); } @@ -3328,7 +3511,7 @@ mod internal_tests { let missing = dir.path().join("ghost.txt"); let scanned = vec![(missing, "t".to_string())]; let fake = FakeFs::default(); - assert!(reconcile_scan(dir.path(), scanned, &ctx, &fake).is_err()); + assert!(reconcile_scan(&TableAnchors::at_root(dir.path()), scanned, &ctx, &fake).is_err()); } #[test] @@ -4132,7 +4315,8 @@ mod internal_tests { needs_sweep: false, }; let scanned = vec![(abs, "t".to_string())]; - let (files, deleted) = reconcile_scan(dir.path(), scanned, &ctx, &fake).unwrap(); + let (files, deleted) = + reconcile_scan(&TableAnchors::at_root(dir.path()), scanned, &ctx, &fake).unwrap(); assert_eq!(files.len(), 1); assert!(files[0].trusted); assert!(deleted.is_empty()); @@ -4165,7 +4349,8 @@ mod internal_tests { needs_sweep: false, }; let scanned = vec![(abs, "t".to_string())]; - let (files, _deleted) = reconcile_scan(dir.path(), scanned, &ctx, &fake).unwrap(); + let (files, _deleted) = + reconcile_scan(&TableAnchors::at_root(dir.path()), scanned, &ctx, &fake).unwrap(); assert_eq!(files.len(), 1); assert!(!files[0].trusted); } @@ -4192,7 +4377,8 @@ mod internal_tests { needs_sweep: false, }; let fake = FakeFs::default(); - let (files, deleted) = reconcile_scan(dir.path(), Vec::new(), &ctx, &fake).unwrap(); + let (files, deleted) = + reconcile_scan(&TableAnchors::at_root(dir.path()), Vec::new(), &ctx, &fake).unwrap(); assert!(files.is_empty()); assert_eq!(deleted, vec![("gone.txt".to_string(), "t".to_string())]); } @@ -4752,7 +4938,8 @@ mod internal_tests { tables, extensions: Vec::new(), functions: Vec::new(), - matcher: TableMatcher::new(&[], &[]).unwrap(), + groups: Vec::new(), + anchors: TableAnchors::at_root(root), scanned_files, poll_interval: DEFAULT_POLL_INTERVAL, persist, @@ -5160,6 +5347,103 @@ mod internal_tests { ); assert_eq!(row_names(&db), vec!["a.txt", "b.txt"]); } + + fn anchored_table(name: &str, glob: &str, anchor: &str) -> Table { + Table::new( + name, + format!("CREATE TABLE {name} (x TEXT)"), + glob, + |_| vec![], + ) + .anchored(anchor) + } + + #[test] + fn compile_groups_puts_unanchored_tables_at_the_index_root() { + let tables = vec![Table::new( + "a", + "CREATE TABLE a (x TEXT)", + "*.txt", + |_| vec![], + )]; + let (groups, anchors, names) = compile_groups(&tables, &[], Path::new("/idx")).unwrap(); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].anchor, PathBuf::from("/idx")); + assert_eq!(groups[0].tables, vec!["a".to_string()]); + assert_eq!(anchors.of("a"), Path::new("/idx")); + assert_eq!(names, vec!["a".to_string()]); + } + + #[test] + fn compile_groups_makes_one_group_per_distinct_anchor() { + let tables = vec![ + anchored_table("a", "*.txt", "/one"), + anchored_table("b", "*.md", "/two"), + anchored_table("c", "*.rs", "/one"), + ]; + let (groups, anchors, _) = compile_groups(&tables, &[], Path::new("/idx")).unwrap(); + assert_eq!(groups.len(), 2); + assert_eq!(groups[0].anchor, PathBuf::from("/one")); + assert_eq!(groups[0].tables, vec!["a".to_string(), "c".to_string()]); + assert_eq!(groups[1].anchor, PathBuf::from("/two")); + assert_eq!(anchors.of("b"), Path::new("/two")); + assert_eq!(anchors.of("c"), Path::new("/one")); + } + + #[test] + fn compile_groups_rejects_a_duplicate_name_across_anchors() { + let tables = vec![ + anchored_table("a", "*.txt", "/one"), + anchored_table("a", "*.md", "/two"), + ]; + let err = compile_groups(&tables, &[], Path::new("/idx")) + .err() + .unwrap(); + assert!(matches!(err, DirSqlError::DuplicateTable(ref name) if name == "a")); + } + + #[test] + fn anchors_of_an_unknown_table_is_the_index_root() { + let anchors = TableAnchors::at_root(Path::new("/idx")); + assert_eq!(anchors.of("nope"), Path::new("/idx")); + } + + #[test] + fn an_event_under_one_anchor_is_not_offered_to_another() { + let dir = TempDir::new().unwrap(); + let one = dir.path().join("one"); + let two = dir.path().join("two"); + let db = DirSQL::with_ignore_and_fs( + dir.path(), + vec![ + anchored_table("a", "*.txt", one.to_str().unwrap()), + anchored_table("b", "*.txt", two.to_str().unwrap()), + ], + Vec::::new(), + Arc::new(FakeFs::default()), + ) + .unwrap(); + let hits = db.groups_containing(&one.join("x.txt")); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].0.tables, vec!["a".to_string()]); + assert_eq!(hits[0].1, PathBuf::from("x.txt")); + } + + #[test] + fn an_event_under_no_anchor_is_offered_raw_to_every_group() { + let dir = TempDir::new().unwrap(); + let one = dir.path().join("one"); + let db = DirSQL::with_ignore_and_fs( + dir.path(), + vec![anchored_table("a", "*.txt", one.to_str().unwrap())], + Vec::::new(), + Arc::new(FakeFs::default()), + ) + .unwrap(); + let hits = db.groups_containing(Path::new("/elsewhere/x.txt")); + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].1, PathBuf::from("/elsewhere/x.txt")); + } } #[cfg(test)] diff --git a/packages/rust/src/persist.rs b/packages/rust/src/persist.rs index ffb1bb861..4dd9e9910 100644 --- a/packages/rust/src/persist.rs +++ b/packages/rust/src/persist.rs @@ -148,16 +148,21 @@ pub fn is_trusted( /// deterministic order. A mismatch against the cached value triggers a /// full rebuild. pub fn compute_glob_config_hash(tables: &[Table], ignore: &[String]) -> String { - let mut entries: BTreeMap = BTreeMap::new(); + let mut entries: BTreeMap)> = BTreeMap::new(); for table in tables { entries.insert( table.name.clone(), - (table.ddl.clone(), table.glob.clone(), table.strict), + ( + table.ddl.clone(), + table.glob.clone(), + table.strict, + table.anchor_key(), + ), ); } let mut hasher = blake3::Hasher::new(); hasher.update(b"v1\n"); - for (name, (ddl, glob, strict)) in &entries { + for (name, (ddl, glob, strict, anchor)) in &entries { hasher.update(name.as_bytes()); hasher.update(b"\0"); hasher.update(ddl.as_bytes()); @@ -165,6 +170,10 @@ pub fn compute_glob_config_hash(tables: &[Table], ignore: &[String]) -> String { hasher.update(glob.as_bytes()); hasher.update(b"\0"); hasher.update(if *strict { b"1" } else { b"0" }); + if let Some(anchor) = anchor { + hasher.update(b"\0anchor\0"); + hasher.update(anchor.as_bytes()); + } hasher.update(b"\n"); } hasher.update(b"--ignore--\n"); @@ -451,6 +460,17 @@ mod tests { assert_ne!(h1, h2); } + #[test] + fn glob_config_hash_changes_when_anchor_changes() { + let t1 = Table::new("a", "CREATE TABLE a (x TEXT)", "*.json", |_| vec![]); + let t2 = t1.clone().anchored("/elsewhere"); + let t3 = t1.clone().anchored("/other"); + let unanchored = compute_glob_config_hash(&[t1], &[]); + let at_elsewhere = compute_glob_config_hash(&[t2], &[]); + assert_ne!(unanchored, at_elsewhere); + assert_ne!(at_elsewhere, compute_glob_config_hash(&[t3], &[])); + } + #[test] fn glob_config_hash_changes_when_strict_changes() { let mut t1 = Table::new("a", "CREATE TABLE a (x TEXT)", "*.json", |_| vec![]); diff --git a/packages/rust/tests/table_anchor.rs b/packages/rust/tests/table_anchor.rs new file mode 100644 index 000000000..55f54a89c --- /dev/null +++ b/packages/rust/tests/table_anchor.rs @@ -0,0 +1,70 @@ +//! A table may be anchored away from the index root: its glob matches paths +//! relative to its own anchor, and the index root keeps governing the rest. + +use dirsql::{DirSQL, Row, Table, Value}; +use std::fs; +use tempfile::TempDir; + +fn path_row(path: &str) -> Vec { + vec![Row::from([("path".to_string(), Value::Text(path.into()))])] +} + +#[test] +fn an_anchored_table_scans_its_anchor_not_the_index_root() { + let root = TempDir::new().unwrap(); + fs::write(root.path().join("decoy.txt"), "x").unwrap(); + let anchor = TempDir::new().unwrap(); + fs::write(anchor.path().join("present.txt"), "x").unwrap(); + + let table = Table::new("items", "CREATE TABLE items (path TEXT)", "*.txt", |path| { + path_row(path) + }) + .anchored(anchor.path()); + + let db = DirSQL::builder() + .root(root.path()) + .table(table) + .build() + .unwrap(); + + let rows = db.query("SELECT path FROM items").unwrap(); + assert_eq!(rows.len(), 1); + let Value::Text(path) = &rows[0]["path"] else { + panic!("path is text"); + }; + assert!(path.ends_with("present.txt"), "got {path}"); +} + +#[test] +fn an_unanchored_table_still_scans_the_index_root() { + let root = TempDir::new().unwrap(); + fs::write(root.path().join("a.txt"), "x").unwrap(); + let anchor = TempDir::new().unwrap(); + fs::write(anchor.path().join("b.txt"), "x").unwrap(); + + let at_root = Table::new( + "at_root", + "CREATE TABLE at_root (path TEXT)", + "*.txt", + path_row, + ); + let elsewhere = Table::new( + "elsewhere", + "CREATE TABLE elsewhere (path TEXT)", + "*.txt", + path_row, + ) + .anchored(anchor.path()); + + let db = DirSQL::builder() + .root(root.path()) + .tables(vec![at_root, elsewhere]) + .build() + .unwrap(); + + let a = db.query("SELECT path FROM at_root").unwrap(); + let b = db.query("SELECT path FROM elsewhere").unwrap(); + assert_eq!(a.len(), 1); + assert_eq!(b.len(), 1); + assert_ne!(a[0]["path"], b[0]["path"]); +} diff --git a/packages/ts/e2e-attestations/claude-1232-anchor-plumbing.json b/packages/ts/e2e-attestations/claude-1232-anchor-plumbing.json new file mode 100644 index 000000000..34f4107ca --- /dev/null +++ b/packages/ts/e2e-attestations/claude-1232-anchor-plumbing.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1791252658, + "exit_code": 0, + "commit": "c98dc36dee869ebe0a8f8537457e8c6180325c09", + "branch": "claude/1232-anchor-plumbing" +}