diff --git a/packages/python/changelog.d/2026-10-06-config-links.md b/packages/python/changelog.d/2026-10-06-config-links.md new file mode 100644 index 000000000..f7b499d1c --- /dev/null +++ b/packages/python/changelog.d/2026-10-06-config-links.md @@ -0,0 +1,3 @@ +**Changed** A declared `[[table]] glob` skips `node_modules` unless the glob +names it, and follows symlinks, as a path-table does. `**/*.js` no longer lists +`node_modules/...`; `link.md -> top.md` and `linkdir/*` now list their rows. diff --git a/packages/python/e2e-attestations/claude-1258-config-links.json b/packages/python/e2e-attestations/claude-1258-config-links.json new file mode 100644 index 000000000..f729ec37e --- /dev/null +++ b/packages/python/e2e-attestations/claude-1258-config-links.json @@ -0,0 +1,7 @@ +{ + "command": "uv run python -m pytest tests/e2e/glob_speed_symlink_test.py tests/e2e/glob_speed_node_modules_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/query_contract_test.py -x -q", + "ran_at": 1791338642, + "exit_code": 0, + "commit": "81890d53f01c2d6e6e833e0748a6ff952e3382c5", + "branch": "claude/1258-config-links" +} diff --git a/packages/python/migrations.d/2026-10-06-config-links.md b/packages/python/migrations.d/2026-10-06-config-links.md new file mode 100644 index 000000000..1a00dc32f --- /dev/null +++ b/packages/python/migrations.d/2026-10-06-config-links.md @@ -0,0 +1,43 @@ +### Config globs skip node_modules unless named, and follow symlinks + +#### Summary + +`glob = "**/*.js"` used to list files under any `node_modules`; it now does +not, unless a component of the glob is `node_modules`. Symlinked files and +directories, which a config glob used to ignore, are now followed as in a +path-table and in bash. + +#### Required changes + +| Before | After | +| --- | --- | +| `glob = "**/*.js"` (included `node_modules/`) | `glob = "node_modules/**/*.js"` to list it | + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +- A glob with no `node_modules` component never matches under one. +- A symlinked file or directory the glob reaches now yields rows. + +#### Verification + +```bash +cd "$(mktemp -d)" +mkdir real node_modules +echo '[{"p":"r"}]' > real/r.json +echo '[{"p":"n"}]' > node_modules/n.json +ln -s real linkdir +cat > .dirsql.toml <<'TOML' +[[table]] +name = "t" +glob = "**/*.json" +ddl = "CREATE TABLE t (p TEXT)" +on-file = "jq -sc add" +TOML +dirsql query "SELECT p FROM t" -c .dirsql.toml +# expected: no row from node_modules; rows for real/ (symlinked linkdir/ is +# reached by `linkdir/*.json`, not by `**`) +``` diff --git a/packages/rust/changelog.d/2026-10-06-config-links.md b/packages/rust/changelog.d/2026-10-06-config-links.md new file mode 100644 index 000000000..f7b499d1c --- /dev/null +++ b/packages/rust/changelog.d/2026-10-06-config-links.md @@ -0,0 +1,3 @@ +**Changed** A declared `[[table]] glob` skips `node_modules` unless the glob +names it, and follows symlinks, as a path-table does. `**/*.js` no longer lists +`node_modules/...`; `link.md -> top.md` and `linkdir/*` now list their rows. diff --git a/packages/rust/migrations.d/2026-10-06-config-links.md b/packages/rust/migrations.d/2026-10-06-config-links.md new file mode 100644 index 000000000..1a00dc32f --- /dev/null +++ b/packages/rust/migrations.d/2026-10-06-config-links.md @@ -0,0 +1,43 @@ +### Config globs skip node_modules unless named, and follow symlinks + +#### Summary + +`glob = "**/*.js"` used to list files under any `node_modules`; it now does +not, unless a component of the glob is `node_modules`. Symlinked files and +directories, which a config glob used to ignore, are now followed as in a +path-table and in bash. + +#### Required changes + +| Before | After | +| --- | --- | +| `glob = "**/*.js"` (included `node_modules/`) | `glob = "node_modules/**/*.js"` to list it | + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +- A glob with no `node_modules` component never matches under one. +- A symlinked file or directory the glob reaches now yields rows. + +#### Verification + +```bash +cd "$(mktemp -d)" +mkdir real node_modules +echo '[{"p":"r"}]' > real/r.json +echo '[{"p":"n"}]' > node_modules/n.json +ln -s real linkdir +cat > .dirsql.toml <<'TOML' +[[table]] +name = "t" +glob = "**/*.json" +ddl = "CREATE TABLE t (p TEXT)" +on-file = "jq -sc add" +TOML +dirsql query "SELECT p FROM t" -c .dirsql.toml +# expected: no row from node_modules; rows for real/ (symlinked linkdir/ is +# reached by `linkdir/*.json`, not by `**`) +``` diff --git a/packages/rust/src/matcher.rs b/packages/rust/src/matcher.rs index 3878db2bd..b19e7c55b 100644 --- a/packages/rust/src/matcher.rs +++ b/packages/rust/src/matcher.rs @@ -1,5 +1,5 @@ use crate::posix_class::expand_posix_classes; -use crate::scanner::{PathGlob, compile_glob}; +use crate::scanner::{PathGlob, compile_glob, compile_globs}; use globset::GlobBuilder; use regex::{Regex, RegexBuilder}; use std::ffi::OsStr; @@ -80,23 +80,30 @@ pub struct MatchResult { pub table_name: String, } +const NODE_MODULES: &str = "node_modules"; + #[derive(Clone)] struct PatternEntry { pattern: PathGlob, table_name: String, + names_node_modules: bool, } /// Maps file paths to table names based on glob patterns. /// Every matching pattern fires: a file matching N patterns yields N /// `MatchResult`s (one per table), so a file can belong to multiple tables. -/// An ignore list filters paths entirely. `{name}` placeholders in glob -/// patterns are accepted and behave like `*`. +/// An ignore list filters paths entirely. #[derive(Clone)] pub struct TableMatcher { entries: Vec, ignore_set: Vec, ignore_dir_set: Vec, gitignore: bool, + walk: PathGlob, +} + +fn has_node_modules_component(path: &Path) -> bool { + path.components().any(|c| c.as_os_str() == NODE_MODULES) } impl TableMatcher { @@ -107,8 +114,11 @@ impl TableMatcher { entries.push(PatternEntry { pattern: compile_glob(pattern)?, table_name: table_name.to_string(), + names_node_modules: pattern.split('/').any(|c| c == NODE_MODULES), }); } + let globs: Vec<&str> = mappings.iter().map(|(pattern, _)| *pattern).collect(); + let walk = compile_globs(&globs)?; let mut ignore_set = Vec::new(); let mut ignore_dir_set = Vec::new(); @@ -124,6 +134,7 @@ impl TableMatcher { ignore_set, ignore_dir_set, gitignore: false, + walk, }) } @@ -143,18 +154,27 @@ impl TableMatcher { pub fn match_all(&self, path: &Path) -> Vec { self.entries .iter() - .filter(|entry| entry.pattern.is_match_unhidden(path)) + .filter(|entry| { + entry.pattern.is_match_unhidden(path) + && (entry.names_node_modules || !has_node_modules_component(path)) + }) .map(|entry| MatchResult { table_name: entry.table_name.clone(), }) .collect() } - /// Whether any table's glob spells the dot-named `name`. - pub(crate) fn spells_dot_name(&self, name: &OsStr) -> bool { - self.entries - .iter() - .any(|entry| entry.pattern.spells_dot_name(name)) + /// The one glob a walk for every table at once follows. + pub(crate) fn walk_glob(&self) -> &PathGlob { + &self.walk + } + + /// Whether the walk may skip a directory called `name` outright: it is + /// `node_modules` and no table's glob names it. + pub(crate) fn prunes_directory(&self, name: &OsStr) -> bool { + name == NODE_MODULES + && !self.entries.is_empty() + && !self.entries.iter().any(|e| e.names_node_modules) } /// Returns true if the path matches any ignore pattern. @@ -196,6 +216,30 @@ mod tests { assert!(matcher.with_gitignore(true).respects_gitignore()); } + #[test] + fn node_modules_is_skipped_unless_a_table_names_it() { + let matcher = + TableMatcher::new(&[("**/*.js", "all"), ("node_modules/*.js", "nm")], &[]).unwrap(); + assert_eq!(names(&matcher, "pkg/a.js"), vec!["all"]); + assert_eq!( + names(&matcher, "pkg/node_modules/a.js"), + Vec::::new() + ); + assert_eq!(names(&matcher, "node_modules/a.js"), vec!["nm"]); + } + + #[test] + fn the_walk_prunes_node_modules_only_while_no_table_names_it() { + let name = OsStr::new("node_modules"); + let plain = TableMatcher::new(&[("**/*.js", "t")], &[]).unwrap(); + let naming = TableMatcher::new(&[("node_modules/*.js", "t")], &[]).unwrap(); + let none = TableMatcher::new(&[], &[]).unwrap(); + assert!(plain.prunes_directory(name)); + assert!(!plain.prunes_directory(OsStr::new("src"))); + assert!(!naming.prunes_directory(name)); + assert!(!none.prunes_directory(name)); + } + #[test] fn match_all_returns_empty_for_no_match() { let matcher = TableMatcher::new(&[("*.csv", "data")], &[]).unwrap(); diff --git a/packages/rust/src/scanner.rs b/packages/rust/src/scanner.rs index 5abf7e626..33be04602 100644 --- a/packages/rust/src/scanner.rs +++ b/packages/rust/src/scanner.rs @@ -104,7 +104,7 @@ fn scan_below( // regardless of the absolute root directory. let walker = Walk { ignore: matcher, - glob: None, + glob: Some(matcher.walk_glob()), gitignore, in_repo: repo.is_some(), frames: repo.map_or_else(Vec::new, |top| { @@ -318,7 +318,16 @@ impl PathGlob { /// it a lone `*` would cross `/` and the explicit non-recursive spelling would /// silently recurse. `**` still crosses separators. pub fn compile_glob(pattern: &str) -> Result { - let words = crate::brace::expand(pattern); + compile_globs(&[pattern]) +} + +/// One [`PathGlob`] over every word of every pattern: the walk a set of +/// tables shares, entering what any of them could reach. +pub fn compile_globs(patterns: &[&str]) -> Result { + let words: Vec = patterns + .iter() + .flat_map(|pattern| crate::brace::expand(pattern)) + .collect(); let mut files = Vec::new(); let mut spelled_dot_names = Vec::new(); for word in &words { @@ -403,7 +412,11 @@ fn walk( visit: &mut dyn FnMut(PathBuf, &Path), ) { let rel = start.strip_prefix(root).unwrap_or(start); - let states = walker.glob.map_or_else(Vec::new, PathGlob::start); + let states = walker.glob.map_or_else(Vec::new, |glob| { + rel.components().fold(glob.start(), |states, component| { + glob.step(&states, component.as_os_str(), Kind::Dir) + }) + }); let place = Place { walk: walker, dir: start.to_path_buf(), @@ -590,16 +603,16 @@ impl Walk<'_> { /// `.gitignore` files in force. fn admits(&self, depth: usize, is_dir: bool, name: &OsStr, path: &Path, rel: &Path) -> bool { should_descend(depth, is_dir, name, rel, self.ignore) + && !(is_dir && self.ignore.prunes_directory(name)) && self.admits_name(name) && (self.frames.is_empty() || !is_gitignored(&self.frames, path, is_dir)) } fn admits_name(&self, name: &OsStr) -> bool { - !is_dot_named(name) - || match self.glob { - Some(glob) => glob.spells_dot_name(name), - None => self.ignore.spells_dot_name(name), - } + match self.glob { + Some(glob) => !is_dot_named(name) || glob.spells_dot_name(name), + None => true, + } } fn next_states(&self, states: &[usize], name: &OsStr, kind: Kind) -> Vec { @@ -1405,23 +1418,21 @@ mod tests { } #[test] - fn a_table_walk_admits_a_dot_name_only_a_table_glob_spells() { - let ignore = TableMatcher::new(&[(".env", "t")], &[]).unwrap(); - let walk = walk_with(&ignore, None, Vec::new()); - assert!(walk.admits( - 1, - false, - OsStr::new(".env"), - Path::new("/r/.env"), - Path::new(".env") - )); - assert!(!walk.admits( - 1, - false, - OsStr::new(".other"), - Path::new("/r/.other"), - Path::new(".other") - )); + fn a_table_walk_skips_a_node_modules_directory_no_table_names() { + let ignore = TableMatcher::new(&[("**/*.js", "t")], &[]).unwrap(); + let walk = walk_with(&ignore, Some(ignore.walk_glob()), Vec::new()); + let admits = |is_dir, name: &str| { + walk.admits( + 1, + is_dir, + OsStr::new(name), + Path::new(name), + Path::new(name), + ) + }; + assert!(!admits(true, "node_modules")); + assert!(admits(false, "node_modules")); + assert!(admits(true, "src")); } #[test] diff --git a/packages/rust/tests/config_glob_parity.rs b/packages/rust/tests/config_glob_parity.rs index db275edf0..8ab8dbcab 100644 --- a/packages/rust/tests/config_glob_parity.rs +++ b/packages/rust/tests/config_glob_parity.rs @@ -156,3 +156,35 @@ fn a_spelled_dot_directory_is_listed() { assert_eq!(path_table_paths(root.path(), ".hid/*.md"), expected); assert_eq!(config_paths(root.path(), ".hid/*.md"), expected); } + +#[test] +fn node_modules_is_skipped_unless_the_glob_names_it() { + let root = tree(&["top.js", "node_modules/top.js", "pkg/node_modules/dep/i.js"]); + let expected = vec!["top.js"]; + assert_eq!(path_table_paths(root.path(), "**/*.js"), expected); + assert_eq!(config_paths(root.path(), "**/*.js"), expected); +} + +#[test] +fn a_glob_naming_node_modules_lists_it() { + let root = tree(&["top.js", "node_modules/top.js"]); + let expected = vec!["node_modules/top.js"]; + assert_eq!(path_table_paths(root.path(), "node_modules/*.js"), expected); + assert_eq!(config_paths(root.path(), "node_modules/*.js"), expected); +} + +#[cfg(unix)] +#[test] +fn a_symlinked_file_and_directory_are_followed() { + use std::os::unix::fs::symlink; + let root = tree(&["top.md", "real/r.md"]); + symlink("top.md", root.path().join("link.md")).unwrap(); + symlink("real", root.path().join("linkdir")).unwrap(); + for (glob, expected) in [ + ("link.md", vec!["link.md"]), + ("linkdir/*", vec!["linkdir/r.md"]), + ] { + assert_eq!(path_table_paths(root.path(), glob), expected, "{glob}"); + assert_eq!(config_paths(root.path(), glob), expected, "{glob}"); + } +} diff --git a/packages/rust/tests/init_integration.rs b/packages/rust/tests/init_integration.rs index 5fd3e840d..1a173b34a 100644 --- a/packages/rust/tests/init_integration.rs +++ b/packages/rust/tests/init_integration.rs @@ -46,8 +46,11 @@ fn writes_the_default_config_verbatim() { "init must write DEFAULT_CONFIG_TOML verbatim", ); - dirsql::DirSQL::from_config_path(&config_path) - .expect("config produced by `dirsql init` must load via from_config_path"); + dirsql::DirSQL::builder() + .root(cwd.path()) + .config(&config_path) + .build() + .expect("config produced by `dirsql init` must load"); } #[test] @@ -104,7 +107,10 @@ fn force_flag_overwrites_existing_config() { let written = fs::read_to_string(cwd.path().join(".dirsql.toml")).unwrap(); assert_eq!(written, dirsql::cli::DEFAULT_CONFIG_TOML); - dirsql::DirSQL::from_config_path(cwd.path().join(".dirsql.toml")) + dirsql::DirSQL::builder() + .root(cwd.path()) + .config(cwd.path().join(".dirsql.toml")) + .build() .expect("forced-overwrite config must load"); } diff --git a/packages/ts/changelog.d/2026-10-06-config-links.md b/packages/ts/changelog.d/2026-10-06-config-links.md new file mode 100644 index 000000000..f7b499d1c --- /dev/null +++ b/packages/ts/changelog.d/2026-10-06-config-links.md @@ -0,0 +1,3 @@ +**Changed** A declared `[[table]] glob` skips `node_modules` unless the glob +names it, and follows symlinks, as a path-table does. `**/*.js` no longer lists +`node_modules/...`; `link.md -> top.md` and `linkdir/*` now list their rows. diff --git a/packages/ts/e2e-attestations/claude-1258-config-links.json b/packages/ts/e2e-attestations/claude-1258-config-links.json new file mode 100644 index 000000000..88c339807 --- /dev/null +++ b/packages/ts/e2e-attestations/claude-1258-config-links.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1791338676, + "exit_code": 0, + "commit": "d686573e68751258dc25c982adcfaef986ca6176", + "branch": "claude/1258-config-links" +} diff --git a/packages/ts/migrations.d/2026-10-06-config-links.md b/packages/ts/migrations.d/2026-10-06-config-links.md new file mode 100644 index 000000000..1a00dc32f --- /dev/null +++ b/packages/ts/migrations.d/2026-10-06-config-links.md @@ -0,0 +1,43 @@ +### Config globs skip node_modules unless named, and follow symlinks + +#### Summary + +`glob = "**/*.js"` used to list files under any `node_modules`; it now does +not, unless a component of the glob is `node_modules`. Symlinked files and +directories, which a config glob used to ignore, are now followed as in a +path-table and in bash. + +#### Required changes + +| Before | After | +| --- | --- | +| `glob = "**/*.js"` (included `node_modules/`) | `glob = "node_modules/**/*.js"` to list it | + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +- A glob with no `node_modules` component never matches under one. +- A symlinked file or directory the glob reaches now yields rows. + +#### Verification + +```bash +cd "$(mktemp -d)" +mkdir real node_modules +echo '[{"p":"r"}]' > real/r.json +echo '[{"p":"n"}]' > node_modules/n.json +ln -s real linkdir +cat > .dirsql.toml <<'TOML' +[[table]] +name = "t" +glob = "**/*.json" +ddl = "CREATE TABLE t (p TEXT)" +on-file = "jq -sc add" +TOML +dirsql query "SELECT p FROM t" -c .dirsql.toml +# expected: no row from node_modules; rows for real/ (symlinked linkdir/ is +# reached by `linkdir/*.json`, not by `**`) +```