diff --git a/packages/python/changelog.d/2026-10-06-config-gitignore.md b/packages/python/changelog.d/2026-10-06-config-gitignore.md new file mode 100644 index 000000000..1b774e752 --- /dev/null +++ b/packages/python/changelog.d/2026-10-06-config-gitignore.md @@ -0,0 +1,3 @@ +**Changed** Declared `[[table]]` tables respect `.gitignore` by default, as +path-tables do: inside a git repo, a file a `.gitignore` ignores is no longer a +row, at startup or while watching. `--no-ignore` turns this off for them too. diff --git a/packages/python/e2e-attestations/claude-1258-config-gitignore.json b/packages/python/e2e-attestations/claude-1258-config-gitignore.json new file mode 100644 index 000000000..25c6e95a4 --- /dev/null +++ b/packages/python/e2e-attestations/claude-1258-config-gitignore.json @@ -0,0 +1,7 @@ +{ + "command": "uv run python -m pytest tests/e2e/glob_speed_gitignore_test.py tests/e2e/glob_speed_dotfiles_test.py tests/e2e/glob_speed_recursive_test.py tests/e2e/glob_speed_matchers_test.py tests/e2e/query_contract_test.py -x -q", + "ran_at": 1791329772, + "exit_code": 0, + "commit": "f828a8181c981ab9c5c775117cb60edf3ccd9dcc", + "branch": "claude/1258-config-gitignore" +} diff --git a/packages/python/migrations.d/2026-10-06-config-gitignore.md b/packages/python/migrations.d/2026-10-06-config-gitignore.md new file mode 100644 index 000000000..773415440 --- /dev/null +++ b/packages/python/migrations.d/2026-10-06-config-gitignore.md @@ -0,0 +1,45 @@ +### Config tables respect .gitignore + +#### Summary + +A declared `[[table]]` used to read every file its glob matched, whatever +`.gitignore` said. It now skips gitignored files, as a path-table does. This +breaks a config that relied on indexing an ignored file; pass `--no-ignore` +(or `no_ignore` in the SDKs) to restore the old behavior. + +#### Required changes + +| Before | After | +| --- | --- | +| a table indexed gitignored files | pass `--no-ignore`, or stop ignoring the file | + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +- Inside a git repo (a directory holding `.git`), files a `.gitignore` ignores + are not rows of a `[[table]]`, at startup or while watching. +- Outside a repo nothing changes. + +#### Verification + +```bash +cd "$(mktemp -d)" +git init -q +printf 'skip.json\n' > .gitignore +echo '[{"p":"keep"}]' > keep.json +echo '[{"p":"skip"}]' > skip.json +cat > .dirsql.toml <<'TOML' +[[table]] +name = "t" +glob = "*.json" +ddl = "CREATE TABLE t (p TEXT)" +on-file = "jq -sc add" +TOML +dirsql query "SELECT p FROM t" -c .dirsql.toml +# expected: [{"p":"keep"}] +dirsql query "SELECT p FROM t ORDER BY p" -c .dirsql.toml --no-ignore +# expected: [{"p":"keep"},{"p":"skip"}] +``` diff --git a/packages/rust/changelog.d/2026-10-06-config-gitignore.md b/packages/rust/changelog.d/2026-10-06-config-gitignore.md new file mode 100644 index 000000000..1b774e752 --- /dev/null +++ b/packages/rust/changelog.d/2026-10-06-config-gitignore.md @@ -0,0 +1,3 @@ +**Changed** Declared `[[table]]` tables respect `.gitignore` by default, as +path-tables do: inside a git repo, a file a `.gitignore` ignores is no longer a +row, at startup or while watching. `--no-ignore` turns this off for them too. diff --git a/packages/rust/migrations.d/2026-10-06-config-gitignore.md b/packages/rust/migrations.d/2026-10-06-config-gitignore.md new file mode 100644 index 000000000..773415440 --- /dev/null +++ b/packages/rust/migrations.d/2026-10-06-config-gitignore.md @@ -0,0 +1,45 @@ +### Config tables respect .gitignore + +#### Summary + +A declared `[[table]]` used to read every file its glob matched, whatever +`.gitignore` said. It now skips gitignored files, as a path-table does. This +breaks a config that relied on indexing an ignored file; pass `--no-ignore` +(or `no_ignore` in the SDKs) to restore the old behavior. + +#### Required changes + +| Before | After | +| --- | --- | +| a table indexed gitignored files | pass `--no-ignore`, or stop ignoring the file | + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +- Inside a git repo (a directory holding `.git`), files a `.gitignore` ignores + are not rows of a `[[table]]`, at startup or while watching. +- Outside a repo nothing changes. + +#### Verification + +```bash +cd "$(mktemp -d)" +git init -q +printf 'skip.json\n' > .gitignore +echo '[{"p":"keep"}]' > keep.json +echo '[{"p":"skip"}]' > skip.json +cat > .dirsql.toml <<'TOML' +[[table]] +name = "t" +glob = "*.json" +ddl = "CREATE TABLE t (p TEXT)" +on-file = "jq -sc add" +TOML +dirsql query "SELECT p FROM t" -c .dirsql.toml +# expected: [{"p":"keep"}] +dirsql query "SELECT p FROM t ORDER BY p" -c .dirsql.toml --no-ignore +# expected: [{"p":"keep"},{"p":"skip"}] +``` diff --git a/packages/rust/src/lib.rs b/packages/rust/src/lib.rs index 6d262e406..f5a930f96 100644 --- a/packages/rust/src/lib.rs +++ b/packages/rust/src/lib.rs @@ -732,6 +732,14 @@ impl DirSQL { if group.matcher.is_ignored(&rel_path_buf) { continue; } + if group.matcher.respects_gitignore() + && scanner::is_gitignored_path( + &abs_path, + self.inner.fs.is_dir(&abs_path).unwrap_or(false), + ) + { + continue; + } // A directory that appears whole (`mkdir`, or a populated directory // renamed into the tree) arrives as one event naming the directory and @@ -1110,6 +1118,13 @@ impl DirSQL { } = resolved; let (groups, anchors, table_names) = compile_groups(&tables, &ignore, &root)?; + let groups = groups + .into_iter() + .map(|group| AnchorGroup { + matcher: group.matcher.with_gitignore(!no_ignore), + ..group + }) + .collect::>(); // Resolve the persistent context before scanning, so the scan can // consult the cached file index. @@ -2935,6 +2950,23 @@ mod internal_tests { assert_eq!(found, vec!["a.txt"]); } + /// The startup walk honors `.gitignore` unless the build asked not to. + #[test] + fn prepare_hands_the_walk_a_matcher_that_honors_gitignore_unless_no_ignore() { + let dir = TempDir::new().unwrap(); + for no_ignore in [false, true] { + let honored = Cell::new(None); + let mut resolved = resolved_over(dir.path(), vec![txt_table()], false); + resolved.no_ignore = no_ignore; + DirSQL::prepare_resolved_with(resolved, &|_, matcher, _| { + honored.set(Some(matcher.respects_gitignore())); + Vec::new() + }) + .unwrap(); + assert_eq!(honored.get(), Some(!no_ignore)); + } + } + /// Skipping the walk skips only the walk: with persist on, the cache is /// still opened and reconciled against the (empty) scan. #[test] diff --git a/packages/rust/src/matcher.rs b/packages/rust/src/matcher.rs index efd029402..cda0180a4 100644 --- a/packages/rust/src/matcher.rs +++ b/packages/rust/src/matcher.rs @@ -97,6 +97,7 @@ pub struct TableMatcher { entries: Vec, ignore_set: Vec, ignore_dir_set: Vec, + gitignore: bool, } /// Byte spans of the `{name}` placeholders in `pattern`, in order: `(start, @@ -182,9 +183,20 @@ impl TableMatcher { entries, ignore_set, ignore_dir_set, + gitignore: false, }) } + /// Whether a scan under this matcher honors `.gitignore` files. + pub fn with_gitignore(mut self, gitignore: bool) -> Self { + self.gitignore = gitignore; + self + } + + pub(crate) fn respects_gitignore(&self) -> bool { + self.gitignore + } + /// Returns one [`MatchResult`] per matching pattern, in declaration order. /// A file matching N patterns yields N results (fan-out). Empty when /// nothing matches. @@ -230,6 +242,13 @@ mod tests { assert_eq!(names(&matcher, "report.csv"), vec!["data"]); } + #[test] + fn a_matcher_honors_gitignore_only_once_asked_to() { + let matcher = TableMatcher::new(&[], &[]).unwrap(); + assert!(!matcher.respects_gitignore()); + assert!(matcher.with_gitignore(true).respects_gitignore()); + } + #[test] fn match_all_returns_empty_for_no_match() { let matcher = TableMatcher::new(&[("*.csv", "data")], &[]).unwrap(); diff --git a/packages/rust/src/scanner.rs b/packages/rust/src/scanner.rs index 22a2a6d52..3ab8d1266 100644 --- a/packages/rust/src/scanner.rs +++ b/packages/rust/src/scanner.rs @@ -93,15 +93,23 @@ fn scan_below( ) -> Vec<(PathBuf, String)> { let mut results = Vec::new(); let mut seen: u64 = 0; + let gitignore = matcher.respects_gitignore(); + let repo = if gitignore { + enclosing_repo(start, &holds_git) + } else { + None + }; // Match against relative path so globs like "comments/**/*.jsonl" work // regardless of the absolute root directory. let walker = Walk { ignore: matcher, glob: None, - gitignore: false, - in_repo: false, - frames: Vec::new(), + gitignore, + in_repo: repo.is_some(), + frames: repo.map_or_else(Vec::new, |top| { + gitignores_above(start, top, &load_gitignore) + }), dirs: false, }; walk( @@ -741,6 +749,26 @@ fn load_gitignore(dir: &Path) -> Option { builder.build().ok() } +/// Whether the `.gitignore` files of the repo enclosing `path` ignore it, or +/// a directory above it, as the walk would have: for a path a watcher reports +/// without having walked to it. +pub(crate) fn is_gitignored_path(path: &Path, is_dir: bool) -> bool { + let Some(top) = path + .parent() + .and_then(|dir| enclosing_repo(dir, &holds_git)) + else { + return false; + }; + for frame in gitignores_above(path, top, &load_gitignore).iter().rev() { + match frame.matched_path_or_any_parents(path, is_dir) { + Match::Ignore(_) => return true, + Match::Whitelist(_) => return false, + Match::None => {} + } + } + false +} + /// Whether the `.gitignore` files in force mark `path` ignored. Deeper files /// take precedence (git's rule), and a whitelisting `!pattern` un-ignores. fn is_gitignored(frames: &[Arc], path: &Path, is_dir: bool) -> bool { @@ -1664,4 +1692,14 @@ mod tests { let frames = [frame("", &["!keep.log"]), frame("sub", &["*.log"])]; assert!(is_gitignored(&frames, Path::new("sub/keep.log"), false)); } + + #[test] + fn is_gitignored_path_ignores_a_path_the_repo_gitignore_matches() { + let repo = tempfile::tempdir().unwrap(); + fs::create_dir(repo.path().join(".git")).unwrap(); + fs::write(repo.path().join(".gitignore"), "*.log\n").unwrap(); + + assert!(is_gitignored_path(&repo.path().join("debug.log"), false)); + assert!(!is_gitignored_path(&repo.path().join("app.js"), false)); + } } diff --git a/packages/rust/tests/config_gitignore_watch.rs b/packages/rust/tests/config_gitignore_watch.rs new file mode 100644 index 000000000..286b0958f --- /dev/null +++ b/packages/rust/tests/config_gitignore_watch.rs @@ -0,0 +1,50 @@ +//! A live config table skips a gitignored file the way its startup scan does. +//! Real `notify` watcher, real filesystem, real SQLite, SDK public API. + +use std::collections::HashMap; +use std::time::{Duration, Instant}; +use std::{fs, thread}; + +use dirsql::{DirSQL, RowEvent, Table, Value}; +use tempfile::TempDir; + +fn names(db: &DirSQL) -> Vec { + let mut out: Vec = db + .query("SELECT name FROM t") + .unwrap() + .iter() + .map(|r| match r.get("name") { + Some(Value::Text(s)) => s.clone(), + other => panic!("name was not text: {other:?}"), + }) + .collect(); + out.sort(); + out +} + +#[test] +fn a_gitignored_file_created_while_watching_is_not_indexed() { + let dir = TempDir::new().unwrap(); + fs::create_dir(dir.path().join(".git")).unwrap(); + fs::write(dir.path().join(".gitignore"), "skip.md\n").unwrap(); + let table = Table::new("t", "CREATE TABLE t (name TEXT)", "*.md", |path| { + let name = path.rsplit(['/', '\\']).next().unwrap().to_string(); + vec![HashMap::from([("name".into(), Value::Text(name))])] + }); + let db = DirSQL::new(dir.path(), vec![table]).unwrap(); + db.start_watching().unwrap(); + thread::sleep(Duration::from_millis(250)); + + fs::write(dir.path().join("skip.md"), "x").unwrap(); + fs::write(dir.path().join("keep.md"), "x").unwrap(); + + let deadline = Instant::now() + Duration::from_secs(5); + let mut seen = Vec::new(); + while Instant::now() < deadline && !seen.iter().any(|e| matches!(e, RowEvent::Insert { .. })) { + seen.extend(db.poll_events(Duration::from_millis(200)).unwrap()); + } + thread::sleep(Duration::from_millis(500)); + seen.extend(db.poll_events(Duration::from_millis(200)).unwrap()); + + assert_eq!(names(&db), vec!["keep.md"], "events: {seen:?}"); +} diff --git a/packages/rust/tests/config_glob_parity.rs b/packages/rust/tests/config_glob_parity.rs index 2d6ab92b3..e579ce862 100644 --- a/packages/rust/tests/config_glob_parity.rs +++ b/packages/rust/tests/config_glob_parity.rs @@ -20,6 +20,10 @@ fn tree(files: &[&str]) -> TempDir { } fn config_paths(root: &Path, glob: &str) -> Vec { + config_paths_with(root, glob, false) +} + +fn config_paths_with(root: &Path, glob: &str, no_ignore: bool) -> Vec { let base = root.to_path_buf(); let table = Table::new("t", "CREATE TABLE t (p TEXT)", glob, move |path| { let rel = Path::new(path) @@ -30,7 +34,12 @@ fn config_paths(root: &Path, glob: &str) -> Vec { Value::Text(rel.to_string_lossy().replace('\\', "/")), )])] }); - let db = DirSQL::new(root, vec![table]).unwrap(); + let db = DirSQL::builder() + .root(root) + .table(table) + .no_ignore(no_ignore) + .build() + .unwrap(); texts(&db, "SELECT p FROM t", "p") } @@ -61,6 +70,30 @@ fn a_brace_range_expands_as_it_does_in_a_path_table() { assert_eq!(config_paths(root.path(), "f{1..3}.md"), expected); } +fn repo(files: &[&str], gitignore: &str) -> TempDir { + let root = tree(files); + fs::create_dir(root.path().join(".git")).unwrap(); + fs::write(root.path().join(".gitignore"), gitignore).unwrap(); + root +} + +#[test] +fn a_gitignored_file_is_not_a_row_of_a_config_table() { + let root = repo(&["keep.md", "skip.md", "dist/out.md"], "skip.md\ndist/\n"); + let expected = vec!["keep.md"]; + assert_eq!(path_table_paths(root.path(), "**/*.md"), expected); + assert_eq!(config_paths(root.path(), "**/*.md"), expected); +} + +#[test] +fn no_ignore_lists_a_gitignored_file_in_a_config_table() { + let root = repo(&["keep.md", "skip.md"], "skip.md\n"); + assert_eq!( + config_paths_with(root.path(), "*.md", true), + vec!["keep.md", "skip.md"] + ); +} + #[test] fn an_unbalanced_brace_or_bracket_is_literal_as_it_is_in_a_path_table() { let root = tree(&["{a.md", "a}.md", "[a.md", "a.md"]); diff --git a/packages/ts/changelog.d/2026-10-06-config-gitignore.md b/packages/ts/changelog.d/2026-10-06-config-gitignore.md new file mode 100644 index 000000000..1b774e752 --- /dev/null +++ b/packages/ts/changelog.d/2026-10-06-config-gitignore.md @@ -0,0 +1,3 @@ +**Changed** Declared `[[table]]` tables respect `.gitignore` by default, as +path-tables do: inside a git repo, a file a `.gitignore` ignores is no longer a +row, at startup or while watching. `--no-ignore` turns this off for them too. diff --git a/packages/ts/e2e-attestations/claude-1258-config-gitignore.json b/packages/ts/e2e-attestations/claude-1258-config-gitignore.json new file mode 100644 index 000000000..c5b000dbc --- /dev/null +++ b/packages/ts/e2e-attestations/claude-1258-config-gitignore.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1791330205, + "exit_code": 0, + "commit": "83316069645de387cebc224a9b9540fdd517ec84", + "branch": "claude/1258-config-gitignore" +} diff --git a/packages/ts/migrations.d/2026-10-06-config-gitignore.md b/packages/ts/migrations.d/2026-10-06-config-gitignore.md new file mode 100644 index 000000000..773415440 --- /dev/null +++ b/packages/ts/migrations.d/2026-10-06-config-gitignore.md @@ -0,0 +1,45 @@ +### Config tables respect .gitignore + +#### Summary + +A declared `[[table]]` used to read every file its glob matched, whatever +`.gitignore` said. It now skips gitignored files, as a path-table does. This +breaks a config that relied on indexing an ignored file; pass `--no-ignore` +(or `no_ignore` in the SDKs) to restore the old behavior. + +#### Required changes + +| Before | After | +| --- | --- | +| a table indexed gitignored files | pass `--no-ignore`, or stop ignoring the file | + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +- Inside a git repo (a directory holding `.git`), files a `.gitignore` ignores + are not rows of a `[[table]]`, at startup or while watching. +- Outside a repo nothing changes. + +#### Verification + +```bash +cd "$(mktemp -d)" +git init -q +printf 'skip.json\n' > .gitignore +echo '[{"p":"keep"}]' > keep.json +echo '[{"p":"skip"}]' > skip.json +cat > .dirsql.toml <<'TOML' +[[table]] +name = "t" +glob = "*.json" +ddl = "CREATE TABLE t (p TEXT)" +on-file = "jq -sc add" +TOML +dirsql query "SELECT p FROM t" -c .dirsql.toml +# expected: [{"p":"keep"}] +dirsql query "SELECT p FROM t ORDER BY p" -c .dirsql.toml --no-ignore +# expected: [{"p":"keep"},{"p":"skip"}] +```