diff --git a/packages/python/e2e-attestations/claude-1351-dot-dirs.json b/packages/python/e2e-attestations/claude-1351-dot-dirs.json new file mode 100644 index 000000000..ab3926656 --- /dev/null +++ b/packages/python/e2e-attestations/claude-1351-dot-dirs.json @@ -0,0 +1,7 @@ +{ + "command": "just test-e2e", + "ran_at": 1791299799, + "exit_code": 0, + "commit": "bc8156e317596d423f903feb4014d3ddda21c960", + "branch": "claude/1351-dot-dirs" +} diff --git a/packages/python/tests/e2e/glob_speed_dotfiles_test.py b/packages/python/tests/e2e/glob_speed_dotfiles_test.py new file mode 100644 index 000000000..15b6d69c1 --- /dev/null +++ b/packages/python/tests/e2e/glob_speed_dotfiles_test.py @@ -0,0 +1,129 @@ +"""Glob speed: dotfiles stay hidden unless the pattern spells the dot. + +A deep, wide tree where half the top-level directories are dot-named and +dot-named directories and dotfiles recur below. `'./**/*.md'` must skip +every one of them; `'./**/.*'` reaches dotfiles because the pattern spells +the dot, and `'./.cache/**/*.md'` reaches under a dot-named directory +because the pattern names it. Native is `find` pruning what bash globstar +with dotglob off would not enter. No mocks: real console script, real +process, real filesystem. +""" + +from __future__ import annotations + +import shutil + +import pytest + +from .speed_of_light import ( + assert_speed_of_light, + best_of, + cli, + dirsql_rows, + dirsql_timeout, + grow_until_native_takes_a_second, + hopeless_seconds, + startup_seconds, + timed, +) + +MARKDOWN = ( + "find . -mindepth 1 -name '.*' -prune -o -type f -name '*.md' -printf '%P\\n'" +) +DOTFILES = "find . -mindepth 1 -name '.*' -prune -type f -printf '%P\\n'" +UNDER_CACHE = "find .cache -mindepth 1 -name '.*' -prune -o -type f -name '*.md' -print" +FANOUT = 8 + + +def home(i): + """The top of item `i`'s subtree: half of them dot-named.""" + return ("a", "b", ".cache", ".tmp")[i % 4] + + +def build_tree(root, lo, hi): + """Items `lo` through `hi - 1`, each a file three directories below its + home. Every seventh sits in a dot-named directory, every fifth is a + dotfile, and every ninth a `.txt`.""" + for i in range(lo, hi): + j = i // 4 + folder = ( + root + / home(i) + / f"x{j % FANOUT}" + / f"y{j // FANOUT % FANOUT}" + / f"z{j // FANOUT**2 % FANOUT}" + ) + if i % 7 == 0: + folder = folder / ".d" + folder.mkdir(parents=True, exist_ok=True) + if i % 5 == 0: + name = f".h{i}.md" + elif i % 9 == 0: + name = f"n{i}.txt" + else: + name = f"n{i}.md" + (folder / name).touch() + + +def matched(n, glob): + def hit(i): + visible = i % 7 != 0 + markdown = i % 5 != 0 and i % 9 != 0 + if glob == "./**/*.md": + return not home(i).startswith(".") and visible and markdown + if glob == "./**/.*": + return not home(i).startswith(".") and visible and i % 5 == 0 + return home(i) == ".cache" and visible and markdown + + return sum(1 for i in range(n) if hit(i)) + + +def native(root, script): + proc, seconds = timed(["bash", "-c", script], root, timeout=600) + assert proc.returncode == 0, proc.stderr + return sorted(proc.stdout.splitlines()), seconds + + +def describe_dotfile_glob_speed_of_light(): + @pytest.fixture + def root(tmp_path): + tree = tmp_path / "dotfiles" + tree.mkdir() + try: + yield tree + finally: + shutil.rmtree(tree, ignore_errors=True) + + @pytest.mark.parametrize( + ("glob", "script"), + [ + ("./**/*.md", MARKDOWN), + ("./**/.*", DOTFILES), + ("./.cache/**/*.md", UNDER_CACHE), + ], + ) + def it_matches_native_files_within_the_bar(root, glob, script): + startup = startup_seconds() + n = grow_until_native_takes_a_second( + lambda lo, hi: build_tree(root, lo, hi), + lambda: native(root, script), + start=100_000, + ceiling=2**22, + ) + expected, native_seconds = best_of(lambda: native(root, script), "native") + assert len(expected) == matched(n, glob) + + def dirsql(): + proc, seconds = timed( + [cli(), "query", f"SELECT path FROM '{glob}'"], + root, + timeout=dirsql_timeout(native_seconds), + ) + return sorted(row for (row,) in dirsql_rows(proc, ("path",))), seconds + + actual, dirsql_seconds = best_of( + dirsql, "dirsql", hopeless_seconds(native_seconds, startup) + ) + + assert actual == expected + assert_speed_of_light(glob, native_seconds, dirsql_seconds, startup) diff --git a/packages/rust/changelog.d/2026-10-06-dot-directories-stay-hidden.md b/packages/rust/changelog.d/2026-10-06-dot-directories-stay-hidden.md new file mode 100644 index 000000000..dbfcd6e45 --- /dev/null +++ b/packages/rust/changelog.d/2026-10-06-dot-directories-stay-hidden.md @@ -0,0 +1,4 @@ +**Fixed** A path-table glob enters a dot-named directory only where a +component spells it at that depth, as bash does. `'./**/.*'` used to list +dotfiles inside dot-named directories such as `.cache/`; it now lists only +dot-named files whose parent directories are not themselves dot-named. diff --git a/packages/rust/migrations.d/2026-10-06-dot-directories-stay-hidden.md b/packages/rust/migrations.d/2026-10-06-dot-directories-stay-hidden.md new file mode 100644 index 000000000..391148bec --- /dev/null +++ b/packages/rust/migrations.d/2026-10-06-dot-directories-stay-hidden.md @@ -0,0 +1,32 @@ +### Core: a spelled dot component no longer opens dot directories above it + +#### Summary + +A path-table glob whose dot component sits below `**`, such as `'./**/.*'`, +used to descend into every dot-named directory and return the dotfiles +inside. It now enters a dot-named directory only where a component spells it +at that depth, matching bash and the path-tables reference. Queries on such +globs can return fewer rows. + +#### Required changes + +_None._ + +#### Deprecations removed + +_None._ + +#### Behavior changes without code changes + +- `'./**/.*'` no longer lists `.cache/.z`; it still lists `.x` and `a/.y`. +- To read inside a dot-named directory, spell it: `'./.cache/**/.*'`. + +#### Verification + +```bash +cd "$(mktemp -d)" +mkdir -p a .cache +touch .x a/.y .cache/.z +dirsql query "SELECT path FROM './**/.*' ORDER BY path" +# expected: [{"path":".x"},{"path":"a/.y"}] +``` diff --git a/packages/rust/src/scanner.rs b/packages/rust/src/scanner.rs index 6c0bd8a07..30edcfcc8 100644 --- a/packages/rust/src/scanner.rs +++ b/packages/rust/src/scanner.rs @@ -168,8 +168,10 @@ fn is_wanted(glob: &PathGlob, ignore: &TableMatcher, rel_path: &Path) -> bool { /// where the walk may follow a symlink. /// /// A component beginning with `.` is spelled; `.claude` and `.env` admit -/// exactly those names, `.*` any dot-named entry, as in the shell. Every -/// other dot-named file or directory is skipped, the rule `ls` and `fd` use. +/// exactly those names, `.*` any dot-named entry, as in the shell, and only at +/// the depth the component sits: `**` never consumes a dot-named directory. +/// Every other dot-named file or directory is skipped, the rule `ls` and `fd` +/// use. /// A dot-named symlink is a dot-named entry like any other. /// /// The symlink rule is bash's (`globstar`, 4.3 and later). A symlinked file @@ -191,6 +193,9 @@ pub struct PathGlob { enum Component { AnyDepth, Name(Pattern), + /// A component spelled with a leading `.`, the only kind that consumes a + /// dot-named entry. + DotName(Pattern), End, } @@ -224,11 +229,16 @@ impl PathGlob { /// `states`; an `End` means a whole word has matched. fn step(&self, states: &[usize], name: &OsStr, kind: Kind) -> Vec { let mut next = Vec::new(); + let dot_named = is_dot_named(name); for &i in states { match self.components.get(i) { + Some(Component::AnyDepth) if dot_named => {} Some(Component::AnyDepth) if kind == Kind::LinkedDir => next.push(i + 1), Some(Component::AnyDepth) => next.push(i), - Some(Component::Name(m)) if m.is_match(Path::new(name)) => next.push(i + 1), + Some(Component::Name(m)) if !dot_named && m.is_match(Path::new(name)) => { + next.push(i + 1) + } + Some(Component::DotName(m)) if m.is_match(Path::new(name)) => next.push(i + 1), _ => {} } } @@ -306,6 +316,7 @@ fn compile_components(word: &str) -> Option> { .split('/') .map(|c| match c { "**" => Ok(Component::AnyDepth), + _ if is_dot_named(OsStr::new(c)) => Pattern::new(c).map(Component::DotName), _ => Pattern::new(c).map(Component::Name), }) .collect::, _>>() @@ -1294,6 +1305,28 @@ mod tests { assert_eq!(states_after(&glob, &[("a", Kind::File)]), vec![0, 1]); } + #[test] + fn a_double_star_never_consumes_a_dot_named_entry() { + let glob = compile_glob("**/.*").unwrap(); + assert_eq!(states_after(&glob, &[(".cache", Kind::Dir)]), vec![2]); + assert!(states_after(&glob, &[(".cache", Kind::Dir), (".z", Kind::File)]).is_empty()); + } + + #[test] + fn only_a_component_spelled_with_a_dot_consumes_a_dot_named_entry() { + let glob = compile_glob("*/x").unwrap(); + assert!(states_after(&glob, &[(".d", Kind::Dir)]).is_empty()); + let glob = compile_glob(".*/x").unwrap(); + assert_eq!(states_after(&glob, &[(".d", Kind::Dir)]), vec![1]); + } + + #[test] + fn a_spelled_dot_component_consumes_only_the_names_it_matches() { + let glob = compile_glob(".claude/x").unwrap(); + assert_eq!(states_after(&glob, &[(".claude", Kind::Dir)]), vec![1]); + assert!(states_after(&glob, &[(".env", Kind::Dir)]).is_empty()); + } + #[test] fn a_double_star_stops_on_a_symlinked_directory() { let glob = compile_glob("**/x").unwrap(); diff --git a/packages/rust/tests/scanner.rs b/packages/rust/tests/scanner.rs index 891e2e04f..2e45c9c8b 100644 --- a/packages/rust/tests/scanner.rs +++ b/packages/rust/tests/scanner.rs @@ -218,3 +218,50 @@ fn scan_glob_applies_gitignore_only_inside_a_repo() { assert_eq!(results, vec![std::path::PathBuf::from("zz-sibling/z.log")]); } + +fn dot_tree() -> TempDir { + let root = TempDir::new().unwrap(); + for dir in ["a/.cache", ".d/.cache", ".cache"] { + fs::create_dir_all(root.path().join(dir)).unwrap(); + } + for file in [ + ".x", + "a/.y", + ".cache/.z", + "a/.cache/m.md", + ".d/.cache/m.md", + ".cache/m.md", + ] { + fs::write(root.path().join(file), "").unwrap(); + } + root +} + +fn scan_paths(root: &TempDir, pattern: &str) -> Vec { + let ignore = TableMatcher::new(&[], &[]).unwrap(); + let glob = compile_glob(pattern).unwrap(); + scan_glob(root.path(), &glob, &ignore, false) + .into_iter() + .map(|p| { + p.components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/") + }) + .collect() +} + +#[test] +fn scan_glob_a_spelled_dot_component_does_not_open_dot_directories_above_it() { + let root = dot_tree(); + assert_eq!(scan_paths(&root, "**/.*"), vec![".x", "a/.y"]); +} + +#[test] +fn scan_glob_a_double_star_does_not_cross_a_dot_directory_the_next_component_spells() { + let root = dot_tree(); + assert_eq!( + scan_paths(&root, "**/.cache/*.md"), + vec![".cache/m.md", "a/.cache/m.md"] + ); +} diff --git a/packages/ts/e2e-attestations/claude-1309-glob-speed.json b/packages/ts/e2e-attestations/claude-1309-glob-speed.json new file mode 100644 index 000000000..01dbd4e18 --- /dev/null +++ b/packages/ts/e2e-attestations/claude-1309-glob-speed.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1791269930, + "exit_code": 0, + "commit": "3d387a5513bd2e0bf3ab85e12d31eb6c95b130e9", + "branch": "claude/1309-glob-speed" +} diff --git a/packages/ts/e2e-attestations/claude-1351-dot-dirs.json b/packages/ts/e2e-attestations/claude-1351-dot-dirs.json new file mode 100644 index 000000000..6ee747c47 --- /dev/null +++ b/packages/ts/e2e-attestations/claude-1351-dot-dirs.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1791279141, + "exit_code": 0, + "commit": "c8bff51731fa6e0813efed3c4fe1f726f9457fc3", + "branch": "claude/1351-dot-dirs" +}