From 67b43b24221a31758806eb48b9dea28e48ae3e04 Mon Sep 17 00:00:00 2001 From: Nick Sheth Date: Thu, 24 Sep 2026 14:59:05 -0700 Subject: [PATCH] fix(scanner): index dot-directories instead of skipping them by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ast-grep skips hidden files and directories by default (its own "hidden" ignore rule), and codemap never overrode that default, so a project whose source lives only under a dot-directory (e.g. `.agents/`) was invisible to --deps/--importers with no config option to opt in. scanDirectory now passes `--no-ignore hidden` so ast-grep walks into dot-directories in general, and re-excludes `.git` plus every directory name in the scanner's own IgnoredDirs list (node_modules, vendor, testdata, build output, language caches, ...) via --globs, so relaxing the hidden-file default doesn't also start indexing those or slow down large repos by default. `.codemap` (codemap's own state directory) is excluded the same way. ast-grep's --globs match the literal path it walks with no awareness of where the scan root begins: given an absolute root, a bare `**/testdata/**`-style exclude matches even when "testdata" is an *ancestor* of root rather than something inside it — which is exactly how this package's own scanner tests are laid out (fixtures under ../testdata/), and running the fix against them found this the hard way (first draft returned zero files for every ../testdata/... fixture in the suite). Fixed by running the ast-grep subprocess with its cwd set to root and "." as the scanned path (cmd.Dir = root), so every walked path — and therefore every glob match — is root-relative and can never see anything above root. Added: - TestScanDirectoryInvokesAstGrepWithHiddenDirsEnabled: locks in the exact args (--no-ignore hidden, the --globs excludes, "." as the scan path) and that ast-grep's cwd is root. - TestScanDirectoryIndexesHiddenDirectories: end-to-end, a file under .agents/ is now indexed and its importer resolves correctly. - TestScanDirectoryDoesNotExcludeRootsNestedUnderAnIgnoredDirName: the regression for the ancestor-name false-exclusion bug above. - TestScanDirectoryStillExcludesKnownNeverSourceDirs: .git, node_modules, and vendor stay excluded even with hidden-dir scanning on. `go test ./...` passes, including this repo's own ../testdata/-based fixtures, which the ancestor-exclusion bug (caught before this commit) would otherwise have broken. --- scanner/astgrep.go | 42 ++++++++- scanner/astgrep_test.go | 200 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 241 insertions(+), 1 deletion(-) diff --git a/scanner/astgrep.go b/scanner/astgrep.go index bf2b057..75ba69e 100644 --- a/scanner/astgrep.go +++ b/scanner/astgrep.go @@ -12,6 +12,7 @@ import ( "path/filepath" "regexp" "runtime" + "sort" "strings" "sync" "syscall" @@ -271,6 +272,24 @@ func (s *AstGrepScanner) Available() bool { return s.binary != "" } +// hiddenScanExcludes lists directory names ast-grep must keep skipping once +// --no-ignore hidden lets it walk into dot-directories in general: `.git` +// (whose internals are never source) plus every name codemap's own walker +// already treats as never-source (IgnoredDirs — vendor trees, build output, +// language caches), so relaxing the hidden-file default doesn't also start +// indexing those. `.codemap` is codemap's own state directory (skills, +// config, cache) and is excluded the same way the plain file walker strips +// it from its inventory (see ScanConfiguredFilesWithFilters). +func hiddenScanExcludes() []string { + names := make([]string, 0, len(IgnoredDirs)+1) + for name := range IgnoredDirs { + names = append(names, name) + } + names = append(names, ".codemap") + sort.Strings(names) + return names +} + // findNestedGitRepos returns subdirectory names that contain their own .git // These are separate repositories (not submodules) that should be excluded // from scanning to avoid hanging on large nested repos. @@ -362,15 +381,36 @@ func (s *AstGrepScanner) scanDirectory(parent context.Context, root string) ([]F } else { args = append(args, "--inline-rules", inlineRules) } + // ast-grep skips dot-directories by default (its "hidden" ignore rule), + // so a project whose source lives under e.g. `.agents/` is invisible to + // it with no way to opt in. --no-ignore hidden turns that default off, + // but it also stops ast-grep from treating `.git` as special and opens + // up every other genuinely-internal dot-directory codemap already + // excludes by name (IgnoredDirs) — so both are re-excluded explicitly + // via --globs, at any depth, right alongside it. + // + // ast-grep's --globs match the literal path string it walks, with no + // awareness of where "root" begins: given an absolute root, a pattern + // like "**/testdata/**" matches even when "testdata" is an *ancestor* of + // root rather than something inside it (this repo's own scanner tests, + // which build fixtures under ../testdata/, hit exactly that false + // exclusion). Running the scan with its cwd set to root and "." as the + // scanned path keeps every walked path root-relative, so the globs can + // never see anything above root. + args = append(args, "--no-ignore", "hidden") + for _, name := range hiddenScanExcludes() { + args = append(args, "--globs", "!**/"+name+"/**") + } for _, repo := range findNestedGitRepos(root) { args = append(args, "--globs", "!"+repo+"/**") } - args = append(args, root) + args = append(args, ".") ctx, cancel := context.WithTimeout(parent, astGrepScanTimeout) defer cancel() cmd := astGrepCommand(ctx, s.binary, args...) + cmd.Dir = root cmd.WaitDelay = 100 * time.Millisecond out, err := cmd.Output() if err != nil { diff --git a/scanner/astgrep_test.go b/scanner/astgrep_test.go index 6e9dbfc..ed40f3c 100644 --- a/scanner/astgrep_test.go +++ b/scanner/astgrep_test.go @@ -573,3 +573,203 @@ class App extends StatelessWidget { } } } + +// TestScanDirectoryInvokesAstGrepWithHiddenDirsEnabled locks in the exact +// invocation shape the hidden-directory fix depends on: --no-ignore hidden +// (so ast-grep stops skipping dot-directories by default), --globs excludes +// for the directories codemap has always treated as never-source (so +// relaxing that default doesn't newly index .git, vendor, node_modules, +// etc.), a "." scan path, and cmd.Dir set to root. The last two matter +// together: ast-grep's --globs match the literal walked path with no +// awareness of where root begins, so an absolute root under a directory +// that happens to share a name with one of those excludes (this repo's own +// scanner tests build fixtures under ../testdata/) would otherwise be +// wrongly excluded in its entirety. See TestScanDirectoryDoesNotExcludeRootsNestedUnderAnIgnoredDirName. +func TestScanDirectoryInvokesAstGrepWithHiddenDirsEnabled(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("requires shell script execution") + } + + tmpDir := t.TempDir() + capturedArgs := filepath.Join(tmpDir, "args.txt") + capturedPwd := filepath.Join(tmpDir, "pwd.txt") + fakeBinary := filepath.Join(tmpDir, "fake-sg.sh") + script := "#!/bin/sh\nprintf '%s\\n' \"$@\" > \"$CAPTURE_ARGS\"\npwd > \"$CAPTURE_PWD\"\nprintf '[]\\n'\n" + if err := os.WriteFile(fakeBinary, []byte(script), 0755); err != nil { + t.Fatal(err) + } + t.Setenv("CAPTURE_ARGS", capturedArgs) + t.Setenv("CAPTURE_PWD", capturedPwd) + + scanner := &AstGrepScanner{rulesDir: tmpDir, binary: fakeBinary} + if _, err := scanner.ScanDirectory(context.Background(), tmpDir); err != nil { + t.Fatalf("ScanDirectory() error = %v", err) + } + + args, err := os.ReadFile(capturedArgs) + if err != nil { + t.Fatalf("read captured args: %v", err) + } + argLines := strings.Split(strings.TrimRight(string(args), "\n"), "\n") + + if !strings.Contains(string(args), "--no-ignore\nhidden\n") { + t.Fatalf("expected --no-ignore hidden in args, got: %s", args) + } + for _, want := range []string{"!**/.git/**", "!**/node_modules/**", "!**/vendor/**", "!**/testdata/**"} { + if !strings.Contains(string(args), want) { + t.Errorf("expected a --globs exclude for %q, got: %s", want, args) + } + } + if got := argLines[len(argLines)-1]; got != "." { + t.Fatalf("scan path argument = %q, want %q (root-relative, via cmd.Dir)", got, ".") + } + + pwdBytes, err := os.ReadFile(capturedPwd) + if err != nil { + t.Fatalf("read captured pwd: %v", err) + } + gotPwd := canonicalTestPath(strings.TrimSpace(string(pwdBytes))) + wantPwd := canonicalTestPath(tmpDir) + if gotPwd != wantPwd { + t.Fatalf("ast-grep cwd = %q, want %q (root)", gotPwd, wantPwd) + } +} + +// TestScanDirectoryIndexesHiddenDirectories is the end-to-end regression for +// the reported bug: a project whose source lives only under a dot-directory +// (not .git) was invisible to --deps/--importers with no way to opt in, +// because ast-grep skips hidden files and directories by default and +// codemap never overrode that default. +func TestScanDirectoryIndexesHiddenDirectories(t *testing.T) { + scanner, err := NewAstGrepScanner() + if err != nil { + t.Fatalf("NewAstGrepScanner() error = %v", err) + } + t.Cleanup(scanner.Close) + if !scanner.Available() { + t.Skip("ast-grep not available") + } + + root := t.TempDir() + hidden := filepath.Join(root, ".agents") + if err := os.MkdirAll(hidden, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(hidden, "a.js"), []byte("const b = require('./b');\nmodule.exports = { b };\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(hidden, "b.js"), []byte("module.exports = { hello: () => 'hi' };\n"), 0o644); err != nil { + t.Fatal(err) + } + + outcome, err := scanner.ScanDirectory(context.Background(), root) + if err != nil { + t.Fatalf("ScanDirectory() error = %v", err) + } + paths := make(map[string]bool) + for _, a := range outcome.Analyses { + paths[filepath.ToSlash(a.Path)] = true + } + if !paths[".agents/a.js"] { + t.Fatalf("analyses = %#v, want .agents/a.js indexed", outcome.Analyses) + } + + fg, err := BuildFileGraphFromOutcome(context.Background(), root, outcome, Filters{}) + if err != nil { + t.Fatalf("BuildFileGraphFromOutcome() error = %v", err) + } + importers := fg.Importers[".agents/b.js"] + if len(importers) != 1 || importers[0] != ".agents/a.js" { + t.Fatalf("importers of .agents/b.js = %v, want [.agents/a.js]", importers) + } +} + +// TestScanDirectoryDoesNotExcludeRootsNestedUnderAnIgnoredDirName is the +// regression for a bug the hidden-directory fix's first draft introduced: +// excluding codemap's never-source directory names (node_modules, vendor, +// testdata, ...) via bare --globs patterns matches those names anywhere in +// the walked path, including ancestors of root that happen to share the +// name. A project checked out under a path like ".../testdata/myproject" +// (exactly how this package's own fixtures live, under ../testdata/) must +// still be scanned in full. +func TestScanDirectoryDoesNotExcludeRootsNestedUnderAnIgnoredDirName(t *testing.T) { + scanner, err := NewAstGrepScanner() + if err != nil { + t.Fatalf("NewAstGrepScanner() error = %v", err) + } + t.Cleanup(scanner.Close) + if !scanner.Available() { + t.Skip("ast-grep not available") + } + + base := t.TempDir() + // "testdata" and "vendor" are both in IgnoredDirs; root sits under one + // of them as an ancestor, not as a subdirectory of root. + root := filepath.Join(base, "testdata", "myproject") + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "a.js"), []byte("const b = require('./b');\n"), 0o644); err != nil { + t.Fatal(err) + } + + outcome, err := scanner.ScanDirectory(context.Background(), root) + if err != nil { + t.Fatalf("ScanDirectory() error = %v", err) + } + if len(outcome.Analyses) == 0 { + t.Fatalf("analyses is empty; root nested under an ignored-dir-named ancestor was wrongly excluded in full") + } + paths := make(map[string]bool) + for _, a := range outcome.Analyses { + paths[filepath.ToSlash(a.Path)] = true + } + if !paths["a.js"] { + t.Fatalf("analyses = %#v, want a.js indexed", outcome.Analyses) + } +} + +// TestScanDirectoryStillExcludesKnownNeverSourceDirs confirms that turning +// on hidden-directory scanning did not also start indexing .git internals +// or vendored/build directories that codemap's walker has always excluded. +func TestScanDirectoryStillExcludesKnownNeverSourceDirs(t *testing.T) { + scanner, err := NewAstGrepScanner() + if err != nil { + t.Fatalf("NewAstGrepScanner() error = %v", err) + } + t.Cleanup(scanner.Close) + if !scanner.Available() { + t.Skip("ast-grep not available") + } + + root := t.TempDir() + for _, dir := range []string{".git/hooks", "node_modules/pkg", "vendor/pkg"} { + full := filepath.Join(root, filepath.FromSlash(dir)) + if err := os.MkdirAll(full, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(full, "dep.js"), []byte("const x = require('./x');\n"), 0o644); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(root, "real.js"), []byte("const x = require('./x');\n"), 0o644); err != nil { + t.Fatal(err) + } + + outcome, err := scanner.ScanDirectory(context.Background(), root) + if err != nil { + t.Fatalf("ScanDirectory() error = %v", err) + } + paths := make(map[string]bool) + for _, a := range outcome.Analyses { + paths[filepath.ToSlash(a.Path)] = true + } + if !paths["real.js"] { + t.Fatalf("analyses = %#v, want real.js indexed", outcome.Analyses) + } + for path := range paths { + if strings.Contains(path, ".git/") || strings.Contains(path, "node_modules/") || strings.Contains(path, "vendor/") { + t.Fatalf("analyses = %#v, still indexed a never-source directory (%s)", outcome.Analyses, path) + } + } +}