diff --git a/README.md b/README.md index dd9946f..21fa655 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,12 @@ if labels.Has(roles.Fixture) { } ``` +Scanners that already enumerate files, such as brief, can call `Match` on +each accepted repository-relative path during their existing scan. This adds +no file reads or evidence allocations and does not require a second traversal. +`Match` supports concurrent calls; content analysis and directory exclusion +policies remain with the caller. + Paths use `/` separators. A trailing slash denotes a directory, so `vendor/` is a vendor directory while `vendor` alone is a filename. Empty paths, absolute paths, NUL bytes, repeated separators and `.` or `..` components @@ -150,6 +156,20 @@ err := roles.Walk(tree, roles.WalkOptions{}, func(path string, result roles.Resu }) ``` +Use `WalkMatch` when the visitor only needs labels. It reuses inherited role +sets without collecting or copying evidence, which reduces allocation for +deep trees. It has the same traversal order, limits and callback error handling +as `Walk`, and also has a classifier method for vendor-root context. + +```go +err := roles.WalkMatch(tree, roles.WalkOptions{}, func(path string, set roles.Set) error { + if set.Has(roles.CI) { + fmt.Println(path) + } + return nil +}) +``` + The default limits are one million entries and 256 path components. `WalkOptions` can change either limit; exceeding one returns `ErrLimit`, with earlier callback results already delivered. Traversal is lexical and @@ -255,8 +275,12 @@ non-UTF-8 paths rather than replacing their bytes in JSON: ```sh go run ./cmd/roles vendor/sqlite/LICENSE testdata/package-lock.json go run ./cmd/roles -root . +go run ./cmd/roles -labels-only -root . ``` +`-labels-only` uses `Match` or `WalkMatch` and emits `null` evidence. JSON +encoding still allocates output records; library callers can use `Set` directly. + ## Testing Tests cover corpus examples, rule reachability, path validation, inherited @@ -281,7 +305,9 @@ go tool pprof /tmp/roles.cpu ``` Benchmarks separate label matching, evidence allocation, bounded content -checks and filesystem traversal. `BenchmarkMillionPaths` measures a million +checks and filesystem traversal. `BenchmarkWalkDisk` and `BenchmarkWalkMatchDisk` +compare evidence-producing and label-only walks over the same wide, monorepo +and deep layouts. `BenchmarkMillionPaths` measures a million synthetic monorepo paths; repository inventories provide additional path workloads. These measurements exclude blob I/O and are not a throughput claim for a complete repository scan. diff --git a/benchmark_test.go b/benchmark_test.go index 3e4c1d7..30e33c2 100644 --- a/benchmark_test.go +++ b/benchmark_test.go @@ -72,13 +72,28 @@ func BenchmarkBlob(b *testing.B) { } func BenchmarkWalkDisk(b *testing.B) { + benchmarkWalkDisk(b, false) +} + +func BenchmarkWalkMatchDisk(b *testing.B) { + benchmarkWalkDisk(b, true) +} + +func benchmarkWalkDisk(b *testing.B, labelsOnly bool) { + b.Helper() for _, shape := range []string{"wide", "monorepo", "deep"} { b.Run(shape, func(b *testing.B) { tree := diskBenchmarkTree(b, shape) b.ResetTimer() b.ReportAllocs() for b.Loop() { - if err := roles.Walk(tree.FS(), roles.WalkOptions{}, func(string, roles.Result) error { return nil }); err != nil { + var err error + if labelsOnly { + err = roles.WalkMatch(tree.FS(), roles.WalkOptions{}, func(string, roles.Set) error { return nil }) + } else { + err = roles.Walk(tree.FS(), roles.WalkOptions{}, func(string, roles.Result) error { return nil }) + } + if err != nil { b.Fatal(err) } } diff --git a/cmd/roles/main.go b/cmd/roles/main.go index 16629a8..274e63c 100644 --- a/cmd/roles/main.go +++ b/cmd/roles/main.go @@ -22,6 +22,7 @@ func main() { func run(args []string, output io.Writer) error { flags := flag.NewFlagSet("roles", flag.ContinueOnError) root := flags.String("root", "", "Walk a repository directory without reading file contents") + labelsOnly := flags.Bool("labels-only", false, "Emit roles without collecting evidence") if err := flags.Parse(args); err != nil { return err } @@ -43,11 +44,17 @@ func run(args []string, output io.Writer) error { if err != nil { return err } - err = roles.Walk(tree.FS(), roles.WalkOptions{}, emit) + if *labelsOnly { + err = roles.WalkMatch(tree.FS(), roles.WalkOptions{}, func(name string, set roles.Set) error { + return emit(name, roles.Result{Roles: set.List()}) + }) + } else { + err = roles.Walk(tree.FS(), roles.WalkOptions{}, emit) + } return errors.Join(err, tree.Close()) } for _, name := range flags.Args() { - result, err := roles.Classify(name) + result, err := classify(name, *labelsOnly) if err != nil { return fmt.Errorf("%q: %w", name, err) } @@ -57,3 +64,11 @@ func run(args []string, output io.Writer) error { } return nil } + +func classify(name string, labelsOnly bool) (roles.Result, error) { + if labelsOnly { + set, err := roles.Match(name) + return roles.Result{Roles: set.List()}, err + } + return roles.Classify(name) +} diff --git a/cmd/roles/main_test.go b/cmd/roles/main_test.go index 88d52dc..2d71f82 100644 --- a/cmd/roles/main_test.go +++ b/cmd/roles/main_test.go @@ -64,3 +64,45 @@ func TestRunInvalidUTF8(t *testing.T) { t.Fatal("invalid path was silently replaced in JSON") } } + +func TestRunLabelsOnly(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "vendor"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "vendor", "LICENSE"), nil, 0600); err != nil { + t.Fatal(err) + } + for _, args := range [][]string{ + {"-labels-only", "vendor/LICENSE"}, + {"-labels-only", "-root", root}, + } { + var out bytes.Buffer + if err := run(args, &out); err != nil { + t.Fatal(err) + } + decoder := json.NewDecoder(&out) + found := false + for decoder.More() { + var got struct { + Path string + roles.Result + } + if err := decoder.Decode(&got); err != nil { + t.Fatal(err) + } + if len(got.Evidence) != 0 { + t.Fatal("unexpected evidence", got) + } + if got.Path == "vendor/LICENSE" { + found = true + if !got.Has(roles.Vendor) || !got.Has(roles.Legal) || len(got.Roles) != 2 { + t.Fatal(got) + } + } + } + if !found { + t.Fatal("missing file result") + } + } +} diff --git a/corpus.go b/corpus.go index 99c5ba6..1a77f66 100644 --- a/corpus.go +++ b/corpus.go @@ -27,13 +27,13 @@ type rule struct { } var ( - directoryRules = map[string][]rule{} - filenameRules = map[string][]rule{} - foldedStems = map[int][]rule{} - foldedDirectories []rule - pathDirectories = map[string][]rule{} - foldedPrefixes []rule - suffixRules [256][]rule + directoryRules = map[string][]*rule{} + filenameRules = map[string][]*rule{} + foldedStems = map[int][]*rule{} + foldedDirectories []*rule + pathDirectories = map[string][]*rule{} + foldedPrefixes []*rule + suffixRules [256][]*rule ) func init() { @@ -42,7 +42,8 @@ func init() { panic(err) } seen := map[string]bool{} - for _, r := range rules { + for i := range rules { + r := &rules[i] r.bit = roleBit(r.Role) r.directorySuffix = "/" + r.Pattern if r.ID == "" || seen[r.ID] || r.bit == 0 || r.Pattern == "" || r.Source == "" { diff --git a/determinism_test.go b/determinism_test.go index 7f8875b..6f713e0 100644 --- a/determinism_test.go +++ b/determinism_test.go @@ -34,6 +34,10 @@ func TestConcurrentDeterminism(t *testing.T) { if err != nil || !reflect.DeepEqual(got, expected) { t.Fatalf("non-deterministic result: %#v, %v", got, err) } + set, err := classifier.Match(name) + if err != nil || !reflect.DeepEqual(set.List(), expected.Roles) { + t.Fatalf("non-deterministic labels: %v, %v", set, err) + } } } }) diff --git a/legal.go b/legal.go index 96ada05..75569e0 100644 --- a/legal.go +++ b/legal.go @@ -27,16 +27,15 @@ func LegalFileName(name string) (license, notice bool) { } func prefixMatches(name, prefix string) bool { - if len(name) < len(prefix) || !strings.EqualFold(name[:len(prefix)], prefix) { + if len(name) < len(prefix) { return false } - if len(name) == len(prefix) { - return true - } - switch name[len(prefix)] { - case '.', '-', '_': - return true - default: - return false + if len(name) > len(prefix) { + switch name[len(prefix)] { + case '.', '-', '_': + default: + return false + } } + return strings.EqualFold(name[:len(prefix)], prefix) } diff --git a/repositories_test.go b/repositories_test.go index 58e2034..55cf008 100644 --- a/repositories_test.go +++ b/repositories_test.go @@ -71,3 +71,21 @@ func BenchmarkRepositories(b *testing.B) { }) } } + +func BenchmarkRepositoriesParallel(b *testing.B) { + for _, repo := range loadRepositories(b) { + b.Run(repo.Name, func(b *testing.B) { + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + _, _ = roles.Match(repo.Paths[i]) + i++ + if i == len(repo.Paths) { + i = 0 + } + } + }) + }) + } +} diff --git a/roles.go b/roles.go index 5cd1d0b..13320ac 100644 --- a/roles.go +++ b/roles.go @@ -173,7 +173,7 @@ type matchState struct { func (s matchState) result() Result { return Result{Roles: s.set.List(), Evidence: s.evidence} } -func (s *matchState) add(r rule, matchedPath string, explain bool) { +func (s *matchState) add(r *rule, matchedPath string, explain bool) { s.set |= r.bit if explain { s.evidence = append(s.evidence, Evidence{Rule: r.ID, Role: r.Role, Path: matchedPath, Subtype: r.Subtype, Ecosystem: r.Ecosystem, Origin: r.Source}) @@ -243,7 +243,7 @@ func matchFile(state *matchState, base, full string, explain bool) { } } -func suffixMatches(base string, r rule) bool { +func suffixMatches(base string, r *rule) bool { if r.Kind == "suffix-fold" { return len(base) >= len(r.Pattern) && strings.EqualFold(base[len(base)-len(r.Pattern):], r.Pattern) } diff --git a/walk.go b/walk.go index d50f023..e1a196b 100644 --- a/walk.go +++ b/walk.go @@ -28,6 +28,11 @@ func Walk(tree fs.FS, options WalkOptions, visit func(string, Result) error) err return defaults.Walk(tree, options, visit) } +// WalkMatch visits the same entries as Walk without collecting evidence. +func WalkMatch(tree fs.FS, options WalkOptions, visit func(string, Set) error) error { + return defaults.WalkMatch(tree, options, visit) +} + type ancestor struct { path string set Set @@ -36,8 +41,29 @@ type ancestor struct { // Walk reuses inherited matches and includes this classifier's vendor roots. func (c *Classifier) Walk(tree fs.FS, options WalkOptions, visit func(string, Result) error) error { - if tree == nil || visit == nil { - return errors.New("filesystem and visitor are required") + if visit == nil { + return errors.New("visitor is required") + } + return c.walk(tree, options, true, func(name string, state matchState) error { + result := state.result() + result.Evidence = append([]Evidence(nil), state.evidence...) + return visit(name, result) + }) +} + +// WalkMatch includes this classifier's vendor roots without collecting evidence. +func (c *Classifier) WalkMatch(tree fs.FS, options WalkOptions, visit func(string, Set) error) error { + if visit == nil { + return errors.New("visitor is required") + } + return c.walk(tree, options, false, func(name string, state matchState) error { + return visit(name, state.set) + }) +} + +func (c *Classifier) walk(tree fs.FS, options WalkOptions, explain bool, visit func(string, matchState) error) error { + if tree == nil { + return errors.New("filesystem is required") } if options.MaxEntries < 0 || options.MaxDepth < 0 { return errors.New("walk limits must be non-negative") @@ -82,15 +108,13 @@ func (c *Classifier) Walk(tree fs.FS, options WalkOptions, visit func(string, Re parentState := parents[len(parents)-1] state := matchState{set: parentState.set, evidence: evidence[:parentState.evidenceEnd]} if entry.IsDir() { - c.directory(&state, name[split+1:], name, true) + c.directory(&state, name[split+1:], name, explain) parents = append(parents, ancestor{path: name, set: state.set, evidenceEnd: len(state.evidence)}) name += "/" } else { - matchFile(&state, name[split+1:], name, true) + matchFile(&state, name[split+1:], name, explain) } evidence = state.evidence - result := state.result() - result.Evidence = append([]Evidence(nil), state.evidence...) - return visit(name, result) + return visit(name, state) }) } diff --git a/walk_test.go b/walk_test.go index 45240ce..5b9b827 100644 --- a/walk_test.go +++ b/walk_test.go @@ -12,6 +12,7 @@ import ( ) func TestWalk(t *testing.T) { + const symlink = "link" tree := fstest.MapFS{ "packages/api/.buildkite/pipeline.yml": {}, "packages/api/Form.DESIGNER.CS": {}, @@ -23,7 +24,7 @@ func TestWalk(t *testing.T) { "modules/a/testdata/package-lock.json": {}, "modules/b/testdata/package-lock.json": {}, "deps/crates/src/lib.rs": {}, - "link": {Mode: fs.ModeSymlink, Data: []byte("../outside")}, + symlink: {Mode: fs.ModeSymlink, Data: []byte("../outside")}, } c, err := roles.New([]roles.VendorRoot{{Path: "deps/crates", EvidencePath: cargoConfig}}) if err != nil { @@ -31,7 +32,7 @@ func TestWalk(t *testing.T) { } count := 0 err = c.Walk(tree, roles.WalkOptions{}, func(name string, got roles.Result) error { - if name == "link" { + if name == symlink { t.Fatal("visited symlink") } want, err := c.Classify(name) @@ -50,6 +51,60 @@ func TestWalk(t *testing.T) { if err != nil || count != 28 { t.Fatalf("count %d, error %v", count, err) } + count = 0 + err = c.WalkMatch(tree, roles.WalkOptions{}, func(name string, got roles.Set) error { + want, err := c.Match(name) + if err != nil || got != want || name == symlink { + t.Fatalf("WalkMatch(%q) = %v, want %v, error %v", name, got, want, err) + } + count++ + return nil + }) + if err != nil || count != 28 { + t.Fatalf("label-only count %d, error %v", count, err) + } +} + +func TestWalkMatchControl(t *testing.T) { + tree := fstest.MapFS{"vendor/lib/file.go": {}, "src/entry.go": {}} + var names []string + err := roles.WalkMatch(tree, roles.WalkOptions{}, func(name string, set roles.Set) error { + names = append(names, name) + if set.Has(roles.Vendor) { + return fs.SkipDir + } + return nil + }) + if err != nil || !reflect.DeepEqual(names, []string{"src/", "src/entry.go", "vendor/"}) { + t.Fatal(names, err) + } + count := 0 + err = roles.WalkMatch(tree, roles.WalkOptions{}, func(string, roles.Set) error { + count++ + return fs.SkipAll + }) + if err != nil || count != 1 { + t.Fatal(count, err) + } + visit := func(string, roles.Set) error { return nil } + for _, options := range []roles.WalkOptions{{MaxDepth: 1}, {MaxEntries: 1}} { + if err := roles.WalkMatch(tree, options, visit); !errors.Is(err, roles.ErrLimit) { + t.Fatal(err) + } + } + if err := roles.WalkMatch(tree, roles.WalkOptions{}, nil); err == nil { + t.Fatal("accepted nil visitor") + } + if err := roles.WalkMatch(nil, roles.WalkOptions{}, visit); err == nil { + t.Fatal("accepted nil filesystem") + } + if err := roles.WalkMatch(failingFS{}, roles.WalkOptions{}, visit); !errors.Is(err, fs.ErrPermission) { + t.Fatal(err) + } + failure := errors.New("visitor failed") + if err := roles.WalkMatch(tree, roles.WalkOptions{}, func(string, roles.Set) error { return failure }); !errors.Is(err, failure) { + t.Fatal(err) + } } func TestWalkLimitsAndErrors(t *testing.T) { @@ -124,6 +179,16 @@ func FuzzWalk(f *testing.F) { if err != nil && !errors.Is(err, roles.ErrInvalidPath) { t.Fatal(err) } + matchErr := roles.WalkMatch(tree, roles.WalkOptions{}, func(name string, got roles.Set) error { + want, err := roles.Match(name) + if err == nil && got != want { + t.Fatalf("label-only path/tree disagreement: %q", name) + } + return err + }) + if !errors.Is(matchErr, err) { + t.Fatalf("walk errors differ: %v, %v", err, matchErr) + } }) }