Speed up path matching and add label-only traversal

README.md

@@ -113,6 +113,12 @@ if labels.Has(roles.Fixture) {
 }
 ```
 
+Scanners that already enumerate files, such as brief, can call `Match` on
+each accepted repository-relative path during their existing scan. This adds
+no file reads or evidence allocations and does not require a second traversal.
+`Match` supports concurrent calls; content analysis and directory exclusion
+policies remain with the caller.
+
 Paths use `/` separators. A trailing slash denotes a directory, so `vendor/`
 is a vendor directory while `vendor` alone is a filename. Empty paths,
 absolute paths, NUL bytes, repeated separators and `.` or `..` components
@@ -150,6 +156,20 @@ err := roles.Walk(tree, roles.WalkOptions{}, func(path string, result roles.Resu
 })
 ```
 
+Use `WalkMatch` when the visitor only needs labels. It reuses inherited role
+sets without collecting or copying evidence, which reduces allocation for
+deep trees. It has the same traversal order, limits and callback error handling
+as `Walk`, and also has a classifier method for vendor-root context.
+
+```go
+err := roles.WalkMatch(tree, roles.WalkOptions{}, func(path string, set roles.Set) error {
+    if set.Has(roles.CI) {
+        fmt.Println(path)
+    }
+    return nil
+})
+```
+
 The default limits are one million entries and 256 path components.
 `WalkOptions` can change either limit; exceeding one returns `ErrLimit`,
 with earlier callback results already delivered. Traversal is lexical and
@@ -255,8 +275,12 @@ non-UTF-8 paths rather than replacing their bytes in JSON:
 ```sh
 go run ./cmd/roles vendor/sqlite/LICENSE testdata/package-lock.json
 go run ./cmd/roles -root .
+go run ./cmd/roles -labels-only -root .
 ```
 
+`-labels-only` uses `Match` or `WalkMatch` and emits `null` evidence. JSON
+encoding still allocates output records; library callers can use `Set` directly.
+
 ## Testing
 
 Tests cover corpus examples, rule reachability, path validation, inherited
@@ -281,7 +305,9 @@ go tool pprof /tmp/roles.cpu
 ```
 
 Benchmarks separate label matching, evidence allocation, bounded content
-checks and filesystem traversal. `BenchmarkMillionPaths` measures a million
+checks and filesystem traversal. `BenchmarkWalkDisk` and `BenchmarkWalkMatchDisk`
+compare evidence-producing and label-only walks over the same wide, monorepo
+and deep layouts. `BenchmarkMillionPaths` measures a million
 synthetic monorepo paths; repository inventories provide additional path
 workloads. These measurements exclude blob I/O and are not a throughput
 claim for a complete repository scan.

benchmark_test.go

@@ -72,13 +72,28 @@ func BenchmarkBlob(b *testing.B) {
 }
 
 func BenchmarkWalkDisk(b *testing.B) {
+	benchmarkWalkDisk(b, false)
+}
+
+func BenchmarkWalkMatchDisk(b *testing.B) {
+	benchmarkWalkDisk(b, true)
+}
+
+func benchmarkWalkDisk(b *testing.B, labelsOnly bool) {
+	b.Helper()
 	for _, shape := range []string{"wide", "monorepo", "deep"} {
 		b.Run(shape, func(b *testing.B) {
 			tree := diskBenchmarkTree(b, shape)
 			b.ResetTimer()
 			b.ReportAllocs()
 			for b.Loop() {
-				if err := roles.Walk(tree.FS(), roles.WalkOptions{}, func(string, roles.Result) error { return nil }); err != nil {
+				var err error
+				if labelsOnly {
+					err = roles.WalkMatch(tree.FS(), roles.WalkOptions{}, func(string, roles.Set) error { return nil })
+				} else {
+					err = roles.Walk(tree.FS(), roles.WalkOptions{}, func(string, roles.Result) error { return nil })
+				}
+				if err != nil {
 					b.Fatal(err)
 				}
 			}

cmd/roles/main.go

@@ -22,6 +22,7 @@ func main() {
 func run(args []string, output io.Writer) error {
 	flags := flag.NewFlagSet("roles", flag.ContinueOnError)
 	root := flags.String("root", "", "Walk a repository directory without reading file contents")
+	labelsOnly := flags.Bool("labels-only", false, "Emit roles without collecting evidence")
 	if err := flags.Parse(args); err != nil {
 		return err
 	}
@@ -43,11 +44,17 @@ func run(args []string, output io.Writer) error {
 		if err != nil {
 			return err
 		}
-		err = roles.Walk(tree.FS(), roles.WalkOptions{}, emit)
+		if *labelsOnly {
+			err = roles.WalkMatch(tree.FS(), roles.WalkOptions{}, func(name string, set roles.Set) error {
+				return emit(name, roles.Result{Roles: set.List()})
+			})
+		} else {
+			err = roles.Walk(tree.FS(), roles.WalkOptions{}, emit)
+		}
 		return errors.Join(err, tree.Close())
 	}
 	for _, name := range flags.Args() {
-		result, err := roles.Classify(name)
+		result, err := classify(name, *labelsOnly)
 		if err != nil {
 			return fmt.Errorf("%q: %w", name, err)
 		}
@@ -57,3 +64,11 @@ func run(args []string, output io.Writer) error {
 	}
 	return nil
 }
+
+func classify(name string, labelsOnly bool) (roles.Result, error) {
+	if labelsOnly {
+		set, err := roles.Match(name)
+		return roles.Result{Roles: set.List()}, err
+	}
+	return roles.Classify(name)
+}

cmd/roles/main_test.go

@@ -64,3 +64,45 @@ func TestRunInvalidUTF8(t *testing.T) {
 		t.Fatal("invalid path was silently replaced in JSON")
 	}
 }
+
+func TestRunLabelsOnly(t *testing.T) {
+	root := t.TempDir()
+	if err := os.MkdirAll(filepath.Join(root, "vendor"), 0700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(root, "vendor", "LICENSE"), nil, 0600); err != nil {
+		t.Fatal(err)
+	}
+	for _, args := range [][]string{
+		{"-labels-only", "vendor/LICENSE"},
+		{"-labels-only", "-root", root},
+	} {
+		var out bytes.Buffer
+		if err := run(args, &out); err != nil {
+			t.Fatal(err)
+		}
+		decoder := json.NewDecoder(&out)
+		found := false
+		for decoder.More() {
+			var got struct {
+				Path string
+				roles.Result
+			}
+			if err := decoder.Decode(&got); err != nil {
+				t.Fatal(err)
+			}
+			if len(got.Evidence) != 0 {
+				t.Fatal("unexpected evidence", got)
+			}
+			if got.Path == "vendor/LICENSE" {
+				found = true
+				if !got.Has(roles.Vendor) || !got.Has(roles.Legal) || len(got.Roles) != 2 {
+					t.Fatal(got)
+				}
+			}
+		}
+		if !found {
+			t.Fatal("missing file result")
+		}
+	}
+}

corpus.go

@@ -27,13 +27,13 @@ type rule struct {
 }
 
 var (
-	directoryRules    = map[string][]rule{}
-	filenameRules     = map[string][]rule{}
-	foldedStems       = map[int][]rule{}
-	foldedDirectories []rule
-	pathDirectories   = map[string][]rule{}
-	foldedPrefixes    []rule
-	suffixRules       [256][]rule
+	directoryRules    = map[string][]*rule{}
+	filenameRules     = map[string][]*rule{}
+	foldedStems       = map[int][]*rule{}
+	foldedDirectories []*rule
+	pathDirectories   = map[string][]*rule{}
+	foldedPrefixes    []*rule
+	suffixRules       [256][]*rule
 )
 
 func init() {
@@ -42,7 +42,8 @@ func init() {
 		panic(err)
 	}
 	seen := map[string]bool{}
-	for _, r := range rules {
+	for i := range rules {
+		r := &rules[i]
 		r.bit = roleBit(r.Role)
 		r.directorySuffix = "/" + r.Pattern
 		if r.ID == "" || seen[r.ID] || r.bit == 0 || r.Pattern == "" || r.Source == "" {

determinism_test.go

@@ -34,6 +34,10 @@ func TestConcurrentDeterminism(t *testing.T) {
 					if err != nil || !reflect.DeepEqual(got, expected) {
 						t.Fatalf("non-deterministic result: %#v, %v", got, err)
 					}
+					set, err := classifier.Match(name)
+					if err != nil || !reflect.DeepEqual(set.List(), expected.Roles) {
+						t.Fatalf("non-deterministic labels: %v, %v", set, err)
+					}
 				}
 			}
 		})

legal.go

@@ -27,16 +27,15 @@ func LegalFileName(name string) (license, notice bool) {
 }
 
 func prefixMatches(name, prefix string) bool {
-	if len(name) < len(prefix) || !strings.EqualFold(name[:len(prefix)], prefix) {
+	if len(name) < len(prefix) {
 		return false
 	}
-	if len(name) == len(prefix) {
-		return true
-	}
-	switch name[len(prefix)] {
-	case '.', '-', '_':
-		return true
-	default:
-		return false
+	if len(name) > len(prefix) {
+		switch name[len(prefix)] {
+		case '.', '-', '_':
+		default:
+			return false
+		}
 	}
+	return strings.EqualFold(name[:len(prefix)], prefix)
 }

repositories_test.go

@@ -71,3 +71,21 @@ func BenchmarkRepositories(b *testing.B) {
 		})
 	}
 }
+
+func BenchmarkRepositoriesParallel(b *testing.B) {
+	for _, repo := range loadRepositories(b) {
+		b.Run(repo.Name, func(b *testing.B) {
+			b.ReportAllocs()
+			b.RunParallel(func(pb *testing.PB) {
+				i := 0
+				for pb.Next() {
+					_, _ = roles.Match(repo.Paths[i])
+					i++
+					if i == len(repo.Paths) {
+						i = 0
+					}
+				}
+			})
+		})
+	}
+}

roles.go

@@ -173,7 +173,7 @@ type matchState struct {
 
 func (s matchState) result() Result { return Result{Roles: s.set.List(), Evidence: s.evidence} }
 
-func (s *matchState) add(r rule, matchedPath string, explain bool) {
+func (s *matchState) add(r *rule, matchedPath string, explain bool) {
 	s.set |= r.bit
 	if explain {
 		s.evidence = append(s.evidence, Evidence{Rule: r.ID, Role: r.Role, Path: matchedPath, Subtype: r.Subtype, Ecosystem: r.Ecosystem, Origin: r.Source})
@@ -243,7 +243,7 @@ func matchFile(state *matchState, base, full string, explain bool) {
 	}
 }
 
-func suffixMatches(base string, r rule) bool {
+func suffixMatches(base string, r *rule) bool {
 	if r.Kind == "suffix-fold" {
 		return len(base) >= len(r.Pattern) && strings.EqualFold(base[len(base)-len(r.Pattern):], r.Pattern)
 	}

walk.go

@@ -28,6 +28,11 @@ func Walk(tree fs.FS, options WalkOptions, visit func(string, Result) error) err
 	return defaults.Walk(tree, options, visit)
 }
 
+// WalkMatch visits the same entries as Walk without collecting evidence.
+func WalkMatch(tree fs.FS, options WalkOptions, visit func(string, Set) error) error {
+	return defaults.WalkMatch(tree, options, visit)
+}
+
 type ancestor struct {
 	path        string
 	set         Set
@@ -36,8 +41,29 @@ type ancestor struct {
 
 // Walk reuses inherited matches and includes this classifier's vendor roots.
 func (c *Classifier) Walk(tree fs.FS, options WalkOptions, visit func(string, Result) error) error {
-	if tree == nil || visit == nil {
-		return errors.New("filesystem and visitor are required")
+	if visit == nil {
+		return errors.New("visitor is required")
+	}
+	return c.walk(tree, options, true, func(name string, state matchState) error {
+		result := state.result()
+		result.Evidence = append([]Evidence(nil), state.evidence...)
+		return visit(name, result)
+	})
+}
+
+// WalkMatch includes this classifier's vendor roots without collecting evidence.
+func (c *Classifier) WalkMatch(tree fs.FS, options WalkOptions, visit func(string, Set) error) error {
+	if visit == nil {
+		return errors.New("visitor is required")
+	}
+	return c.walk(tree, options, false, func(name string, state matchState) error {
+		return visit(name, state.set)
+	})
+}
+
+func (c *Classifier) walk(tree fs.FS, options WalkOptions, explain bool, visit func(string, matchState) error) error {
+	if tree == nil {
+		return errors.New("filesystem is required")
 	}
 	if options.MaxEntries < 0 || options.MaxDepth < 0 {
 		return errors.New("walk limits must be non-negative")
@@ -82,15 +108,13 @@ func (c *Classifier) Walk(tree fs.FS, options WalkOptions, visit func(string, Re
 		parentState := parents[len(parents)-1]
 		state := matchState{set: parentState.set, evidence: evidence[:parentState.evidenceEnd]}
 		if entry.IsDir() {
-			c.directory(&state, name[split+1:], name, true)
+			c.directory(&state, name[split+1:], name, explain)
 			parents = append(parents, ancestor{path: name, set: state.set, evidenceEnd: len(state.evidence)})
 			name += "/"
 		} else {
-			matchFile(&state, name[split+1:], name, true)
+			matchFile(&state, name[split+1:], name, explain)
 		}
 		evidence = state.evidence
-		result := state.result()
-		result.Evidence = append([]Evidence(nil), state.evidence...)
-		return visit(name, result)
+		return visit(name, state)
 	})
 }