From d49c42d4e08d6689c38ed24c3688941e36cd96c9 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Wed, 16 Sep 2026 19:22:32 +0100 Subject: [PATCH] Skip Git metadata and guard corpus versions --- README.md | 4 ++ cmd/roles/main.go | 20 ++++++- cmd/roles/main_test.go | 75 +++++++++++++++++++++++- corpus_version_test.go | 126 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 220 insertions(+), 5 deletions(-) create mode 100644 corpus_version_test.go diff --git a/README.md b/README.md index 21fa655..dfe012b 100644 --- a/README.md +++ b/README.md @@ -242,6 +242,8 @@ and dependency-directory conventions from [GitHub's gitignore templates](https:/ vendor, documentation and generated-file conventions, adapted to these roles. [NOTICE](NOTICE) records source revisions and attribution. `CorpusVersion` identifies the embedded classification semantics for caller caches. +Tests bind each version to a fingerprint of the canonical rules and their +classification output, so semantic changes require a new version entry. Coverage differs from [GitHub Linguist](https://github.com/github-linguist/linguist): roles treats `testdata` as test/fixture rather than vendor. Build-output and cache directories such as `dist` and `cache` @@ -280,6 +282,8 @@ go run ./cmd/roles -labels-only -root . `-labels-only` uses `Match` or `WalkMatch` and emits `null` evidence. JSON encoding still allocates output records; library callers can use `Set` directly. +Tree mode prunes Git metadata directories and `.git` files. Library traversal +applies no automatic directory exclusions. ## Testing diff --git a/cmd/roles/main.go b/cmd/roles/main.go index 274e63c..844ad33 100644 --- a/cmd/roles/main.go +++ b/cmd/roles/main.go @@ -6,7 +6,9 @@ import ( "flag" "fmt" "io" + "io/fs" "os" + "strings" "unicode/utf8" "github.com/git-pkgs/roles" @@ -44,12 +46,21 @@ func run(args []string, output io.Writer) error { if err != nil { return err } + emitTree := func(name string, result roles.Result) error { + if isGitMetadata(name) { + if strings.HasSuffix(name, "/") { + return fs.SkipDir + } + return nil + } + return emit(name, result) + } if *labelsOnly { err = roles.WalkMatch(tree.FS(), roles.WalkOptions{}, func(name string, set roles.Set) error { - return emit(name, roles.Result{Roles: set.List()}) + return emitTree(name, roles.Result{Roles: set.List()}) }) } else { - err = roles.Walk(tree.FS(), roles.WalkOptions{}, emit) + err = roles.Walk(tree.FS(), roles.WalkOptions{}, emitTree) } return errors.Join(err, tree.Close()) } @@ -65,6 +76,11 @@ func run(args []string, output io.Writer) error { return nil } +func isGitMetadata(name string) bool { + name = strings.TrimSuffix(name, "/") + return name == ".git" || strings.HasSuffix(name, "/.git") +} + func classify(name string, labelsOnly bool) (roles.Result, error) { if labelsOnly { set, err := roles.Match(name) diff --git a/cmd/roles/main_test.go b/cmd/roles/main_test.go index 2d71f82..ae3a054 100644 --- a/cmd/roles/main_test.go +++ b/cmd/roles/main_test.go @@ -5,11 +5,18 @@ import ( "encoding/json" "os" "path/filepath" + "slices" + "strings" "testing" "github.com/git-pkgs/roles" ) +const ( + labelsOnlyFlag = "-labels-only" + rootFlag = "-root" +) + func TestRun(t *testing.T) { var out bytes.Buffer if err := run([]string{"vendor/sqlite/LICENSE"}, &out); err != nil { @@ -39,7 +46,7 @@ func TestRunTree(t *testing.T) { t.Fatal(err) } var out bytes.Buffer - if err := run([]string{"-root", root}, &out); err != nil { + if err := run([]string{rootFlag, root}, &out); err != nil { t.Fatal(err) } decoder := json.NewDecoder(&out) @@ -58,6 +65,68 @@ func TestRunTree(t *testing.T) { } } +func TestRunTreeSkipsGitMetadata(t *testing.T) { + const ( + githubWorkflow = ".github/workflows/ci.yml" + projectGitFile = "project.git/file" + sourceFile = "src/main.go" + ) + root := t.TempDir() + for _, name := range []string{ + ".git/objects/object", + "nested/.git/config", + "submodule/.git", + githubWorkflow, + projectGitFile, + sourceFile, + } { + path := filepath.Join(root, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + } + for _, args := range [][]string{{rootFlag, root}, {labelsOnlyFlag, rootFlag, root}} { + t.Run(strings.Join(args, " "), func(t *testing.T) { + paths := runTreePaths(t, args) + for _, path := range paths { + clean := strings.TrimSuffix(path, "/") + if clean == ".git" || strings.HasPrefix(clean, ".git/") || strings.Contains(clean, "/.git/") || strings.HasSuffix(clean, "/.git") { + t.Fatalf("emitted Git metadata path %q", path) + } + } + for _, name := range []string{githubWorkflow, projectGitFile, sourceFile} { + if !slices.Contains(paths, name) { + t.Errorf("missing non-metadata path %q", name) + } + } + }) + } +} + +func runTreePaths(t *testing.T, args []string) []string { + t.Helper() + var out bytes.Buffer + if err := run(args, &out); err != nil { + t.Fatal(err) + } + var paths []string + decoder := json.NewDecoder(&out) + for decoder.More() { + var got struct { + Path string + roles.Result + } + if err := decoder.Decode(&got); err != nil { + t.Fatal(err) + } + paths = append(paths, got.Path) + } + return paths +} + func TestRunInvalidUTF8(t *testing.T) { var out bytes.Buffer if err := run([]string{"src/\xff.go"}, &out); err == nil || out.Len() != 0 { @@ -74,8 +143,8 @@ func TestRunLabelsOnly(t *testing.T) { t.Fatal(err) } for _, args := range [][]string{ - {"-labels-only", "vendor/LICENSE"}, - {"-labels-only", "-root", root}, + {labelsOnlyFlag, "vendor/LICENSE"}, + {labelsOnlyFlag, rootFlag, root}, } { var out bytes.Buffer if err := run(args, &out); err != nil { diff --git a/corpus_version_test.go b/corpus_version_test.go new file mode 100644 index 0000000..d6c4187 --- /dev/null +++ b/corpus_version_test.go @@ -0,0 +1,126 @@ +package roles_test + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "strings" + "testing" + + "github.com/git-pkgs/roles" +) + +type fingerprintRule struct { + ID string `json:"id"` + Kind string `json:"kind"` + Pattern string `json:"pattern"` + Role roles.Role `json:"role"` + Subtype string `json:"subtype,omitempty"` + Ecosystem string `json:"ecosystem,omitempty"` + Source string `json:"source"` + Extensions []string `json:"extensions,omitempty"` +} + +const ( + fingerprintCargo = "cargo" + fingerprintNoticePath = "vendor/LICENSES/NOTICE" + fingerprintVendorRoot = "deps/crates" +) + +func TestCorpusVersionFingerprint(t *testing.T) { + data, err := os.ReadFile("corpus/rules.json") + if err != nil { + t.Fatal(err) + } + var corpusData any + if err := json.Unmarshal(data, &corpusData); err != nil { + t.Fatal(err) + } + canonical, err := json.Marshal(corpusData) + if err != nil { + t.Fatal(err) + } + var corpus []fingerprintRule + if err := json.Unmarshal(data, &corpus); err != nil { + t.Fatal(err) + } + hash := sha256.New() + _, _ = hash.Write(canonical) + for _, rule := range corpus { + path := rule.Pattern + if strings.HasSuffix(rule.Kind, "-fold") { + path = alternateCase(path) + } + switch rule.Kind { + case "directory", "directory-fold", "directory-path": + path += "/arbitrary.bin" + case "suffix", "suffix-fold": + path = "file" + path + case "stem-fold": + path += rule.Extensions[0] + } + result, err := roles.Classify(path) + if err != nil { + t.Fatal(err) + } + _, _ = fmt.Fprintf(hash, "\n%s\n", path) + if err := json.NewEncoder(hash).Encode(result); err != nil { + t.Fatal(err) + } + } + + for _, path := range []string{ + fingerprintNoticePath, + "packages/parser/vendor/src/parser_test.go", + "modules/a/testdata/package-lock.json", + "packages/api/.github/workflows/ci.yml", + "src/Form.Designer.cs", + "main.go", + } { + result, err := roles.Classify(path) + if err != nil { + t.Fatal(err) + } + _, _ = fmt.Fprintf(hash, "\n%s\n", path) + if err := json.NewEncoder(hash).Encode(result); err != nil { + t.Fatal(err) + } + } + + classifier, err := roles.New([]roles.VendorRoot{{Path: fingerprintVendorRoot, Ecosystem: fingerprintCargo, EvidencePath: ".cargo/config.toml"}}) + if err != nil { + t.Fatal(err) + } + contextResult, err := classifier.Classify("deps/crates/example/src/lib.rs") + if err != nil { + t.Fatal(err) + } + if err := json.NewEncoder(hash).Encode(contextResult); err != nil { + t.Fatal(err) + } + blob, err := roles.ClassifyBlob("src/client.go", []byte("// Code generated by clientgen. DO NOT EDIT.\npackage client\n")) + if err != nil { + t.Fatal(err) + } + if err := json.NewEncoder(hash).Encode(blob); err != nil { + t.Fatal(err) + } + + got := hex.EncodeToString(hash.Sum(nil)) + want, ok := map[string]string{ + "1": "3d5ee1d9281183f54b6c440d484920f1fdef90f305a01b649be7f2a961739019", + }[roles.CorpusVersion] + if !ok || got != want { + t.Fatalf("classification semantics changed without a CorpusVersion bump: version=%q fingerprint=%s", roles.CorpusVersion, got) + } +} + +func alternateCase(value string) string { + lower := strings.ToLower(value) + if value == lower { + return strings.ToUpper(value) + } + return lower +}