From ed817bd0459804e3ec72de654df82e2d2d060465 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 6 Sep 2026 21:34:51 +0100 Subject: [PATCH] Stream rewritten SPDX identifiers --- parse.go | 2 +- rewrite.go | 73 +++++++++++++++++++++++++++++--------- rewrite_stream_test.go | 79 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 136 insertions(+), 18 deletions(-) create mode 100644 rewrite_stream_test.go diff --git a/parse.go b/parse.go index dd44c58..2e0861d 100644 --- a/parse.go +++ b/parse.go @@ -13,7 +13,7 @@ type Expression interface { String() string // Licenses returns all license identifiers in the expression. Licenses() []string - rewriteIdentifiers(func(string) string) Expression + writeRewritten(*strings.Builder, func(string) string, tokenType) isExpr() } diff --git a/rewrite.go b/rewrite.go index f919245..fcfd821 100644 --- a/rewrite.go +++ b/rewrite.go @@ -1,5 +1,7 @@ package spdx +import "strings" + // RewriteIdentifiers applies rewrite to each license identifier, exception // identifier, and complete license reference in expression order. Operators, // modifiers, and operator precedence are preserved. Replacements are inserted @@ -10,38 +12,75 @@ func RewriteIdentifiers(expression Expression, rewrite func(string) string) stri if rewrite == nil { return expression.String() } - return expression.rewriteIdentifiers(rewrite).String() + switch value := expression.(type) { + case *License: + license := value.rewrittenLicense(rewrite) + return license.String() + case *LicenseRef: + return rewrite(value.String()) + case *SpecialValue: + return value.Value + } + var out strings.Builder + expression.writeRewritten(&out, rewrite, tokenEOF) + return out.String() } -func (license *License) rewriteIdentifiers(rewrite func(string) string) Expression { - rewritten := &License{ - ID: rewrite(license.ID), - Plus: license.Plus, +func (license *License) writeRewritten(out *strings.Builder, rewrite func(string) string, parent tokenType) { + rewritten := license.rewrittenLicense(rewrite) + paren := parent == tokenOr && rewritten.Exception != "" + if paren { + out.WriteByte('(') + } + out.WriteString(rewritten.ID) + if rewritten.Plus { + out.WriteByte('+') + } + if rewritten.Exception != "" { + out.WriteString(" WITH ") + out.WriteString(rewritten.Exception) + } + if paren { + out.WriteByte(')') } +} + +func (license *License) rewrittenLicense(rewrite func(string) string) License { + rewritten := License{ID: rewrite(license.ID), Plus: license.Plus} if license.Exception != "" { rewritten.Exception = rewrite(license.Exception) } return rewritten } -func (reference *LicenseRef) rewriteIdentifiers(rewrite func(string) string) Expression { - return &License{ID: rewrite(reference.String())} +func (reference *LicenseRef) writeRewritten(out *strings.Builder, rewrite func(string) string, _ tokenType) { + out.WriteString(rewrite(reference.String())) } -func (expression *AndExpression) rewriteIdentifiers(rewrite func(string) string) Expression { - return &AndExpression{ - Left: expression.Left.rewriteIdentifiers(rewrite), - Right: expression.Right.rewriteIdentifiers(rewrite), +func (expression *AndExpression) writeRewritten(out *strings.Builder, rewrite func(string) string, parent tokenType) { + if parent == tokenOr { + out.WriteByte('(') + } + expression.Left.writeRewritten(out, rewrite, tokenAnd) + out.WriteString(" AND ") + expression.Right.writeRewritten(out, rewrite, tokenAnd) + if parent == tokenOr { + out.WriteByte(')') } } -func (expression *OrExpression) rewriteIdentifiers(rewrite func(string) string) Expression { - return &OrExpression{ - Left: expression.Left.rewriteIdentifiers(rewrite), - Right: expression.Right.rewriteIdentifiers(rewrite), +func (expression *OrExpression) writeRewritten(out *strings.Builder, rewrite func(string) string, parent tokenType) { + if parent == tokenAnd { + out.WriteByte('(') + } + expression.Left.writeRewritten(out, rewrite, tokenOr) + out.WriteString(" OR ") + expression.Right.writeRewritten(out, rewrite, tokenOr) + if parent == tokenAnd { + out.WriteByte(')') } } -func (special *SpecialValue) rewriteIdentifiers(func(string) string) Expression { - return &SpecialValue{Value: special.Value} +func (special *SpecialValue) writeRewritten(out *strings.Builder, _ func(string) string, _ tokenType) { + out.WriteString(special.Value) } diff --git a/rewrite_stream_test.go b/rewrite_stream_test.go new file mode 100644 index 0000000..623d609 --- /dev/null +++ b/rewrite_stream_test.go @@ -0,0 +1,79 @@ +package spdx + +import ( + "fmt" + "strings" + "testing" +) + +func cloneRewritten(expression Expression, rewrite func(string) string) Expression { + switch e := expression.(type) { + case *License: + out := &License{ID: rewrite(e.ID), Plus: e.Plus} + if e.Exception != "" { + out.Exception = rewrite(e.Exception) + } + return out + case *LicenseRef: + return &License{ID: rewrite(e.String())} + case *AndExpression: + return &AndExpression{Left: cloneRewritten(e.Left, rewrite), Right: cloneRewritten(e.Right, rewrite)} + case *OrExpression: + return &OrExpression{Left: cloneRewritten(e.Left, rewrite), Right: cloneRewritten(e.Right, rewrite)} + case *SpecialValue: + return &SpecialValue{Value: e.Value} + default: + panic("unexpected expression type") + } +} + +func FuzzRewriteMatchesClone(f *testing.F) { + for _, input := range []string{"MIT", "MIT OR Apache-2.0 AND ISC", "GPL-2.0+ OR GPL-2.0 WITH Classpath-exception-2.0", "LicenseRef-custom", "DocumentRef-x:LicenseRef-y AND (MIT OR ISC)", "NONE"} { + f.Add(input, false) + f.Add(input, true) + } + f.Fuzz(func(t *testing.T, input string, erase bool) { + if len(input) > 4096 { + t.Skip() + } + expression, err := ParseSyntax(input) + if err != nil { + return + } + original := expression.String() + rewrite := func(id string) string { + if erase && strings.Contains(id, "exception") { + return "" + } + return strings.ToLower(id) + } + want := cloneRewritten(expression, rewrite).String() + if got := RewriteIdentifiers(expression, rewrite); got != want { + t.Fatalf("%q: got %q, want %q", input, got, want) + } + if expression.String() != original { + t.Fatal("input expression mutated") + } + }) +} + +func BenchmarkRewriteSize(b *testing.B) { + for _, size := range []int{1, 4, 32, 128} { + expression, err := ParseSyntax(strings.TrimSuffix(strings.Repeat("MIT OR ", size), " OR ")) + if err != nil { + b.Fatal(err) + } + for _, clone := range []bool{false, true} { + b.Run(fmt.Sprintf("%d/clone=%t", size, clone), func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + if clone { + _ = cloneRewritten(expression, strings.ToLower).String() + } else { + _ = RewriteIdentifiers(expression, strings.ToLower) + } + } + }) + } + } +}