Stream rewritten SPDX identifiers

parse.go

@@ -13,7 +13,7 @@ type Expression interface {
 	String() string
 	// Licenses returns all license identifiers in the expression.
 	Licenses() []string
-	rewriteIdentifiers(func(string) string) Expression
+	writeRewritten(*strings.Builder, func(string) string, tokenType)
 	isExpr()
 }
 

rewrite.go

@@ -1,5 +1,7 @@
 package spdx
 
+import "strings"
+
 // RewriteIdentifiers applies rewrite to each license identifier, exception
 // identifier, and complete license reference in expression order. Operators,
 // modifiers, and operator precedence are preserved. Replacements are inserted
@@ -10,38 +12,75 @@ func RewriteIdentifiers(expression Expression, rewrite func(string) string) stri
 	if rewrite == nil {
 		return expression.String()
 	}
-	return expression.rewriteIdentifiers(rewrite).String()
+	switch value := expression.(type) {
+	case *License:
+		license := value.rewrittenLicense(rewrite)
+		return license.String()
+	case *LicenseRef:
+		return rewrite(value.String())
+	case *SpecialValue:
+		return value.Value
+	}
+	var out strings.Builder
+	expression.writeRewritten(&out, rewrite, tokenEOF)
+	return out.String()
 }
 
-func (license *License) rewriteIdentifiers(rewrite func(string) string) Expression {
-	rewritten := &License{
-		ID:   rewrite(license.ID),
-		Plus: license.Plus,
+func (license *License) writeRewritten(out *strings.Builder, rewrite func(string) string, parent tokenType) {
+	rewritten := license.rewrittenLicense(rewrite)
+	paren := parent == tokenOr && rewritten.Exception != ""
+	if paren {
+		out.WriteByte('(')
+	}
+	out.WriteString(rewritten.ID)
+	if rewritten.Plus {
+		out.WriteByte('+')
+	}
+	if rewritten.Exception != "" {
+		out.WriteString(" WITH ")
+		out.WriteString(rewritten.Exception)
+	}
+	if paren {
+		out.WriteByte(')')
 	}
+}
+
+func (license *License) rewrittenLicense(rewrite func(string) string) License {
+	rewritten := License{ID: rewrite(license.ID), Plus: license.Plus}
 	if license.Exception != "" {
 		rewritten.Exception = rewrite(license.Exception)
 	}
 	return rewritten
 }
 
-func (reference *LicenseRef) rewriteIdentifiers(rewrite func(string) string) Expression {
-	return &License{ID: rewrite(reference.String())}
+func (reference *LicenseRef) writeRewritten(out *strings.Builder, rewrite func(string) string, _ tokenType) {
+	out.WriteString(rewrite(reference.String()))
 }
 
-func (expression *AndExpression) rewriteIdentifiers(rewrite func(string) string) Expression {
-	return &AndExpression{
-		Left:  expression.Left.rewriteIdentifiers(rewrite),
-		Right: expression.Right.rewriteIdentifiers(rewrite),
+func (expression *AndExpression) writeRewritten(out *strings.Builder, rewrite func(string) string, parent tokenType) {
+	if parent == tokenOr {
+		out.WriteByte('(')
+	}
+	expression.Left.writeRewritten(out, rewrite, tokenAnd)
+	out.WriteString(" AND ")
+	expression.Right.writeRewritten(out, rewrite, tokenAnd)
+	if parent == tokenOr {
+		out.WriteByte(')')
 	}
 }
 
-func (expression *OrExpression) rewriteIdentifiers(rewrite func(string) string) Expression {
-	return &OrExpression{
-		Left:  expression.Left.rewriteIdentifiers(rewrite),
-		Right: expression.Right.rewriteIdentifiers(rewrite),
+func (expression *OrExpression) writeRewritten(out *strings.Builder, rewrite func(string) string, parent tokenType) {
+	if parent == tokenAnd {
+		out.WriteByte('(')
+	}
+	expression.Left.writeRewritten(out, rewrite, tokenOr)
+	out.WriteString(" OR ")
+	expression.Right.writeRewritten(out, rewrite, tokenOr)
+	if parent == tokenAnd {
+		out.WriteByte(')')
 	}
 }
 
-func (special *SpecialValue) rewriteIdentifiers(func(string) string) Expression {
-	return &SpecialValue{Value: special.Value}
+func (special *SpecialValue) writeRewritten(out *strings.Builder, _ func(string) string, _ tokenType) {
+	out.WriteString(special.Value)
 }

rewrite_stream_test.go

@@ -0,0 +1,79 @@
+package spdx
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+)
+
+func cloneRewritten(expression Expression, rewrite func(string) string) Expression {
+	switch e := expression.(type) {
+	case *License:
+		out := &License{ID: rewrite(e.ID), Plus: e.Plus}
+		if e.Exception != "" {
+			out.Exception = rewrite(e.Exception)
+		}
+		return out
+	case *LicenseRef:
+		return &License{ID: rewrite(e.String())}
+	case *AndExpression:
+		return &AndExpression{Left: cloneRewritten(e.Left, rewrite), Right: cloneRewritten(e.Right, rewrite)}
+	case *OrExpression:
+		return &OrExpression{Left: cloneRewritten(e.Left, rewrite), Right: cloneRewritten(e.Right, rewrite)}
+	case *SpecialValue:
+		return &SpecialValue{Value: e.Value}
+	default:
+		panic("unexpected expression type")
+	}
+}
+
+func FuzzRewriteMatchesClone(f *testing.F) {
+	for _, input := range []string{"MIT", "MIT OR Apache-2.0 AND ISC", "GPL-2.0+ OR GPL-2.0 WITH Classpath-exception-2.0", "LicenseRef-custom", "DocumentRef-x:LicenseRef-y AND (MIT OR ISC)", "NONE"} {
+		f.Add(input, false)
+		f.Add(input, true)
+	}
+	f.Fuzz(func(t *testing.T, input string, erase bool) {
+		if len(input) > 4096 {
+			t.Skip()
+		}
+		expression, err := ParseSyntax(input)
+		if err != nil {
+			return
+		}
+		original := expression.String()
+		rewrite := func(id string) string {
+			if erase && strings.Contains(id, "exception") {
+				return ""
+			}
+			return strings.ToLower(id)
+		}
+		want := cloneRewritten(expression, rewrite).String()
+		if got := RewriteIdentifiers(expression, rewrite); got != want {
+			t.Fatalf("%q: got %q, want %q", input, got, want)
+		}
+		if expression.String() != original {
+			t.Fatal("input expression mutated")
+		}
+	})
+}
+
+func BenchmarkRewriteSize(b *testing.B) {
+	for _, size := range []int{1, 4, 32, 128} {
+		expression, err := ParseSyntax(strings.TrimSuffix(strings.Repeat("MIT OR ", size), " OR "))
+		if err != nil {
+			b.Fatal(err)
+		}
+		for _, clone := range []bool{false, true} {
+			b.Run(fmt.Sprintf("%d/clone=%t", size, clone), func(b *testing.B) {
+				b.ReportAllocs()
+				for b.Loop() {
+					if clone {
+						_ = cloneRewritten(expression, strings.ToLower).String()
+					} else {
+						_ = RewriteIdentifiers(expression, strings.ToLower)
+					}
+				}
+			})
+		}
+	}
+}