diff --git a/Morpheus.Tests/SimHasherTests.cs b/Morpheus.Tests/SimHasherTests.cs index cef0a3a..82844b0 100644 --- a/Morpheus.Tests/SimHasherTests.cs +++ b/Morpheus.Tests/SimHasherTests.cs @@ -4,6 +4,14 @@ namespace Morpheus.Tests; public class SimHasherTests { + [Theory] + [InlineData("a ! b", "a b")] + [InlineData("a \u0301 b", "a b")] + public void Normalize_CollapsesWhitespaceAcrossRemovedCharacters(string input, string expected) + { + Assert.Equal(expected, SimHasher.Normalize(input)); + } + [Fact] public void ComputeSimHash_ReturnsZeroHashForVeryShortNormalizedText() { diff --git a/Utilities/Text/SimHasher.cs b/Utilities/Text/SimHasher.cs index 424e1df..c1d303d 100644 --- a/Utilities/Text/SimHasher.cs +++ b/Utilities/Text/SimHasher.cs @@ -27,7 +27,6 @@ public static string Normalize(string input) } continue; } - lastWasSpace = false; // Strip combining marks (diacritics) var cat = CharUnicodeInfo.GetUnicodeCategory(ch.ToString(), 0); @@ -42,6 +41,7 @@ public static string Normalize(string input) if (Rune.IsDigit(ch)) { sb.Append('0'); + lastWasSpace = false; continue; } @@ -50,6 +50,7 @@ public static string Normalize(string input) continue; sb.Append(ch.ToString()); + lastWasSpace = false; } var result = sb.ToString().Trim();