From 65fbf5805a6971ea2039ee640426eaa397aa5f8c Mon Sep 17 00:00:00 2001 From: John Simons Date: Fri, 9 Oct 2026 17:10:33 +1000 Subject: [PATCH] Fix full-text search matching messages that only share part of a term WebSearchToTsQuery and FREETEXT split each search term into separate words, so searching for a message id like "billing-1234-5678" also matched any message whose id shared a hyphen-separated piece, e.g. "billing-9999-5678". Build each term as a quoted phrase instead (ToTsQuery on PostgreSQL, CONTAINS on SQL Server), escaping the characters that would otherwise break the query syntax, so a term only matches messages containing it in full. A trailing * still requests a prefix match, matching the RavenDB persister's behaviour. --- .../PostgreSqlFullTextSearchDialect.cs | 20 ++++++++- .../SqlServerFullTextSearchDialect.cs | 17 ++++++-- .../FullTextSearchIndexTests.cs | 8 +++- .../EFCore/EFQueryTests.cs | 40 ++++++++++++++++++ .../PostgreSqlFullTextSearchDialect.cs | 26 +++++++++--- .../SqlServerFullTextSearchDialect.cs | 25 +++++++---- .../Infrastructure/IFullTextSearchDialect.cs | 6 ++- .../FullTextSearchIndexTests.cs | 10 ++++- .../EFCore/MessagesViewDataStoreTests.cs | 42 +++++++++++++++++++ 9 files changed, 172 insertions(+), 22 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs index cd48f14bb8..b23237fb15 100644 --- a/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs +++ b/src/ServiceControl.Audit.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs @@ -12,8 +12,24 @@ public IQueryable Search(IQueryable sour message.HeadersJson + " " + (message.BodyText ?? "").Substring(0, FullTextSearchSql.IndexedBodyLength) + " " + (message.MessageType ?? "").Replace(".", " ").Replace("+", " ")) - .Matches(EF.Functions.WebSearchToTsQuery(FullTextSearchSql.Configuration, ToOrQuery(searchTerms)))); + .Matches(EF.Functions.ToTsQuery(FullTextSearchSql.Configuration, ToOrQuery(searchTerms)))); static string ToOrQuery(string searchTerms) => - string.Join(" OR ", searchTerms.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + string.Join(" | ", searchTerms + .Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(ToPhrase) + .OfType()); + + static string? ToPhrase(string term) + { + var word = term.TrimEnd('*'); + if (word.Length == 0) + { + return null; + } + + var phrase = $"'{word.Replace(@"\", @"\\").Replace("'", "''")}'"; + + return word.Length < term.Length ? phrase + ":*" : phrase; + } } diff --git a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerFullTextSearchDialect.cs b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerFullTextSearchDialect.cs index a9608980a2..fe1dc2cee2 100644 --- a/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerFullTextSearchDialect.cs +++ b/src/ServiceControl.Audit.Persistence.EFCore.SqlServer/SqlServerFullTextSearchDialect.cs @@ -6,8 +6,17 @@ namespace ServiceControl.Audit.Persistence.EFCore.SqlServer; class SqlServerFullTextSearchDialect : IFullTextSearchDialect { - public IQueryable Search(IQueryable source, string searchTerms) => - source.Where(message => - EF.Functions.FreeText(message.HeadersJson, searchTerms) || - EF.Functions.FreeText(message.BodyText!, searchTerms)); + public IQueryable Search(IQueryable source, string searchTerms) + { + var searchCondition = ToPhrases(searchTerms); + + return source.Where(message => + EF.Functions.Contains(message.HeadersJson, searchCondition) || + EF.Functions.Contains(message.BodyText!, searchCondition)); + } + + static string ToPhrases(string searchTerms) => + string.Join(" OR ", searchTerms + .Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(term => $"\"{term.Replace("\"", "\"\"")}\"")); } diff --git a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs index 26ffbc3253..fc88ca1a37 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs @@ -19,7 +19,13 @@ public void Search_uses_the_indexed_expression() [Test] public void Terms_are_ored() { - Assert.That(SearchQuery("forty two"), Does.Contain("='forty OR two'")); + Assert.That(SearchQuery("forty two"), Does.Contain("'forty' | 'two'")); + } + + [Test] + public void A_trailing_star_makes_a_term_a_prefix() + { + Assert.That(SearchQuery("forty*"), Does.Contain("'forty':*")); } static string SearchQuery(string searchTerms) diff --git a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs index 7e78b13dc7..ed284acaa3 100644 --- a/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests/EFCore/EFQueryTests.cs @@ -82,6 +82,46 @@ public async Task Finds_a_word_that_is_quoted_in_single_quotes() Assert.That(found.Results, Has.Count.EqualTo(1)); } + [Test] + public async Task Does_not_match_messages_that_share_part_of_the_message_id() + { + await Ingest(MakeMessage(messageId: "billing-1234-5678"), MakeMessage(messageId: "billing-9999-5678")); + + var found = await MessagesViewStore.QueryMessages("billing-1234-5678", new PagingInfo(), new SortInfo("time_sent", "desc")); + + Assert.That(found.Results.Select(view => view.MessageId), Is.EqualTo(new[] { "billing-1234-5678" })); + } + + [Test] + public async Task Matches_a_term_ending_in_a_star_as_a_prefix() + { + await Ingest(MakeMessage(extraHeaders: new Dictionary { [SearchableHeader] = SearchableWord }), MakeMessage()); + + var found = await MessagesViewStore.QueryMessages("merid*", new PagingInfo(), new SortInfo("time_sent", "desc")); + + Assert.That(found.Results, Has.Count.EqualTo(1)); + } + + [TestCase("*")] + [TestCase(@"meridian\")] + [TestCase(@"merid\*")] + [TestCase("it's*")] + [TestCase("'")] + [TestCase("\"")] + [TestCase("foo \"bar")] + [TestCase("-")] + [TestCase("a & b")] + [TestCase("!(meridian")] + [TestCase("meridian:*")] + [TestCase("NEAR")] + [TestCase("AND")] + public async Task Searches_for_query_syntax_without_failing(string searchTerms) + { + await Ingest(MakeMessage(extraHeaders: new Dictionary { [SearchableHeader] = SearchableWord })); + + Assert.DoesNotThrowAsync(() => MessagesViewStore.QueryMessages(searchTerms, new PagingInfo(), new SortInfo("time_sent", "desc"))); + } + [Test] public async Task Sorts_by_status_in_the_order_the_primary_merges_instance_pages() { diff --git a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs index 8461a8a12c..08a37ccc4e 100644 --- a/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs +++ b/src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlFullTextSearchDialect.cs @@ -17,11 +17,27 @@ public IQueryable Search(IQueryable so message.HeadersJson + " " + (message.BodyText ?? "").Substring(0, FullTextSearchSql.IndexedBodyLength) + " " + (message.MessageType ?? "").Replace(".", " ").Replace("+", " ")) - .Matches(EF.Functions.WebSearchToTsQuery(FullTextSearchSql.Configuration, ToOrQuery(searchTerms)))); + .Matches(EF.Functions.ToTsQuery(FullTextSearchSql.Configuration, ToOrQuery(searchTerms)))); - // websearch_to_tsquery ANDs bare terms; the RavenDB persister ORs them, so the terms are - // rejoined with the operator that syntax understands. It also never throws on odd input, which - // a hand built tsquery would. static string ToOrQuery(string searchTerms) => - string.Join(" OR ", searchTerms.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + string.Join(" | ", searchTerms + .Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(ToPhrase) + .OfType()); + + // A quoted lexeme is parsed into a phrase of the words it contains, and :* makes each of them a + // prefix. to_tsquery throws on an unescaped backslash or quote, and on an empty pair of quotes, + // which is what a bare * would leave. + static string? ToPhrase(string term) + { + var word = term.TrimEnd('*'); + if (word.Length == 0) + { + return null; + } + + var phrase = $"'{word.Replace(@"\", @"\\").Replace("'", "''")}'"; + + return word.Length < term.Length ? phrase + ":*" : phrase; + } } diff --git a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerFullTextSearchDialect.cs b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerFullTextSearchDialect.cs index cc323d9f2e..c894af5280 100644 --- a/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerFullTextSearchDialect.cs +++ b/src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerFullTextSearchDialect.cs @@ -6,11 +6,22 @@ namespace ServiceControl.Persistence.EFCore.SqlServer; class SqlServerFullTextSearchDialect : IFullTextSearchDialect { - // FREETEXT ORs the terms itself, so the search string needs no parsing here. Both columns are - // covered by the index the AddFullTextSearch migration creates, and a NULL body simply does not - // match. - public IQueryable Search(IQueryable source, string searchTerms) => - source.Where(message => - EF.Functions.FreeText(message.HeadersJson, searchTerms) || - EF.Functions.FreeText(message.BodyText!, searchTerms)); + // Both columns are covered by the index the AddFullTextSearch migration creates, and a NULL body + // simply does not match. + public IQueryable Search(IQueryable source, string searchTerms) + { + var searchCondition = ToPhrases(searchTerms); + + return source.Where(message => + EF.Functions.Contains(message.HeadersJson, searchCondition) || + EF.Functions.Contains(message.BodyText!, searchCondition)); + } + + // CONTAINS parses its argument as query syntax, where two bare words in a row are an error. A + // quoted term is a phrase: the pieces the word breaker splits it into must appear together and + // in order, and a trailing * makes it a prefix. + static string ToPhrases(string searchTerms) => + string.Join(" OR ", searchTerms + .Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(term => $"\"{term.Replace("\"", "\"\"")}\"")); } diff --git a/src/ServiceControl.Persistence.EFCore/Infrastructure/IFullTextSearchDialect.cs b/src/ServiceControl.Persistence.EFCore/Infrastructure/IFullTextSearchDialect.cs index 45b3642e93..5150963ec1 100644 --- a/src/ServiceControl.Persistence.EFCore/Infrastructure/IFullTextSearchDialect.cs +++ b/src/ServiceControl.Persistence.EFCore/Infrastructure/IFullTextSearchDialect.cs @@ -11,8 +11,10 @@ namespace ServiceControl.Persistence.EFCore.Infrastructure; public interface IFullTextSearchDialect { /// - /// Terms are ORed, matching the RavenDB persister, whose Search defaults to SearchOperator.Or. - /// Callers guarantee the terms are not blank. + /// Terms are ORed, matching the RavenDB persister, whose Search defaults to SearchOperator.Or, and + /// a trailing * makes a term a prefix, as it does there. Each term matches as a phrase, so a + /// message id only matches the messages that contain all of it. Callers guarantee the terms are + /// not blank. /// IQueryable Search(IQueryable source, string searchTerms); } diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs b/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs index 29686a18dc..68ac2bfb83 100644 --- a/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/FullTextSearchIndexTests.cs @@ -23,7 +23,15 @@ public void Terms_are_ored() { var sql = SearchQuery("forty two"); - Assert.That(sql, Does.Contain("='forty OR two'")); + Assert.That(sql, Does.Contain("'forty' | 'two'")); + } + + [Test] + public void A_trailing_star_makes_a_term_a_prefix() + { + var sql = SearchQuery("forty*"); + + Assert.That(sql, Does.Contain("'forty':*")); } static string SearchQuery(string searchTerms) diff --git a/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs b/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs index 8656464871..bbf1a75e32 100644 --- a/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs +++ b/src/ServiceControl.Persistence.Tests/EFCore/MessagesViewDataStoreTests.cs @@ -380,6 +380,48 @@ public async Task Ors_the_search_terms() await AssertSearchFinds("zarquon capacitor", first, second); } + [Test] + public async Task Does_not_match_messages_that_share_part_of_the_message_id() + { + var matching = new IngestedFailure { MessageId = "billing-1234-5678", ExceptionMessage = "the zarquon overheated" }; + var sharingPieces = new IngestedFailure { MessageId = "billing-9999-5678", ExceptionMessage = "the zarquon overheated" }; + + await Ingest(matching, sharingPieces); + await AssertSearchFinds("zarquon", matching, sharingPieces); + + await AssertSearchFinds(matching.MessageId, matching); + } + + [Test] + public async Task Matches_a_term_ending_in_a_star_as_a_prefix() + { + var matching = new IngestedFailure { ExceptionMessage = "the zarquon overheated" }; + + await Ingest(matching, new IngestedFailure()); + + await AssertSearchFinds("zarq*", matching); + } + + [TestCase("*")] + [TestCase(@"zarquon\")] + [TestCase(@"zarq\*")] + [TestCase("it's*")] + [TestCase("'")] + [TestCase("\"")] + [TestCase("foo \"bar")] + [TestCase("-")] + [TestCase("a & b")] + [TestCase("!(zarquon")] + [TestCase("zarquon:*")] + [TestCase("NEAR")] + [TestCase("AND")] + public async Task Searches_for_query_syntax_without_failing(string searchTerms) + { + await Ingest(new IngestedFailure { ExceptionMessage = "the zarquon overheated" }); + + Assert.DoesNotThrowAsync(() => MessagesViewStore.GetAllMessagesForSearch(searchTerms, new PagingInfo(), new SortInfo())); + } + [Test] public async Task Searches_within_an_endpoint() {