Use Lucene search engine for new RavenDB databases and flag indexes still on Corax (#5833)

* ✨ Use Lucene search engine for newly created RavenDB databases

New databases are created with Indexing.Static/Auto.SearchEngineType=Lucene:
Lucene indexes are smaller, use less memory and are faster for the index
definitions ServiceControl uses. Existing databases keep the engine they
currently have configured, so no full index rebuild is triggered on upgrade.

* ✨ Log a warning at startup when RavenDB indexes still use Corax

New databases use Lucene, existing databases keep their configured search
engine. Operators of existing databases now get a WARN at startup listing the
indexes still on Corax so they can plan the transition to Lucene.

* 📝 Clarify rebuild impact in Corax search engine warning

* ✨ Add custom check reporting RavenDB indexes still using Corax

Surfaces the same information as the startup warning in ServicePulse via a
custom check on both the Primary (main + throughput database) and Audit
instances, so operators see it without inspecting logs.

Author: ramonsmits

src/ServiceControl.Audit.Persistence.RavenDB/CustomChecks/CheckRavenDBSearchEngine.cs

@@ -0,0 +1,22 @@
+namespace ServiceControl.Audit.Persistence.RavenDB.CustomChecks;
+
+using System;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using NServiceBus.CustomChecks;
+using ServiceControl.RavenDB;
+
+class CheckRavenDBSearchEngine(IRavenDocumentStoreProvider documentStoreProvider, DatabaseConfiguration databaseConfiguration) : CustomCheck("Audit Database Search Engine", "ServiceControl.Audit Health", TimeSpan.FromHours(1))
+{
+    public override async Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
+    {
+        var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken);
+
+        var coraxIndexes = await StartupChecks.FindIndexesUsingCorax(documentStore, databaseConfiguration.Name, cancellationToken);
+
+        return coraxIndexes.Length == 0
+            ? CheckResult.Pass
+            : CheckResult.Failed(StartupChecks.CoraxIndexesMessage(coraxIndexes.Select(i => $"{databaseConfiguration.Name}/{i}")));
+    }
+}

src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs

@@ -15,6 +15,7 @@
 using Raven.Client.ServerWide.Operations;
 using Raven.Client.ServerWide.Operations.Configuration;
 using Indexes;
+using ServiceControl.RavenDB;
 using SagaAudit;
 
 class DatabaseSetup(DatabaseConfiguration configuration)
@@ -27,6 +28,8 @@ public async Task Execute(IDocumentStore documentStore, CancellationToken cancel
 
         await CreateIndexes(documentStore, configuration.EnableFullTextSearch, cancellationToken);
 
+        await StartupChecks.WarnIfIndexesUseCorax(documentStore, configuration.Name, cancellationToken);
+
         await LicenseStatusCheck.WaitForLicenseOrThrow(documentStore, cancellationToken);
         await ConfigureExpiration(documentStore, cancellationToken);
     }
@@ -41,7 +44,9 @@ async Task CreateDatabase(IDocumentStore documentStore, string databaseName, Can
             {
                 var databaseRecord = new DatabaseRecord(databaseName);
 
-                SetSearchEngineType(databaseRecord, SearchEngineType.Corax);
+                // New databases use Lucene: smaller indexes, lower memory usage and faster for our index definitions.
+                // Existing databases keep the engine they were created with, see UpdateDatabaseSettings.
+                SetSearchEngineType(databaseRecord, SearchEngineType.Lucene);
 
                 await documentStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(databaseRecord), cancellationToken);
             }
@@ -56,6 +61,9 @@ async Task UpdateDatabaseSettings(IDocumentStore documentStore, string databaseN
     {
         var databaseRecord = await documentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken) ?? throw new InvalidOperationException($"Database '{databaseName}' does not exist.");
 
+        // Existing databases keep their configured search engine. Changing it would trigger a full rebuild of all
+        // indexes, which can take a long time and a lot of resources on large databases. Databases created before the
+        // search engine was pinned explicitly get Corax, which was the default at the time.
         if (!SetSearchEngineType(databaseRecord, SearchEngineType.Corax))
         {
             return;

src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistence.cs

@@ -22,6 +22,7 @@ public void AddPersistence(IServiceCollection services)
                 endpointConfiguration.AddCustomCheck<CheckFreeDiskSpace>();
                 endpointConfiguration.AddCustomCheck<CheckMinimumStorageRequiredForIngestion>();
                 endpointConfiguration.AddCustomCheck<CheckRavenDBIndexLag>();
+                endpointConfiguration.AddCustomCheck<CheckRavenDBSearchEngine>();
             }
 
             services.AddSingleton<IAuditDataStore, RavenAuditDataStore>();

src/ServiceControl.Audit.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt

@@ -1,4 +1,5 @@
 ServiceControl.Audit Health: Audit Database Index Lag
+ServiceControl.Audit Health: Audit Database Search Engine
 ServiceControl.Audit Health: Audit Message Ingestion Process
 ServiceControl.Audit Health: RavenDB dirty memory
 Storage space: ServiceControl.Audit database

src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs

@@ -9,22 +9,43 @@ namespace ServiceControl.Audit.Persistence.Tests;
 using Raven.Client.Documents.Operations.Indexes;
 using Raven.Client.Exceptions;
 using Raven.Client.Exceptions.Documents.Indexes;
+using ServiceControl.RavenDB;
 
 [TestFixture]
 class IndexSetupTests : PersistenceTestFixture
 {
     [Test]
-    public async Task Corax_should_be_the_default_search_engine_type()
+    public async Task Lucene_should_be_the_default_search_engine_type_for_new_databases()
     {
         var indexes = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexesOperation(0, int.MaxValue));
 
         foreach (var index in indexes)
         {
             var indexStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(DatabaseSetup.MessagesViewIndexWithFulltextSearchName));
-            Assert.That(indexStats.SearchEngineType, Is.EqualTo(SearchEngineType.Corax), $"{index.Name} is not using Corax");
+            Assert.That(indexStats.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene), $"{index.Name} is not using Lucene");
         }
     }
 
+    [Test]
+    public async Task Startup_check_should_not_report_corax_indexes_for_new_database()
+    {
+        var coraxIndexes = await StartupChecks.FindIndexesUsingCorax(configuration.DocumentStore, configuration.DocumentStore.Database, TestTimeoutCancellationToken);
+
+        Assert.That(coraxIndexes, Is.Empty);
+    }
+
+    [Test]
+    public async Task Startup_check_should_report_indexes_using_corax()
+    {
+        var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Corax.ToString() } };
+
+        await UpdateIndex(index);
+
+        var coraxIndexes = await StartupChecks.FindIndexesUsingCorax(configuration.DocumentStore, configuration.DocumentStore.Database, TestTimeoutCancellationToken);
+
+        Assert.That(coraxIndexes, Is.EqualTo(new[] { index.IndexName }));
+    }
+
     [Test]
     public async Task Free_text_search_index_should_be_used_by_default()
     {
@@ -50,49 +71,49 @@ public async Task Free_text_search_index_can_be_opted_out_from()
     [Test]
     public async Task Indexes_should_be_reset_on_setup()
     {
-        var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() } };
+        var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Corax.ToString() } };
 
         var indexWithCustomConfigStats = await UpdateIndex(index);
 
-        Assert.That(indexWithCustomConfigStats.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene));
+        Assert.That(indexWithCustomConfigStats.SearchEngineType, Is.EqualTo(SearchEngineType.Corax));
 
         await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, TestTimeoutCancellationToken);
 
         await WaitForIndexDefinitionUpdate(indexWithCustomConfigStats);
 
         var indexAfterResetStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName));
 
-        Assert.That(indexAfterResetStats.SearchEngineType, Is.EqualTo(SearchEngineType.Corax));
+        Assert.That(indexAfterResetStats.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene));
     }
 
     [Test]
     public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_ignore()
     {
         var index = new MessagesViewIndexWithFullTextSearch
         {
-            Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() },
+            Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Corax.ToString() },
             LockMode = IndexLockMode.LockedIgnore
         };
 
         var indexStatsBefore = await UpdateIndex(index);
 
-        Assert.That(indexStatsBefore.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene));
+        Assert.That(indexStatsBefore.SearchEngineType, Is.EqualTo(SearchEngineType.Corax));
 
         await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, TestTimeoutCancellationToken);
 
         // raven will ignore the update since index was locked, so best we can do is wait a bit and check that settings hasn't changed
         await Task.Delay(1000);
 
         var indexStatsAfter = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName));
-        Assert.That(indexStatsAfter.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene));
+        Assert.That(indexStatsAfter.SearchEngineType, Is.EqualTo(SearchEngineType.Corax));
     }
 
     [Test]
     public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_error()
     {
         var index = new MessagesViewIndexWithFullTextSearch
         {
-            Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() },
+            Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Corax.ToString() },
             LockMode = IndexLockMode.LockedError
         };
 

src/ServiceControl.Persistence.RavenDB/CustomChecks/CheckRavenDBSearchEngine.cs

@@ -0,0 +1,30 @@
+namespace ServiceControl.Persistence.RavenDB.CustomChecks;
+
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using NServiceBus.CustomChecks;
+using ServiceControl.RavenDB;
+
+class CheckRavenDBSearchEngine(IRavenDocumentStoreProvider documentStoreProvider, RavenPersisterSettings settings) : CustomCheck("Error Database Search Engine", "ServiceControl Health", TimeSpan.FromHours(1))
+{
+    public override async Task<CheckResult> PerformCheck(CancellationToken cancellationToken = default)
+    {
+        var documentStore = await documentStoreProvider.GetDocumentStore(cancellationToken);
+
+        var coraxIndexes = new List<string>();
+
+        foreach (var databaseName in new[] { settings.DatabaseName, settings.ThroughputDatabaseName })
+        {
+            foreach (var indexName in await StartupChecks.FindIndexesUsingCorax(documentStore, databaseName, cancellationToken))
+            {
+                coraxIndexes.Add($"{databaseName}/{indexName}");
+            }
+        }
+
+        return coraxIndexes.Count == 0
+            ? CheckResult.Pass
+            : CheckResult.Failed(StartupChecks.CoraxIndexesMessage(coraxIndexes));
+    }
+}

src/ServiceControl.Persistence.RavenDB/DatabaseSetup.cs

@@ -10,6 +10,7 @@ namespace ServiceControl.Persistence.RavenDB
     using Raven.Client.ServerWide;
     using Raven.Client.ServerWide.Operations;
     using Raven.Client.ServerWide.Operations.Configuration;
+    using ServiceControl.RavenDB;
 
     class DatabaseSetup(RavenPersisterSettings settings, IDocumentStore documentStore)
     {
@@ -23,6 +24,9 @@ public async Task Execute(CancellationToken cancellationToken = default)
 
             await IndexCreation.CreateIndexesAsync(typeof(DatabaseSetup).Assembly, documentStore, null, null, cancellationToken);
 
+            await StartupChecks.WarnIfIndexesUseCorax(documentStore, settings.DatabaseName, cancellationToken);
+            await StartupChecks.WarnIfIndexesUseCorax(documentStore, settings.ThroughputDatabaseName, cancellationToken);
+
             await LicenseStatusCheck.WaitForLicenseOrThrow(documentStore, cancellationToken);
             await ConfigureExpiration(settings, cancellationToken);
         }
@@ -36,8 +40,11 @@ async Task CreateDatabase(string databaseName, CancellationToken cancellationTok
                 try
                 {
                     var databaseRecord = new DatabaseRecord(databaseName);
-                    databaseRecord.Settings.Add("Indexing.Auto.SearchEngineType", "Corax");
-                    databaseRecord.Settings.Add("Indexing.Static.SearchEngineType", "Corax");
+
+                    // New databases use Lucene: smaller indexes, lower memory usage and faster for our index definitions.
+                    // Existing databases keep the engine they were created with, see UpdateDatabaseSettings.
+                    databaseRecord.Settings.Add("Indexing.Auto.SearchEngineType", "Lucene");
+                    databaseRecord.Settings.Add("Indexing.Static.SearchEngineType", "Lucene");
 
                     await documentStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(databaseRecord), cancellationToken);
                 }
@@ -57,6 +64,9 @@ async Task UpdateDatabaseSettings(string databaseName, CancellationToken cancell
                 throw new InvalidOperationException($"Database '{databaseName}' does not exist.");
             }
 
+            // Existing databases keep their configured search engine. Changing it would trigger a full rebuild of all
+            // indexes, which can take a long time and a lot of resources on large databases. Databases created before the
+            // search engine was pinned explicitly get Corax, which was the default at the time.
             var updated = false;
 
             updated |= dbRecord.Settings.TryAdd("Indexing.Auto.SearchEngineType", "Corax");

src/ServiceControl.Persistence.RavenDB/RavenPersistence.cs

@@ -48,6 +48,7 @@ public void AddPersistence(IServiceCollection services)
         services.AddCustomCheck<CheckFreeDiskSpace>();
         services.AddCustomCheck<CheckMinimumStorageRequiredForIngestion>();
         services.AddCustomCheck<CheckDirtyMemory>();
+        services.AddCustomCheck<CheckRavenDBSearchEngine>();
 
         services.AddSingleton<MemoryInformationRetriever>();
         services.AddSingleton<OperationsManager>();

src/ServiceControl.Persistence.Tests.RavenDB/ApprovalFiles/CustomCheckTests.VerifyCustomChecks.approved.txt

@@ -1,5 +1,6 @@
 ServiceControl Health: Error Database Index Errors
 ServiceControl Health: Error Database Index Lag
+ServiceControl Health: Error Database Search Engine
 ServiceControl Health: Message Ingestion Process
 ServiceControl Health: RavenDB dirty memory
 Storage space: ServiceControl database

src/ServiceControl.RavenDB/StartupChecks.cs

@@ -1,12 +1,46 @@
 namespace ServiceControl.RavenDB
 {
+    using System.Collections.Generic;
     using System.Reflection;
     using System.Threading;
+    using Microsoft.Extensions.Logging;
     using Raven.Client.Documents;
+    using Raven.Client.Documents.Indexes;
+    using Raven.Client.Documents.Operations.Indexes;
     using Raven.Client.ServerWide.Operations;
+    using ServiceControl.Infrastructure;
 
     public static class StartupChecks
     {
+        public static async Task WarnIfIndexesUseCorax(IDocumentStore store, string databaseName, CancellationToken cancellationToken = default)
+        {
+            // New databases are created with Lucene, existing databases keep whatever search engine they were created
+            // with as switching would trigger a full rebuild of all indexes. Let the operator know so they can plan the
+            // transition to Lucene themselves.
+            var coraxIndexes = await FindIndexesUsingCorax(store, databaseName, cancellationToken);
+
+            if (coraxIndexes.Length > 0)
+            {
+                Logger.LogWarning(CoraxIndexesMessage(coraxIndexes.Select(i => $"{databaseName}/{i}")));
+            }
+        }
+
+        public static string CoraxIndexesMessage(IEnumerable<string> indexes) =>
+            $"The following RavenDB index(es) use the Corax search engine: {string.Join(", ", indexes)}. " +
+            "Lucene indexes are smaller, use less memory and perform better for ServiceControl workloads, and are the default for new databases. " +
+            "Consider switching these indexes to Lucene. Note that switching triggers a full rebuild of the index: on very large databases this can take days depending on the available compute, " +
+            "and while the rebuild is running ingestion and indexing rates can be degraded. Plan the switch accordingly.";
+
+        public static async Task<string[]> FindIndexesUsingCorax(IDocumentStore store, string databaseName, CancellationToken cancellationToken = default)
+        {
+            var indexStats = await store.Maintenance.ForDatabase(databaseName).SendAsync(new GetIndexesStatisticsOperation(), cancellationToken);
+
+            return indexStats
+                .Where(i => i.SearchEngineType == SearchEngineType.Corax)
+                .Select(i => i.Name)
+                .ToArray();
+        }
+
         public static async Task EnsureServerVersion(IDocumentStore store, CancellationToken cancellationToken = default)
         {
             // RavenDB compatibility policy is that the major/minor version of the server must be
@@ -31,5 +65,7 @@ public static async Task EnsureServerVersion(IDocumentStore store, CancellationT
                 throw new Exception($"ServiceControl expects RavenDB Server version {clientProductVersion} or higher, but the server is using {serverProductVersion}.");
             }
         }
+
+        static readonly ILogger Logger = LoggerUtil.CreateStaticLogger(typeof(StartupChecks));
     }
 }