tests: ensure DB schema is setup before the host starts

Author: warwickschroeder

src/ServiceControl.AcceptanceTests/Recoverability/When_hosting_error_ingestion_only.cs

@@ -18,15 +18,13 @@ namespace ServiceControl.AcceptanceTests.Recoverability
     using NUnit.Framework;
     using Particular.ServiceControl.Hosting;
     using ServiceBus.Management.Infrastructure.Settings;
-    using ServiceControl.ExternalIntegrations;
     using ServiceControl.Hosting.Commands;
     using ServiceControl.Infrastructure;
     using ServiceControl.MessageFailures;
     using ServiceControl.Operations;
+    using ServiceControl.Persistence.DataMigration;
     using ServiceControl.Persistence.EFCore.DbContexts;
     using ServiceControl.Persistence.EFCore.Entities;
-    using ServiceControl.Persistence.EFCore.Infrastructure;
-    using ServiceControl.Recoverability;
     using ServiceControl.Transports;
 
     class When_hosting_error_ingestion_only : AcceptanceTest
@@ -72,9 +70,38 @@ public async Task Should_ingest_without_an_endpoint_and_without_the_single_owner
             "InternalCustomChecksHostedService",    // reports this node's ingestion health to the database
             "MetricsReporterHostedService",
             "HealthCheckPublisherHostedService",  // inert, no IHealthCheckPublisher is registered
-            "ExternalIntegrationRequestsDataStore"  // its drain is inert here, nothing calls Subscribe
+            "ExternalIntegrationRequestsDataStore",  // its drain is inert here, nothing calls Subscribe
+            "CheckpointTableIsReadable",            // reads one table and refuses a start against a schema older than the build
+            "RecordHostOpenedOnTarget",             // this node writes to the target, so the stamp belongs here, and it upserts one settings row
+            "FinishedCopyBeforeAnIngestionNodeOpens"  // keeps this node out of a database a copy has not finished filling
         ];
 
+        [Test]
+        public async Task Should_refuse_to_start_while_a_copy_into_the_database_is_unfinished()
+        {
+            var settings = await CreateSettings();
+
+            await new SetupCommand().Execute(new HostArguments([]), settings);
+
+            var host = ErrorIngestionOnlyCommand.BuildHost(settings);
+
+            try
+            {
+                await host.Services.GetRequiredService<IMigrationCheckpointStore>().Upsert(
+                    new MigrationCheckpoint(MigrationCategoryIds.KnownEndpoints, MigrationCategoryState.InProgress, null, 0, 0, null, null, null, null, null, null));
+
+                var exception = Assert.ThrowsAsync<Exception>(() => host.StartAsync());
+
+                Assert.That(exception.Message, Does.Contain("has not finished")
+                    .And.Contain(MigrationCategoryIds.KnownEndpoints)
+                    .And.Contain(MigrationSettings.AllowIncompleteExitKey));
+            }
+            finally
+            {
+                await host.DisposeAsync();
+            }
+        }
+
         [Test]
         public void Should_refuse_to_start_against_unsupported_storage()
         {

src/ServiceControl.Migration.AcceptanceTests/When_a_startup_check_fails.cs

@@ -9,9 +9,10 @@ namespace ServiceControl.Migration.AcceptanceTests;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
 using NUnit.Framework;
 using ServiceControl.Hosting.Commands;
-using ServiceControl.Migration.Checks;
 using ServiceControl.Persistence.DataMigration;
 using ServiceControl.Persistence.EFCore.Abstractions;
 
@@ -63,8 +64,19 @@ public async Task A_retry_history_depth_that_empties_the_migrated_table_is_refus
     [Test]
     public async Task A_target_whose_schema_migrations_were_never_applied_is_refused()
     {
-        // Double quoted so PostgreSQL keeps the capitals, which SQL Server also accepts under its default QUOTED_IDENTIFIER.
-        await QueryTarget(dbContext => dbContext.Database.ExecuteSqlRawAsync("""DELETE FROM "__EFMigrationsHistory" """));
+        await QueryTarget(async dbContext =>
+        {
+            // EF's own history repository, because it is the only thing that knows where the history table is once the persister is given a schema.
+            var history = dbContext.GetService<IHistoryRepository>();
+            var applied = await history.GetAppliedMigrationsAsync();
+
+            foreach (var row in applied)
+            {
+                await dbContext.Database.ExecuteSqlRawAsync(history.GetDeleteScript(row.MigrationId));
+            }
+
+            return applied.Count;
+        });
 
         var refusal = await RefusedStartup();
 

src/ServiceControl.Persistence.Tests.InMemory/PersistenceTestsContext.cs

@@ -24,6 +24,8 @@ public Task Setup(IHostApplicationBuilder hostBuilder)
         return Task.CompletedTask;
     }
 
+    public Task InstallSchema(IHost host) => Task.CompletedTask;
+
     public Task PostSetup(IHost host) => Task.CompletedTask;
 
     public Task TearDown() => Task.CompletedTask;

src/ServiceControl.Persistence.Tests.PostgreSql/PersistenceTestsContext.cs

@@ -53,7 +53,7 @@ public async Task Setup(IHostApplicationBuilder hostBuilder)
         hostBuilder.Services.AddSingleton<TimeProvider>(FakeTime);
     }
 
-    public async Task PostSetup(IHost host)
+    public async Task InstallSchema(IHost host)
     {
         this.host = host;
 
@@ -62,6 +62,8 @@ public async Task PostSetup(IHost host)
         await db.Database.MigrateAsync();
     }
 
+    public Task PostSetup(IHost host) => Task.CompletedTask;
+
     public async Task TearDown()
     {
         DeleteBodyStorage();

src/ServiceControl.Persistence.Tests.RavenDB/DataMigration/RavenMigrationSourceTests.cs

@@ -1,6 +1,7 @@
 namespace ServiceControl.Persistence.Tests.RavenDB.DataMigration;
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using NUnit.Framework;
@@ -92,15 +93,20 @@ public async Task Counts_a_category_with_nothing_in_it_as_zero()
     [Test]
     public async Task Reading_every_category_creates_no_index()
     {
-        await SeedEndpointSettings(2);
-        await MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "HOST01" });
+        await using var source = await OpenMigrationSource();
+
+        Assert.That(source.SupportedCategoryIds, Is.SubsetOf(Seeds.Keys), "A category read with nothing in it cannot show whether its reader builds an index, so a new reader needs its seed adding here.");
+
+        foreach (var categoryId in source.SupportedCategoryIds)
+        {
+            await Seeds[categoryId](this);
+        }
 
         var before = await IndexNames();
 
-        await using (var source = await OpenMigrationSource())
+        foreach (var categoryId in source.SupportedCategoryIds)
         {
-            await CollectBatches(source, KnownEndpointsCategory);
-            await CollectBatches(source, EndpointSettingsCategory);
+            await CollectBatches(source, MigrationCategoryRegistry.Find(categoryId));
         }
 
         var after = await IndexNames();
@@ -112,6 +118,14 @@ public async Task Reading_every_category_creates_no_index()
         });
     }
 
+    // Keyed by category id so a reader added without seed data fails the index test by name instead of passing
+    // over an empty collection.
+    static readonly Dictionary<string, Func<RavenMigrationSourceTests, Task>> Seeds = new()
+    {
+        [MigrationCategoryIds.EndpointSettings] = tests => tests.SeedEndpointSettings(2),
+        [MigrationCategoryIds.KnownEndpoints] = tests => tests.MonitoringDataStore.CreateIfNotExists(new EndpointDetails { Name = "Sales.Orders", HostId = Guid.NewGuid(), Host = "HOST01" })
+    };
+
     string DatabaseName => ((RavenPersisterSettings)PersistenceSettings).DatabaseName;
 
     async Task<string[]> IndexNames() =>
@@ -120,7 +134,6 @@ async Task<string[]> IndexNames() =>
     const int IndexNamePageSize = 1024;
 
     static readonly MigrationCategory EndpointSettingsCategory = MigrationCategoryRegistry.Find("EndpointSettings");
-    static readonly MigrationCategory KnownEndpointsCategory = MigrationCategoryRegistry.Find("KnownEndpoints");
 
     async Task SeedEndpointSettings(int count)
     {

src/ServiceControl.Persistence.Tests.RavenDB/PersistenceTestsContext.cs

@@ -50,6 +50,8 @@ public async Task Setup(IHostApplicationBuilder hostBuilder)
         persistence.AddInstaller(hostBuilder.Services);
     }
 
+    public Task InstallSchema(IHost host) => Task.CompletedTask;
+
     public async Task PostSetup(IHost host)
     {
         DocumentStore = await host.Services.GetRequiredService<IRavenDocumentStoreProvider>().GetDocumentStore();

src/ServiceControl.Persistence.Tests.SqlServer/EndpointSettingsKeyCollationTests.cs

@@ -5,6 +5,8 @@ namespace ServiceControl.Persistence.Tests;
 using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Storage;
 using Microsoft.Extensions.DependencyInjection;
 using NUnit.Framework;
 using ServiceControl.Operations;
@@ -22,19 +24,26 @@ public async Task The_key_columns_own_collation_decides_a_merge_when_the_databas
 
         using (var scope = ServiceProvider.CreateScope())
         {
-            var database = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>().Database;
+            var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>();
+            var database = dbContext.Database;
 
             databaseIgnoresCase = await database
                 .SqlQuery<int>($"SELECT CONVERT(int, DATABASEPROPERTYEX(DB_NAME(), 'ComparisonStyle')) & 1 AS [Value]")
                 .SingleAsync() == 1;
 
+            // Named from the model, because the table sits in the configured schema when the persister has one.
+            var entityType = dbContext.Model.FindEntityType(typeof(EndpointSettingsEntity));
+            var table = dbContext.GetService<ISqlGenerationHelper>().DelimitIdentifier(entityType.GetTableName(), entityType.GetSchema());
+
             // The column is given the opposite of the database default, because a test where the two agree cannot show which one decided.
-            await database.ExecuteSqlRawAsync("""
+            var recollate = $"""
                 DECLARE @columnCollation sysname = IIF(CONVERT(int, DATABASEPROPERTYEX(DB_NAME(), 'ComparisonStyle')) & 1 = 1, N'Latin1_General_CS_AS', N'Latin1_General_CI_AS');
-                ALTER TABLE [EndpointSettings] DROP CONSTRAINT [PK_EndpointSettings];
-                EXEC (N'ALTER TABLE [EndpointSettings] ALTER COLUMN [Name] nvarchar(450) COLLATE ' + @columnCollation + N' NOT NULL');
-                ALTER TABLE [EndpointSettings] ADD CONSTRAINT [PK_EndpointSettings] PRIMARY KEY ([Name]);
-                """);
+                ALTER TABLE {table} DROP CONSTRAINT [PK_EndpointSettings];
+                EXEC (N'ALTER TABLE {table} ALTER COLUMN [Name] nvarchar(450) COLLATE ' + @columnCollation + N' NOT NULL');
+                ALTER TABLE {table} ADD CONSTRAINT [PK_EndpointSettings] PRIMARY KEY ([Name]);
+                """;
+
+            await database.ExecuteSqlRawAsync(recollate);
         }
 
         foreach (var name in new[] { "Sales", "sales" })

src/ServiceControl.Persistence.Tests.SqlServer/PersistenceTestsContext.cs

@@ -52,7 +52,7 @@ public async Task Setup(IHostApplicationBuilder hostBuilder)
         hostBuilder.Services.AddSingleton<TimeProvider>(FakeTime);
     }
 
-    public async Task PostSetup(IHost host)
+    public async Task InstallSchema(IHost host)
     {
         this.host = host;
 
@@ -61,6 +61,8 @@ public async Task PostSetup(IHost host)
         await db.Database.MigrateAsync();
     }
 
+    public Task PostSetup(IHost host) => Task.CompletedTask;
+
     public async Task TearDown()
     {
         DeleteBodyStorage();

src/ServiceControl.Persistence.Tests/EFCore/Migration/MigrationTargetReadinessTests.cs

@@ -5,6 +5,8 @@ namespace ServiceControl.Persistence.Tests;
 using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using NUnit.Framework;
@@ -46,8 +48,13 @@ public async Task The_schema_check_refuses_a_database_whose_migrations_have_not_
         using var scope = ServiceProvider.CreateScope();
         var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>();
 
-        // Double quoted so PostgreSQL keeps the capitals, which SQL Server also accepts under its default QUOTED_IDENTIFIER.
-        await dbContext.Database.ExecuteSqlRawAsync("""DELETE FROM "__EFMigrationsHistory" """);
+        // EF's own history repository, because it is the only thing that knows where the history table is once the persister is given a schema.
+        var history = dbContext.GetService<IHistoryRepository>();
+
+        foreach (var row in await history.GetAppliedMigrationsAsync())
+        {
+            await dbContext.Database.ExecuteSqlRawAsync(history.GetDeleteScript(row.MigrationId));
+        }
 
         var exception = Assert.ThrowsAsync<Exception>(() => Readiness.ContributedChecks().OfType<SchemaIsCurrentCheck>().Single().Run());
 
@@ -123,7 +130,7 @@ public async Task The_target_answers_from_a_container_whose_hosted_services_have
 
         await context.Setup(hostBuilder);
         var host = hostBuilder.Build();
-        await context.PostSetup(host);
+        await context.InstallSchema(host);
 
         return (host, context);
     }

src/ServiceControl.Persistence.Tests/IPersistenceTestsContext.cs

@@ -10,6 +10,13 @@ public interface IPersistenceTestsContext
 {
     Task Setup(IHostApplicationBuilder hostBuilder);
 
+    /// <summary>
+    /// Puts the schema in place on the built host, before it is started, the way <c>--setup</c> does in
+    /// production. The EF Core persisters refuse to start against a database whose schema predates the
+    /// build, and that check runs before any hosted service, so migrating after the start is too late.
+    /// </summary>
+    Task InstallSchema(IHost host);
+
     Task PostSetup(IHost host);
 
     Task TearDown();