Add acceptance tests for SQL Server and PostgreSQL persistence

.github/workflows/ci.yml

@@ -19,7 +19,7 @@ jobs:
       matrix:
         os: [windows-latest, ubuntu-latest]
         # Categories are declared by the <TestCategory> property in each test project.
-        test-category: [ DefaultCore, DefaultAudit, DefaultMonitoring, SqlServer, SqlServerPersistence, AzureServiceBus, RabbitMQClassicConventional, RabbitMQClassicDirect, RabbitMQQuorumConventional, RabbitMQQuorumDirect, AzureStorageQueues, MSMQ, SQS, PrimaryRavenAcceptance, PrimaryRavenPersistence, PostgreSql, PostgreSqlPersistence, IBMMQ ]
+        test-category: [ DefaultCore, DefaultAudit, DefaultMonitoring, SqlServer, SqlServerPersistence, AzureServiceBus, RabbitMQClassicConventional, RabbitMQClassicDirect, RabbitMQQuorumConventional, RabbitMQQuorumDirect, AzureStorageQueues, MSMQ, SQS, PrimaryRavenAcceptance, PrimarySqlServerAcceptance, PrimaryPostgreSqlAcceptance, PrimaryRavenPersistence, PostgreSql, PostgreSqlPersistence, IBMMQ ]
         include:
           - os: windows-latest
             os-name: Windows
@@ -56,7 +56,7 @@ jobs:
       # run in on Windows. A no-op on Linux, but still run there so those actions see the same
       # environment variables on both runners.
       - name: Setup WSL
-        if: startsWith(matrix.test-category, 'RabbitMQ') || contains(fromJSON('["SqlServer", "SqlServerPersistence", "PostgreSql", "PostgreSqlPersistence", "IBMMQ"]'), matrix.test-category)
+        if: startsWith(matrix.test-category, 'RabbitMQ') || contains(fromJSON('["SqlServer", "SqlServerPersistence", "PrimarySqlServerAcceptance", "PostgreSql", "PostgreSqlPersistence", "PrimaryPostgreSqlAcceptance", "IBMMQ"]'), matrix.test-category)
         uses: Particular/setup-wsl-action@v1.2.0
         with:
           # The action defaults to 4GB. The runner has 16GB and the build runs concurrently with the
@@ -88,7 +88,7 @@ jobs:
           catalog: nservicebus
       - name: Setup SQL Server persistence
         uses: Particular/install-sql-server-action@v3.0.0
-        if: matrix.test-category == 'SqlServerPersistence'
+        if: matrix.test-category == 'SqlServerPersistence' || matrix.test-category == 'PrimarySqlServerAcceptance'
         with:
           connection-string-env-var: ServiceControl_Persistence_SqlServer_ConnectionString
           catalog: ServiceControl
@@ -102,7 +102,7 @@ jobs:
           registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Setup PostgreSQL persistence
         uses: Particular/setup-postgres-action@v3.0.0
-        if: matrix.test-category == 'PostgreSqlPersistence'
+        if: matrix.test-category == 'PostgreSqlPersistence' || matrix.test-category == 'PrimaryPostgreSqlAcceptance'
         with:
           connection-string-name: ServiceControl_Persistence_PostgreSql_ConnectionString
           registry-username: ${{ secrets.DOCKERHUB_USERNAME }}

src/ServiceControl.AcceptanceTests.PostgreSql/AcceptanceTestStorageConfiguration.cs

@@ -0,0 +1,98 @@
+namespace ServiceControl.AcceptanceTests.PostgreSql;
+
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Npgsql;
+using ServiceBus.Management.Infrastructure.Settings;
+using ServiceControl.AcceptanceTests.TestSupport;
+using ServiceControl.Persistence.EFCore.Abstractions;
+using ServiceControl.Persistence.EFCore.PostgreSql;
+using ServiceControl.Persistence.Tests;
+
+public class AcceptanceTestStorageConfiguration : IAcceptanceTestStorageConfiguration
+{
+    public string PersistenceType { get; } = "PostgreSQL";
+
+    public async Task CustomizeSettings(Settings settings, CancellationToken cancellationToken = default)
+    {
+        databaseName = $"sc_at_{Guid.NewGuid():n}";
+        serverConnectionString = await PostgreSqlSharedContainer.GetConnectionStringAsync(cancellationToken).ConfigureAwait(false);
+
+        var connectionStringBuilder = new NpgsqlConnectionStringBuilder(serverConnectionString)
+        {
+            Database = databaseName
+        };
+
+        bodyStoragePath = Directory.CreateTempSubdirectory("sc_at_bodies_").FullName;
+
+        settings.PersisterSpecificSettings = new PostgreSqlPersisterSettings
+        {
+            ConnectionString = connectionStringBuilder.ConnectionString,
+            ErrorRetentionPeriod = TimeSpan.FromDays(10),
+            BodyStorage = new FileSystemBodyStorageSettings { StoragePath = bodyStoragePath }
+        };
+    }
+
+    public async Task Cleanup(CancellationToken cancellationToken = default)
+    {
+        if (Interlocked.Exchange(ref cleanupStarted, 1) != 0)
+        {
+            return;
+        }
+
+        try
+        {
+            if (serverConnectionString == null || databaseName == null)
+            {
+                return;
+            }
+
+            var connection = new NpgsqlConnection(serverConnectionString);
+            await using (connection.ConfigureAwait(false))
+            {
+                await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
+                var command = connection.CreateCommand();
+                await using (command.ConfigureAwait(false))
+                {
+                    command.CommandText = $"DROP DATABASE IF EXISTS \"{databaseName}\" WITH (FORCE)";
+                    await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+                }
+            }
+        }
+        finally
+        {
+            if (bodyStoragePath != null)
+            {
+                try
+                {
+                    Directory.Delete(bodyStoragePath, recursive: true);
+                }
+                catch (DirectoryNotFoundException)
+                {
+                }
+            }
+        }
+    }
+
+    public Task<IDisposable> UseDatabaseLifecycleLock(CancellationToken cancellationToken = default)
+    {
+        cancellationToken.ThrowIfCancellationRequested();
+        return Task.FromResult<IDisposable>(NoOpDisposable.Instance);
+    }
+
+    sealed class NoOpDisposable : IDisposable
+    {
+        public static readonly NoOpDisposable Instance = new();
+
+        public void Dispose()
+        {
+        }
+    }
+
+    string serverConnectionString;
+    string databaseName;
+    string bodyStoragePath;
+    int cleanupStarted;
+}

src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj

@@ -0,0 +1,75 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <TestCategory>PrimaryPostgreSqlAcceptance</TestCategory>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Using Include="ServiceControl.AcceptanceTests.PostgreSql" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ServiceControl.AcceptanceTesting\ServiceControl.AcceptanceTesting.csproj" />
+    <ProjectReference Include="..\ServiceControl.Persistence.EFCore.PostgreSql\ServiceControl.Persistence.EFCore.PostgreSql.csproj" />
+    <ProjectReference Include="..\ServiceControl.Transports.Learning\ServiceControl.Transports.Learning.csproj" />
+    <ProjectReference Include="..\ServiceControl\ServiceControl.csproj" />
+    <ProjectReference Include="..\TestHelper\TestHelper.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="GitHubActionsTestLogger" />
+    <PackageReference Include="Microsoft.AspNetCore.Mvc.Testing" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="NServiceBus.Heartbeat" />
+    <PackageReference Include="NServiceBus.SagaAudit" />
+    <PackageReference Include="NUnit" />
+    <PackageReference Include="NUnit.Analyzers" />
+    <PackageReference Include="NUnit3TestAdapter" />
+    <PackageReference Include="Testcontainers.PostgreSql" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Include="..\ServiceControl.AcceptanceTests\**\*.cs" LinkBase="Shared" />
+    <Compile Include="..\ServiceControl.Persistence.Tests.PostgreSql\PostgreSqlSharedContainer.cs" />
+    <Compile Include="..\ServiceControl.Persistence.Tests.PostgreSql\StopSharedPostgreSql.cs" />
+    <Compile Include="..\ServiceControl.UnitTests\NUnitParallelRunnerSettings.cs" />
+
+    <!-- External integration event dispatch is not implemented by EF persistence. -->
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Monitoring\ExternalIntegration\When_a_custom_check_fails.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Monitoring\ExternalIntegration\When_a_custom_check_succeeds.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Monitoring\ExternalIntegration\When_heartbeat_is_restored.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Monitoring\ExternalIntegration\When_heartbeat_loss_is_detected.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\ExternalIntegration\When_a_failed_edit_is_resolved_by_retry.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\ExternalIntegration\When_a_failed_message_is_archived.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\ExternalIntegration\When_a_failed_message_is_resolved_by_retry.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\ExternalIntegration\When_a_failed_message_is_resolved_manually.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\ExternalIntegration\When_a_failed_message_is_unarchived.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\ExternalIntegration\When_a_failed_msg_is_resolved_by_edit.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\ExternalIntegration\When_a_group_is_archived.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\ExternalIntegration\When_a_message_has_failed_detected.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\ExternalIntegration\When_a_reedit_solves_a_failed_msg.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\ExternalIntegration\When_encountered_an_error.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\When_edited_message_fails_to_process.cs" />
+
+    <!-- EF body retrieval does not distinguish an empty body from unavailable content. -->
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\MessageFailures\When_a_retry_for_a_empty_body_message_is_successful.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\When_a_message_is_retried.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\When_a_message_is_retried_with_a_replyTo_header.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\When_a_native_integration_message_is_retried.cs" />
+
+    <!-- EF persistence retains only the latest processing attempt. -->
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\MessageFailures\When_errors_with_same_uniqueid_are_imported.cs" />
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\MessageFailures\When_a_messages_fails_multiple_times.cs" />
+
+    <!-- The EF custom-check query does not provide an ETag. -->
+    <Compile Remove="..\ServiceControl.AcceptanceTests\WebApi\When_a_request_is_repeated_with_its_etag.cs" />
+
+    <!-- The PostgreSQL event-log query does not provide an ETag. -->
+    <Compile Remove="..\ServiceControl.AcceptanceTests\EventLogs\When_the_event_log_is_polled_with_an_etag.cs" />
+
+    <!-- The EF DateTime round trip changes the offset expected by this scenario. -->
+    <Compile Remove="..\ServiceControl.AcceptanceTests\Recoverability\MessageFailures\When_ingesting_failed_message_with_missing_headers.cs" />
+  </ItemGroup>
+
+</Project>

src/ServiceControl.AcceptanceTests.RavenDB/AcceptanceTestStorageConfiguration.cs

@@ -5,18 +5,19 @@
 using System.Threading;
 using System.Threading.Tasks;
 using ServiceBus.Management.Infrastructure.Settings;
+using ServiceControl.AcceptanceTests.TestSupport;
 using ServiceControl.Persistence.Tests;
 using ServiceControl.RavenDB;
 
-public class AcceptanceTestStorageConfiguration
+public class AcceptanceTestStorageConfiguration : IAcceptanceTestStorageConfiguration
 {
     public string PersistenceType { get; } = "RavenDB";
 
 
-    public async Task CustomizeSettings(Settings settings)
+    public async Task CustomizeSettings(Settings settings, CancellationToken cancellationToken = default)
     {
         databaseName = Guid.NewGuid().ToString("n");
-        databaseInstance = await SharedEmbeddedServer.GetInstance();
+        databaseInstance = await SharedEmbeddedServer.GetInstance(cancellationToken);
 
         settings.PersisterSpecificSettings = new RavenPersisterSettings
         {
@@ -26,21 +27,21 @@ public async Task CustomizeSettings(Settings settings)
         };
     }
 
-    public async Task Cleanup()
+    public async Task Cleanup(CancellationToken cancellationToken = default)
     {
         if (databaseInstance == null)
         {
             return;
         }
-        using var _ = await UseDatabaseLifecycleLock();
+        using var _ = await UseDatabaseLifecycleLock(cancellationToken);
         await databaseInstance.DeleteDatabase(databaseName);
     }
 
     /// <summary>
     /// The shared server cannot perform database lifecycle operations in parallel, take this lock when you
     /// need to do one of these operations in a test.
     /// </summary>
-    public static async Task<IDisposable> UseDatabaseLifecycleLock(CancellationToken cancellationToken = default)
+    public async Task<IDisposable> UseDatabaseLifecycleLock(CancellationToken cancellationToken = default)
     {
         await databaseLifecycleLock.WaitAsync(cancellationToken);
         return Disposable.Create(() => databaseLifecycleLock.Release());

src/ServiceControl.AcceptanceTests/Monitoring/CustomChecks/When_critical_storage_threshold_reached.cs → src/ServiceControl.AcceptanceTests.RavenDB/Monitoring/CustomChecks/When_critical_storage_threshold_reached.cs

@@ -1,4 +1,4 @@
-namespace ServiceControl.AcceptanceTests.Monitoring.CustomChecks
+namespace ServiceControl.AcceptanceTests.RavenDB.Monitoring.CustomChecks
 {
     using System;
     using System.Linq;
@@ -97,4 +97,4 @@ public class MyMessageHandler : IHandleMessages<MyMessage>
 
         public class MyMessage : ICommand;
     }
-}
+}

src/ServiceControl.AcceptanceTests.RavenDB/ServiceControl.AcceptanceTests.RavenDB.csproj

@@ -5,6 +5,10 @@
     <TestCategory>PrimaryRavenAcceptance</TestCategory>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Using Include="ServiceControl.AcceptanceTests.RavenDB" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\ServiceControl.AcceptanceTesting\ServiceControl.AcceptanceTesting.csproj" />
     <ProjectReference Include="..\ServiceControl.Persistence.RavenDB\ServiceControl.Persistence.RavenDB.csproj" />

src/ServiceControl.AcceptanceTests.SqlServer/AcceptanceTestStorageConfiguration.cs

@@ -0,0 +1,104 @@
+namespace ServiceControl.AcceptanceTests.SqlServer;
+
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Data.SqlClient;
+using ServiceBus.Management.Infrastructure.Settings;
+using ServiceControl.AcceptanceTests.TestSupport;
+using ServiceControl.Persistence.EFCore.Abstractions;
+using ServiceControl.Persistence.EFCore.SqlServer;
+using ServiceControl.Persistence.Tests;
+
+public class AcceptanceTestStorageConfiguration : IAcceptanceTestStorageConfiguration
+{
+    public string PersistenceType { get; } = "SQLServer";
+
+    public async Task CustomizeSettings(Settings settings, CancellationToken cancellationToken = default)
+    {
+        databaseName = $"sc_at_{Guid.NewGuid():n}";
+        serverConnectionString = await SqlServerSharedContainer.GetConnectionStringAsync(cancellationToken).ConfigureAwait(false);
+
+        var connectionStringBuilder = new SqlConnectionStringBuilder(serverConnectionString)
+        {
+            InitialCatalog = databaseName
+        };
+
+        bodyStoragePath = Directory.CreateTempSubdirectory("sc_at_bodies_").FullName;
+
+        settings.PersisterSpecificSettings = new SqlServerPersisterSettings
+        {
+            ConnectionString = connectionStringBuilder.ConnectionString,
+            ErrorRetentionPeriod = TimeSpan.FromDays(10),
+            BodyStorage = new FileSystemBodyStorageSettings { StoragePath = bodyStoragePath }
+        };
+    }
+
+    public async Task Cleanup(CancellationToken cancellationToken = default)
+    {
+        if (Interlocked.Exchange(ref cleanupStarted, 1) != 0)
+        {
+            return;
+        }
+
+        try
+        {
+            if (serverConnectionString == null || databaseName == null)
+            {
+                return;
+            }
+
+            var connection = new SqlConnection(serverConnectionString);
+            await using (connection.ConfigureAwait(false))
+            {
+                await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
+                var command = connection.CreateCommand();
+                await using (command.ConfigureAwait(false))
+                {
+                    command.CommandText = $"""
+                        IF DB_ID('{databaseName}') IS NOT NULL
+                        BEGIN
+                            ALTER DATABASE [{databaseName}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE;
+                            DROP DATABASE [{databaseName}];
+                        END
+                        """;
+                    await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
+                }
+            }
+        }
+        finally
+        {
+            if (bodyStoragePath != null)
+            {
+                try
+                {
+                    Directory.Delete(bodyStoragePath, recursive: true);
+                }
+                catch (DirectoryNotFoundException)
+                {
+                }
+            }
+        }
+    }
+
+    public Task<IDisposable> UseDatabaseLifecycleLock(CancellationToken cancellationToken = default)
+    {
+        cancellationToken.ThrowIfCancellationRequested();
+        return Task.FromResult<IDisposable>(NoOpDisposable.Instance);
+    }
+
+    sealed class NoOpDisposable : IDisposable
+    {
+        public static readonly NoOpDisposable Instance = new();
+
+        public void Dispose()
+        {
+        }
+    }
+
+    string serverConnectionString;
+    string databaseName;
+    string bodyStoragePath;
+    int cleanupStarted;
+}