John/testing new workflow

.github/workflows/cloud-database-tests-cleanup.yml

@@ -0,0 +1,113 @@
+name: Cloud database tests cleanup
+on:
+  schedule:
+    - cron: '0 3 * * *'
+  workflow_dispatch:
+env:
+  DOTNET_NOLOGO: true
+defaults:
+  run:
+    shell: pwsh
+jobs:
+  azure:
+    name: Azure
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check for secrets
+        env:
+          SECRETS_AVAILABLE: ${{ secrets.SECRETS_AVAILABLE }}
+        run: exit $(If ($env:SECRETS_AVAILABLE -eq 'true') { 0 } Else { 1 })
+      - name: Azure login
+        uses: azure/login@v3.0.2
+        with:
+          creds: ${{ secrets.AZURE_CLOUD_TEST_CREDENTIALS }}
+      - name: Delete leaked resource groups
+        run: |
+          $cutoff = [DateTimeOffset]::UtcNow.AddHours(-4).ToUnixTimeSeconds()
+          $groups = az group list --tag sc-cloud-test=true --query '[].{name:name, created:tags.created}' --output json | ConvertFrom-Json
+
+          if (-not $groups) {
+            Write-Output 'Nothing tagged sc-cloud-test is left in this subscription.'
+            return
+          }
+
+          foreach ($group in $groups) {
+            if (-not $group.created) {
+              Write-Warning "Resource group $($group.name) has no created tag, so its age is unknown. Leaving it, delete it by hand."
+              continue
+            }
+
+            if ([long]$group.created -ge $cutoff) {
+              Write-Output "Leaving $($group.name), it belongs to a run that may still be going."
+              continue
+            }
+
+            Write-Output "Deleting $($group.name)"
+            try {
+              az group delete --name $group.name --yes --no-wait --only-show-errors
+            }
+            catch {
+              Write-Warning "Could not delete $($group.name): $($_.Exception.Message)"
+            }
+          }
+
+  aws:
+    name: AWS
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check for secrets
+        env:
+          SECRETS_AVAILABLE: ${{ secrets.SECRETS_AVAILABLE }}
+        run: exit $(If ($env:SECRETS_AVAILABLE -eq 'true') { 0 } Else { 1 })
+      - name: Setup AWS environment variables
+        run: |
+          echo "AWS_REGION=${{ secrets.AWS_REGION }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+          echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+          echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+      - name: Delete leaked RDS resources and security groups
+        run: |
+          $cutoff = [DateTimeOffset]::UtcNow.AddHours(-4).ToUnixTimeSeconds()
+          $tagged = aws resourcegroupstaggingapi get-resources --tag-filters Key=sc-cloud-test,Values=true --output json | ConvertFrom-Json
+
+          $stale = @()
+          foreach ($resource in $tagged.ResourceTagMappingList) {
+            $created = ($resource.Tags | Where-Object { $_.Key -eq 'created' }).Value
+
+            if (-not $created) {
+              Write-Warning "$($resource.ResourceARN) has no created tag, so its age is unknown. Leaving it, delete it by hand."
+              continue
+            }
+
+            if ([long]$created -ge $cutoff) {
+              Write-Output "Leaving $($resource.ResourceARN), it belongs to a run that may still be going."
+              continue
+            }
+
+            $stale += $resource.ResourceARN
+          }
+
+          if (-not $stale) {
+            Write-Output 'Nothing stale is tagged sc-cloud-test in this account.'
+            return
+          }
+
+          # Instances first, then the clusters that hold them, then the security groups the instances
+          # were using. A security group still in use refuses to go, and is picked up by the next run.
+          $order = @(':db:', ':cluster:', 'security-group/')
+
+          foreach ($pattern in $order) {
+            foreach ($arn in $stale | Where-Object { $_ -like "*$pattern*" }) {
+              $identifier = ($arn -split '[:/]')[-1]
+              Write-Output "Deleting $arn"
+              try {
+                switch -Wildcard ($arn) {
+                  '*:db:*' { aws rds delete-db-instance --db-instance-identifier $identifier --skip-final-snapshot --delete-automated-backups --no-cli-pager --output none; break }
+                  '*:cluster:*' { aws rds delete-db-cluster --db-cluster-identifier $identifier --skip-final-snapshot --no-cli-pager --output none; break }
+                  '*security-group/*' { aws ec2 delete-security-group --group-id $identifier --output none; break }
+                }
+              }
+              catch {
+                Write-Warning "Could not delete ${arn}: $($_.Exception.Message)"
+              }
+            }
+          }

.github/workflows/cloud-database-tests.yml

@@ -0,0 +1,131 @@
+name: Cloud database tests
+on:
+  workflow_dispatch:
+    inputs:
+      target:
+        description: Which managed database to test against
+        required: true
+        default: all
+        type: choice
+        options:
+          - all
+          - azure-sql
+          - azure-postgresql
+          - aurora-postgresql
+          - rds-sqlserver
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+-*'
+env:
+  DOTNET_NOLOGO: true
+defaults:
+  run:
+    shell: pwsh
+concurrency:
+  # Deliberately not cancel-in-progress: cancelling a run mid-flight risks skipping the teardown
+  # step and leaving billable cloud resources behind.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+jobs:
+  plan:
+    name: Plan
+    runs-on: ubuntu-latest
+    outputs:
+      targets: ${{ steps.select.outputs.targets }}
+    steps:
+      - name: Select targets
+        id: select
+        run: |
+          $all = @(
+            @{ target = 'azure-sql';          provider = 'SqlServer'  }
+            @{ target = 'azure-postgresql';   provider = 'PostgreSql' }
+            @{ target = 'aurora-postgresql';  provider = 'PostgreSql' }
+            @{ target = 'rds-sqlserver';      provider = 'SqlServer'  }
+          )
+
+          # A tag push has no input, and runs everything.
+          $requested = '${{ inputs.target }}'
+          $targets = if (-not $requested -or $requested -eq 'all') { $all } else { $all | Where-Object { $_.target -eq $requested } }
+
+          if (-not $targets) {
+            throw "No cloud target named '$requested'."
+          }
+
+          $targets | ForEach-Object { Write-Output "Will test against $($_.target)" }
+
+          $matrix = @{ include = @($targets) }
+          "targets=$(ConvertTo-Json -InputObject $matrix -Compress -Depth 4)" | Out-File -FilePath $Env:GITHUB_OUTPUT -Encoding utf8 -Append
+
+  test:
+    name: ${{ matrix.target }}
+    needs: plan
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.plan.outputs.targets) }}
+    # The two suites that exercise the persister. Named outright rather than selected by test
+    # category, because the category also carries the transport tests, which test the queue
+    # transport against its own connection string and have no business running here.
+    env:
+      PERSISTENCE_PROJECT: src/ServiceControl.Persistence.Tests.${{ matrix.provider }}/ServiceControl.Persistence.Tests.${{ matrix.provider }}.csproj
+      ACCEPTANCE_PROJECT: src/ServiceControl.AcceptanceTests.${{ matrix.provider }}/ServiceControl.AcceptanceTests.${{ matrix.provider }}.csproj
+    steps:
+      - name: Check for secrets
+        env:
+          SECRETS_AVAILABLE: ${{ secrets.SECRETS_AVAILABLE }}
+        run: exit $(If ($env:SECRETS_AVAILABLE -eq 'true') { 0 } Else { 1 })
+      - name: Checkout
+        uses: actions/checkout@v7.0.1
+        with:
+          fetch-depth: 0
+      - name: Setup .NET SDK
+        uses: actions/setup-dotnet@v6.0.0
+        with:
+          global-json-file: global.json
+      - name: Determine runtime version
+        run: |
+          # Read settings from Custom.Build.props
+          [xml]$xml = Get-Content ./src/Custom.Build.props
+          $runtimeVersion = $xml.selectNodes('/Project/PropertyGroup/RuntimeFrameworkVersion').InnerText
+          if (-not ($runtimeVersion)) {
+            throw "Missing RuntimeFrameworkVersion setting in Custom.Build.props"
+          }
+          echo "RuntimeVersion=$runtimeVersion" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+      - name: Install .NET Runtime
+        shell: bash
+        run: |
+          ./tools/dotnet-install.sh --skip-non-versioned-files --install-dir /usr/share/dotnet --runtime dotnet --version ${{ env.RuntimeVersion }}
+          ./tools/dotnet-install.sh --skip-non-versioned-files --install-dir /usr/share/dotnet --runtime aspnetcore --version ${{ env.RuntimeVersion }}
+      - name: Build
+        id: build
+        background: true
+        run: |
+          dotnet build $Env:PERSISTENCE_PROJECT --configuration Release
+          dotnet build $Env:ACCEPTANCE_PROJECT --configuration Release
+      - name: Azure login
+        uses: azure/login@v3.0.2
+        if: startsWith(matrix.target, 'azure-')
+        with:
+          creds: ${{ secrets.AZURE_CLOUD_TEST_CREDENTIALS }}
+      - name: Setup AWS environment variables
+        if: startsWith(matrix.target, 'aurora-') || startsWith(matrix.target, 'rds-')
+        run: |
+          echo "AWS_REGION=${{ secrets.AWS_REGION }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+          echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+          echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+      # Unlike ci.yml, which holds its cloud resources back until the build is done, provisioning
+      # starts first: it is the long pole here at anywhere from 3 to 25 minutes, and the teardown
+      # step runs on failure regardless.
+      - name: Provision database
+        run: ./tools/cloud/${{ matrix.target }}.ps1 -Action Provision -Name sc-ct-${{ github.run_id }}
+      - name: Wait for build
+        wait: build
+      - name: Run tests
+        run: ./tools/run-tests.ps1 -Projects "$Env:PERSISTENCE_PROJECT`n$Env:ACCEPTANCE_PROJECT" -MaxParallel 2
+        env:
+          ServiceControl_TESTS_FILTER: ${{ matrix.provider }}
+          PARTICULARSOFTWARE_LICENSE: ${{ secrets.LICENSETEXT }}
+      - name: Tear down database
+        if: always()
+        run: ./tools/cloud/${{ matrix.target }}.ps1 -Action Teardown -Name sc-ct-${{ github.run_id }}

docs/testing-persistence.md

@@ -8,6 +8,15 @@ ServiceControl supports multiple persistence types
 
 All persistence test projects can be run with `dotnet test` against the corresponding test project in `src/`.
 
+## Test isolation on the SQL persisters
+
+The SQL Server and PostgreSQL suites give each test its own **schema** in one shared database, named `sc_test_<guid>` for persistence tests and `sc_at_<guid>` for acceptance tests, and drop it on teardown. This uses the same `Database/Schema` setting that is offered to customers, so every run exercises that feature.
+
+Two consequences are worth knowing:
+
+* The connection string is used exactly as it is given, so **the database it names must already exist**. The test containers create one; a server you point the environment variable at will not.
+* A test failure that leaves a process behind can leave a schema behind with it. `SELECT nspname FROM pg_namespace WHERE nspname LIKE 'sc\_%'` and `SELECT name FROM sys.schemas WHERE name LIKE 'sc[_]%'` will find any strays.
+
 ## RavenDB
 
 RavenDB persistence tests start an embedded RavenDB instance for the duration of the test run.
@@ -23,7 +32,7 @@ Build that image locally before running SQL Server persistence tests:
 docker buildx build --platform=linux/amd64 --tag particular/servicecontrol-testing-sqlserver:latest ./src/Scripts/Docker/servicecontrol-testing-sqlserver
 ```
 
-If you want to use an existing SQL Server instance instead of a test container, set the `ServiceControl_Persistence_SqlServer_ConnectionString` environment variable to a valid SQL Server connection string.
+If you want to use an existing SQL Server instance instead of a test container, set the `ServiceControl_Persistence_SqlServer_ConnectionString` environment variable to a valid SQL Server connection string. It must name a database that exists, not `master`, because the tests create their schemas in whatever database it points at. The test container creates a `ServiceControlTests` database for this.
 
 ## PostgreSQL
 
@@ -35,4 +44,4 @@ If you want to use an existing PostgreSQL instance instead of a test container,
 ServiceControl_Persistence_PostgreSql_ConnectionString
 ```
 
-to a valid PostgreSQL connection string.
+to a valid PostgreSQL connection string. It must name a database that exists, because the tests create their schemas in whatever database it points at. The test container creates a `servicecontroltests` database for this, so that test schemas do not end up in the `postgres` maintenance database.

src/ServiceControl.AcceptanceTests.PostgreSql/AcceptanceTestStorageConfiguration.cs

@@ -1,10 +1,10 @@
 namespace ServiceControl.AcceptanceTests.PostgreSql;
 
 using System;
+using System.Collections.Concurrent;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
-using Npgsql;
 using ServiceBus.Management.Infrastructure.Settings;
 using ServiceControl.AcceptanceTests.TestSupport;
 using ServiceControl.Persistence.EFCore.Abstractions;
@@ -17,53 +17,39 @@ public class AcceptanceTestStorageConfiguration : IAcceptanceTestStorageConfigur
 
     public async Task CustomizeSettings(Settings settings, CancellationToken cancellationToken = default)
     {
-        databaseName = $"sc_at_{Guid.NewGuid():n}";
-        serverConnectionString = await PostgreSqlSharedContainer.GetConnectionStringAsync(cancellationToken).ConfigureAwait(false);
+        var schema = $"sc_at_{Guid.NewGuid():n}";
+        var bodyStoragePath = Directory.CreateTempSubdirectory("sc_at_bodies_").FullName;
 
-        var connectionStringBuilder = new NpgsqlConnectionStringBuilder(serverConnectionString)
-        {
-            Database = databaseName
-        };
+        connectionString = await PostgreSqlSharedContainer.GetConnectionStringAsync(cancellationToken).ConfigureAwait(false);
+        await TestSchema.Create(connectionString, schema, cancellationToken).ConfigureAwait(false);
 
-        bodyStoragePath = Directory.CreateTempSubdirectory("sc_at_bodies_").FullName;
+        // A test that runs more than one scenario comes back through here, and the runner cleans up
+        // after each one. Recording everything created, rather than keeping only the most recent,
+        // is what stops the earlier schema being stranded in the shared database.
+        schemas.Add(schema);
+        bodyStoragePaths.Add(bodyStoragePath);
 
         settings.PersisterSpecificSettings = new PostgreSqlPersisterSettings
         {
-            ConnectionString = connectionStringBuilder.ConnectionString,
+            ConnectionString = connectionString,
+            Schema = schema,
             ErrorRetentionPeriod = TimeSpan.FromDays(10),
             BodyStorage = new FileSystemBodyStorageSettings { StoragePath = bodyStoragePath }
         };
     }
 
     public async Task Cleanup(CancellationToken cancellationToken = default)
     {
-        if (Interlocked.Exchange(ref cleanupStarted, 1) != 0)
-        {
-            return;
-        }
-
         try
         {
-            if (serverConnectionString == null || databaseName == null)
+            while (schemas.TryTake(out var schema))
             {
-                return;
-            }
-
-            var connection = new NpgsqlConnection(serverConnectionString);
-            await using (connection.ConfigureAwait(false))
-            {
-                await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
-                var command = connection.CreateCommand();
-                await using (command.ConfigureAwait(false))
-                {
-                    command.CommandText = $"DROP DATABASE IF EXISTS \"{databaseName}\" WITH (FORCE)";
-                    await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
-                }
+                await TestSchema.Drop(connectionString, schema, cancellationToken).ConfigureAwait(false);
             }
         }
         finally
         {
-            if (bodyStoragePath != null)
+            while (bodyStoragePaths.TryTake(out var bodyStoragePath))
             {
                 try
                 {
@@ -91,8 +77,7 @@ public void Dispose()
         }
     }
 
-    string serverConnectionString;
-    string databaseName;
-    string bodyStoragePath;
-    int cleanupStarted;
-}
+    readonly ConcurrentBag<string> schemas = [];
+    readonly ConcurrentBag<string> bodyStoragePaths = [];
+    string connectionString;
+}

src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj

@@ -32,6 +32,7 @@
   <ItemGroup>
     <Compile Include="..\ServiceControl.AcceptanceTests\**\*.cs" LinkBase="Shared" />
     <Compile Include="..\ServiceControl.Persistence.Tests.PostgreSql\PostgreSqlSharedContainer.cs" />
+    <Compile Include="..\ServiceControl.Persistence.Tests.PostgreSql\TestSchema.cs" />
     <Compile Include="..\ServiceControl.Persistence.Tests.PostgreSql\StopSharedPostgreSql.cs" />
     <Compile Include="..\ServiceControl.UnitTests\NUnitParallelRunnerSettings.cs" />