Optimize CI by categorizing tests and parallelizing builds even more

.github/workflows/ci.yml

@@ -18,7 +18,8 @@ jobs:
     strategy:
       matrix:
         os: [windows-latest, ubuntu-latest]
-        test-category: [ Default, SqlServer, SqlServerPersistence, AzureServiceBus, RabbitMQ, AzureStorageQueues, MSMQ, SQS, PrimaryRavenAcceptance, PrimaryRavenPersistence, PostgreSQL, PostgreSQLPersistence, IBMMQ ]
+        # Categories are declared by the <TestCategory> property in each test project.
+        test-category: [ DefaultCore, DefaultAudit, DefaultMonitoring, SqlServer, SqlServerPersistence, AzureServiceBus, RabbitMQClassicConventional, RabbitMQClassicDirect, RabbitMQQuorumConventional, RabbitMQQuorumDirect, AzureStorageQueues, MSMQ, SQS, PrimaryRavenAcceptance, PrimaryRavenPersistence, PostgreSql, PostgreSqlPersistence, IBMMQ ]
         include:
           - os: windows-latest
             os-name: Windows
@@ -27,8 +28,6 @@ jobs:
         exclude:
           - os: ubuntu-latest
             test-category: MSMQ
-          - os: windows-latest
-            test-category: IBMMQ
       fail-fast: false
     steps:
       - name: Check for secrets
@@ -43,34 +42,31 @@ jobs:
         uses: actions/setup-dotnet@v6.0.0
         with:
           global-json-file: global.json
-      - parallel:
-        - name: Setup WSL
-          uses: Particular/setup-wsl-action@v1.1.0
-        - name: Download RavenDB Server
-          run: ./tools/download-ravendb-server.ps1
+      - name: Select test projects
+        id: select
+        run: ./tools/select-test-projects.ps1 -Category ${{ matrix.test-category }}
+      - name: Download RavenDB Server
+        if: matrix.test-category == 'DefaultCore'
+        run: ./tools/download-ravendb-server.ps1
       - name: Build
-        run: dotnet build src --configuration Release -graph
-      - name: Zip PowerShell module
-        run: |
-          New-Item assets\PowerShellModules -ItemType Directory
-          Compress-Archive -Path deploy\PowerShellModules\Particular.ServiceControl.Management\* -DestinationPath assets\PowerShellModules\Particular.ServiceControl.Management.zip
-      - name: Upload assets
-        uses: actions/upload-artifact@v7.0.1
-        if: matrix.test-category == 'Default'
+        id: build
+        background: true
+        run: dotnet build tests.proj --configuration Release -graph
+      # Provisions the WSL2 Docker host that the SQL Server, PostgreSQL, RabbitMQ and IBM MQ containers
+      # run in on Windows. A no-op on Linux, but still run there so those actions see the same
+      # environment variables on both runners.
+      - name: Setup WSL
+        if: startsWith(matrix.test-category, 'RabbitMQ') || contains(fromJSON('["SqlServer", "SqlServerPersistence", "PostgreSql", "PostgreSqlPersistence", "IBMMQ"]'), matrix.test-category)
+        uses: Particular/setup-wsl-action@v1.2.0
         with:
-          name: ${{ matrix.os-name }}-assets
-          path: |
-            nugets/
-            zip/
-            assets/
-          retention-days: 1
-      - name: Smoke test PowerShell module import
-        if: matrix.os-name == 'Windows'
-        run: Import-Module ./deploy/PowerShellModules/Particular.ServiceControl.Management
+          # The action defaults to 4GB. The runner has 16GB and the build runs concurrently with the
+          # container starting up, so give the VM real headroom.
+          memory: 8GB
 
       # there is an issue with az cli and python 3.14, so for now we need to pin it
       # once the issue is resolved it should be able to be re-floated
-      # https://github.com/Azure/azure-cli/issues/32980
+      # https://github.com/Azure/azure-cli/issues/32980. 
+      # This can be removed once https://github.com/Azure/azure-cli/issues/32869 is supported.
       - name: Set Python 3.13 (Linux)
         if: matrix.os-name == 'Linux' && matrix.test-category == 'AzureServiceBus'
         uses: actions/setup-python@v5
@@ -84,11 +80,6 @@ jobs:
           python -m pip install --user "azure-cli==2.64.0"
           echo "$HOME/.local/bin" >> "$GITHUB_PATH"
 
-      - name: Azure login
-        uses: azure/login@v3.0.1
-        if: matrix.test-category == 'AzureServiceBus' || matrix.test-category == 'AzureStorageQueues' || matrix.test-category == 'RabbitMQ' || matrix.test-category == 'PostgreSQL' || matrix.test-category == 'PostgreSQLPersistence'
-        with:
-          creds: ${{ secrets.AZURE_ACI_CREDENTIALS }}
       - name: Setup SQL Server
         uses: Particular/install-sql-server-action@v3.0.0
         if: matrix.test-category == 'SqlServer'
@@ -104,25 +95,49 @@ jobs:
           enable-full-text-search: true
       - name: Setup PostgreSQL
         uses: Particular/setup-postgres-action@v3.0.0
-        if: matrix.test-category == 'PostgreSQL'
+        if: matrix.test-category == 'PostgreSql'
         with:
           connection-string-name: ServiceControl_TransportTests_PostgreSQL_ConnectionString
           registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
           registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Setup PostgreSQL persistence
         uses: Particular/setup-postgres-action@v3.0.0
-        if: matrix.test-category == 'PostgreSQLPersistence'
+        if: matrix.test-category == 'PostgreSqlPersistence'
         with:
           connection-string-name: ServiceControl_Persistence_PostgreSql_ConnectionString
           registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
           registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Setup RabbitMQ
         uses: Particular/setup-rabbitmq-action@v2.0.0
-        if: matrix.test-category == 'RabbitMQ'
+        if: startsWith(matrix.test-category, 'RabbitMQ')
         with:
           connection-string-name: ServiceControl_TransportTests_RabbitMQ_ConnectionString
           registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
           registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Setup IBM MQ
+        uses: Particular/setup-ibmmq-action@75b6087748fcdaef474574ca1fbe2b440141f07f
+        if: matrix.test-category == 'IBMMQ'
+        with:
+          connection-string-name: ServiceControl_TransportTests_IBMMQ_ConnectionString
+      - name: Setup SQS environment variables
+        if: matrix.test-category == 'SQS'
+        run: |
+          echo "AWS_REGION=${{ secrets.AWS_REGION }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+          echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+          echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+
+          # Cleanup of queues starting with `GHA-` handled by https://github.com/Particular/NServiceBus.AmazonSQS/blob/master/.github/workflows/tests-cleanup.yml
+          $connectString = "AccessKeyId=${{ secrets.AWS_ACCESS_KEY_ID }};SecretAccessKey=${{ secrets.AWS_SECRET_ACCESS_KEY }};Region=${{ secrets.AWS_REGION }};QueueNamePrefix=GHA-${{ github.run_id }}"
+          echo "ServiceControl_TransportTests_SQS_ConnectionString=$connectString" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+      # Everything above provisions local containers, so it overlaps the build freely. The Azure steps
+      # below create real cloud resources, so they sit behind the wait too.
+      - name: Wait for build
+        wait: build
+      - name: Azure login
+        uses: azure/login@v3.0.1
+        if: matrix.test-category == 'AzureServiceBus' || matrix.test-category == 'AzureStorageQueues'
+        with:
+          creds: ${{ secrets.AZURE_ACI_CREDENTIALS }}
       - name: Setup Azure Service Bus
         uses: Particular/setup-azureservicebus-action@v2.5.0
         if: matrix.test-category == 'AzureServiceBus'
@@ -137,35 +152,43 @@ jobs:
           connection-string-name: ServiceControl_TransportTests_ASQ_ConnectionString
           azure-credentials: ${{ secrets.AZURE_ACI_CREDENTIALS }}
           tag: ServiceControl
-      - name: Setup IBM MQ
-        if: matrix.test-category == 'IBMMQ'
-        run: |
-          docker run --name ibmmq -d -p 1414:1414 -p 9443:9443 `
-            --health-cmd "dspmq" --health-interval 10s --health-timeout 5s --health-retries 10 --health-start-period 30s `
-            -e LICENSE=accept -e MQ_QMGR_NAME=QM1 -e MQ_ADMIN_PASSWORD=passw0rd `
-            icr.io/ibm-messaging/mq:latest
-          # Wait for container health check to pass
-          while ((docker inspect --format '{{.State.Health.Status}}' ibmmq) -ne 'healthy') {
-            Start-Sleep -Seconds 2
-          }
-          echo "ServiceControl_TransportTests_IBMMQ_ConnectionString=mq://admin:passw0rd@localhost:1414/QM1?channel=DEV.ADMIN.SVRCONN&topicprefix=DEV" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
-      - name: Setup SQS environment variables
-        if: matrix.test-category == 'SQS'
-        run: |
-          echo "AWS_REGION=${{ secrets.AWS_REGION }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
-          echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
-          echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
-
-          # Cleanup of queues starting with `GHA-` handled by https://github.com/Particular/NServiceBus.AmazonSQS/blob/master/.github/workflows/tests-cleanup.yml
-          $connectString = "AccessKeyId=${{ secrets.AWS_ACCESS_KEY_ID }};SecretAccessKey=${{ secrets.AWS_SECRET_ACCESS_KEY }};Region=${{ secrets.AWS_REGION }};QueueNamePrefix=GHA-${{ github.run_id }}"
-          echo "ServiceControl_TransportTests_SQS_ConnectionString=$connectString" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
       - name: Run tests
-        uses: Particular/run-tests-action@v1.7.0
+        run: ./tools/run-tests.ps1 -Projects $Env:TEST_PROJECTS
         env:
+          TEST_PROJECTS: ${{ steps.select.outputs.test-projects }}
           ServiceControl_TESTS_FILTER: ${{ matrix.test-category }}
           PARTICULARSOFTWARE_LICENSE: ${{ secrets.LICENSETEXT }}
           AZURE_ACI_CREDENTIALS: ${{ secrets.AZURE_ACI_CREDENTIALS }}
 
+  compile:
+    name: ${{ matrix.os-name }}-Compile
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: windows-latest
+            os-name: Windows
+          - os: ubuntu-latest
+            os-name: Linux
+      fail-fast: false
+    steps:
+      - name: Check for secrets
+        env:
+          SECRETS_AVAILABLE: ${{ secrets.SECRETS_AVAILABLE }}
+        run: exit $(If ($env:SECRETS_AVAILABLE -eq 'true') { 0 } Else { 1 })
+      - name: Checkout
+        uses: actions/checkout@v7.0.1
+        with:
+          fetch-depth: 0
+      - name: Setup .NET SDK
+        uses: actions/setup-dotnet@v6.0.0
+        with:
+          global-json-file: global.json
+      - name: Download RavenDB Server
+        run: ./tools/download-ravendb-server.ps1
+      - name: Build
+        run: dotnet build src --configuration Release -graph
+
   windows-installers:
     uses: ./.github/workflows/build-windows.yml
     secrets: inherit

.gitignore

@@ -2,6 +2,8 @@
 /binaries
 /deploy
 /nugets
+# Generated by tools/select-test-projects.ps1
+/tests.proj
 build32
 *.vshost.*
 .nu

README.md

@@ -46,17 +46,58 @@ Testing using the [CI workflow](/.github/workflows/ci.yml) depends on the follow
 
 Running all tests all the times takes a lot of resources. Tests are filtered based on the `ServiceControl_TESTS_FILTER` environment variable. To run only a subset, e.g., SQS transport tests, define the variable as `ServiceControl_TESTS_FILTER=SQS`. The following list contains all the possible `ServiceControl_TESTS_FILTER` values:
 
-- `Default` - runs only non-transport-specific tests
+Non-transport-specific:
+
+- `DefaultCore`
+- `DefaultAudit`
+- `DefaultMonitoring`
+
+Transports:
+
 - `AzureServiceBus`
 - `AzureStorageQueues`
+- `IBMMQ`
 - `MSMQ`
-- `RabbitMQ`
+- `PostgreSql`
+- `RabbitMQClassicConventional`
+- `RabbitMQClassicDirect`
+- `RabbitMQQuorumConventional`
+- `RabbitMQQuorumDirect`
 - `SqlServer`
-- `SqlServerPersistence`
-- `PostgresSqlPersistence`
 - `SQS`
 
-NOTE: If no variable is defined all tests will be executed.
+Persisters:
+
+- `PostgreSqlPersistence`
+- `PrimaryRavenAcceptance`
+- `PrimaryRavenPersistence`
+- `SqlServerPersistence`
+
+> [!NOTE]
+> If no variable is defined all tests will be executed.
+
+Each category is declared once, by the `<TestCategory>` property in the test project. CI reads that property to build and run only the projects belonging to the category under test, and the build generates the assembly-level `IncludeInTestCategory` attribute from it, which is what `ServiceControl_TESTS_FILTER` matches against at run time.
+
+### Adding a test project
+
+> [!IMPORTANT]
+> Every test project must set `<TestCategory>` in its `.csproj`, alongside `TargetFramework`:
+>
+> ```xml
+> <PropertyGroup>
+>   <TargetFramework>net10.0</TargetFramework>
+>   <TestCategory>DefaultCore</TestCategory>
+> </PropertyGroup>
+> ```
+>
+> CI selects projects by that property alone, so a test project without it is never built and never run. The tests would simply not exist as far as CI is concerned, and nothing would go red. `tools/select-test-projects.ps1` fails the build if a project references `Microsoft.NET.Test.Sdk` without declaring a category, so this cannot be forgotten silently.
+
+Use an existing category from the list above where the tests fit.
+
+> [!WARNING]
+> Introducing a *new* category also means adding it to the `test-category` matrix in [ci.yml](.github/workflows/ci.yml), otherwise no job ever selects it and the tests never run.
+
+Run `./tools/select-test-projects.ps1 -List` to see every category and the projects it selects.
 
 ## Security Configuration
 

src/Directory.Build.props

@@ -31,6 +31,18 @@
     <SourceRoot Include="$([MSBuild]::NormalizePath($(MSBuildThisFileDirectory)..\))" />
   </ItemGroup>
 
+  <!--
+    A test project declares its category once, via <TestCategory>. CI reads that property to decide
+    which projects to build and run; this generates the matching assembly attribute so that setting
+    ServiceControl_TESTS_FILTER locally still narrows a run down to one category. Generating it rather
+    than hand-writing a TestsFilter.cs keeps the two from drifting apart.
+  -->
+  <ItemGroup Condition="'$(TestCategory)' != ''">
+    <AssemblyAttribute Include="IncludeInTestCategoryAttribute">
+      <_Parameter1>$(TestCategory)</_Parameter1>
+    </AssemblyAttribute>
+  </ItemGroup>
+
   <ItemGroup>
    <ProjectCapability Include="DynamicFileNestingEnabled" />
   </ItemGroup>

src/Particular.LicensingComponent.UnitTests/Particular.LicensingComponent.UnitTests.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net10.0</TargetFramework>
+    <TestCategory>DefaultCore</TestCategory>
   </PropertyGroup>
 
   <ItemGroup>

src/ServiceControl.AcceptanceTests.RavenDB/ServiceControl.AcceptanceTests.RavenDB.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net10.0</TargetFramework>
+    <TestCategory>PrimaryRavenAcceptance</TestCategory>
   </PropertyGroup>
 
   <ItemGroup>

src/ServiceControl.Audit.AcceptanceTests.RavenDB/ServiceControl.Audit.AcceptanceTests.RavenDB.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net10.0</TargetFramework>
+    <TestCategory>DefaultAudit</TestCategory>
   </PropertyGroup>
 
   <ItemGroup>
@@ -26,7 +27,6 @@
     <Compile Include="..\ServiceControl.Audit.AcceptanceTests\**\*.cs" Exclude="..\ServiceControl.Audit.AcceptanceTests\AcceptanceTestStorageConfiguration.cs;..\ServiceControl.Audit.AcceptanceTests\obj\**\*.*;..\ServiceControl.Audit.AcceptanceTests\bin\**\*.*" />
     <Compile Include="..\ServiceControl.Audit.Persistence.Tests.RavenDB\SharedEmbeddedServer.cs" />
     <Compile Include="..\ServiceControl.Audit.Persistence.Tests.RavenDB\StopSharedDatabase.cs" />
-    <Compile Remove="..\ServiceControl.Audit.AcceptanceTests\TestsFilter.cs" />
   </ItemGroup>
 
 </Project>

src/ServiceControl.Audit.AcceptanceTests/ServiceControl.Audit.AcceptanceTests.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net10.0</TargetFramework>
+    <TestCategory>DefaultAudit</TestCategory>
   </PropertyGroup>
 
   <ItemGroup>

src/ServiceControl.Audit.Persistence.Tests.RavenDB/ServiceControl.Audit.Persistence.Tests.RavenDB.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net10.0</TargetFramework>
+    <TestCategory>DefaultAudit</TestCategory>
   </PropertyGroup>
 
   <ItemGroup>
@@ -24,7 +25,6 @@
   <ItemGroup>
     <Compile Include="..\ServiceControl.Audit.Persistence.Tests\*.cs" LinkBase="Shared" />
     <Compile Remove="..\ServiceControl.Audit.Persistence.Tests\PersistenceManifestLibraryTests.cs" />
-    <Compile Remove="..\ServiceControl.Audit.Persistence.Tests\TestsFilter.cs" />
   </ItemGroup>
 
 </Project>

src/ServiceControl.Audit.Persistence.Tests/ServiceControl.Audit.Persistence.Tests.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net10.0</TargetFramework>
+    <TestCategory>DefaultAudit</TestCategory>
   </PropertyGroup>
 
   <ItemGroup>

src/ServiceControl.Audit.UnitTests/ServiceControl.Audit.UnitTests.csproj

@@ -2,6 +2,7 @@
 
   <PropertyGroup>
     <TargetFramework>net10.0</TargetFramework>
+    <TestCategory>DefaultAudit</TestCategory>
   </PropertyGroup>
 
   <ItemGroup>

src/ServiceControl.Audit/ServiceControl.Audit.csproj

@@ -59,6 +59,7 @@
     <InternalsVisibleTo Include="ServiceControl.Audit.AcceptanceTests" />
     <InternalsVisibleTo Include="ServiceControl.Audit.AcceptanceTests.RavenDB" />
     <InternalsVisibleTo Include="ServiceControl.Audit.UnitTests" />
+    <InternalsVisibleTo Include="ServiceControl.MultiInstance.AcceptanceTests" />
   </ItemGroup>
 
 </Project>