diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3c7df2df6..a80ce84ecc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,9 @@ env: defaults: run: shell: pwsh +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: build: name: ${{ matrix.os-name }}-${{ matrix.test-category }} @@ -18,13 +21,26 @@ jobs: strategy: matrix: os: [windows-latest, ubuntu-latest] - # Categories are declared by the property in each test project. - test-category: [ DefaultCore, DefaultAudit, DefaultMonitoring, SqlServer, SqlServerPersistence, AzureServiceBus, RabbitMQClassicConventional, RabbitMQClassicDirect, RabbitMQQuorumConventional, RabbitMQQuorumDirect, AzureStorageQueues, MSMQ, SQS, PrimaryRavenAcceptance, PrimarySqlServerAcceptance, PrimaryPostgreSqlAcceptance, PrimaryRavenPersistence, PostgreSql, PostgreSqlPersistence, IBMMQ ] + # Categories are declared by the property in each test project. A category can + # span several projects that share infrastructure, so that the container is provisioned once + # and the union of their build closures is compiled once. + test-category: [ DefaultCore, DefaultMonitoring, Raven, SqlServer, PostgreSql, RabbitMQ, AzureServiceBus, AzureStorageQueues, MSMQ, SQS, IBMMQ ] include: - os: windows-latest os-name: Windows - os: ubuntu-latest os-name: Linux + # How many of a category's assemblies run at once, so that a multi-project category costs the + # slowest of them rather than the sum. Defaults to 1 where unset. Dial down if the shared + # infrastructure turns out not to tolerate the concurrency. + - test-category: Raven + max-parallel: 3 + - test-category: RabbitMQ + max-parallel: 4 + - test-category: SqlServer + max-parallel: 3 + - test-category: PostgreSql + max-parallel: 3 exclude: - os: ubuntu-latest test-category: MSMQ @@ -56,13 +72,12 @@ jobs: # run in on Windows. A no-op on Linux, but still run there so those actions see the same # environment variables on both runners. - name: Setup WSL - if: startsWith(matrix.test-category, 'RabbitMQ') || contains(fromJSON('["SqlServer", "SqlServerPersistence", "PrimarySqlServerAcceptance", "PostgreSql", "PostgreSqlPersistence", "PrimaryPostgreSqlAcceptance", "IBMMQ"]'), matrix.test-category) + if: contains(fromJSON('["RabbitMQ", "SqlServer", "PostgreSql", "IBMMQ"]'), matrix.test-category) uses: Particular/setup-wsl-action@v1.2.0 with: # The action defaults to 4GB. The runner has 16GB and the build runs concurrently with the # container starting up, so give the VM real headroom. memory: 8GB - # there is an issue with az cli and python 3.14, so for now we need to pin it # once the issue is resolved it should be able to be re-floated # https://github.com/Azure/azure-cli/issues/32980. @@ -79,37 +94,29 @@ jobs: python -m pip install --upgrade pip python -m pip install --user "azure-cli==2.64.0" echo "$HOME/.local/bin" >> "$GITHUB_PATH" - - name: Setup SQL Server uses: Particular/install-sql-server-action@v3.0.0 if: matrix.test-category == 'SqlServer' - with: - connection-string-env-var: ServiceControl_TransportTests_SQL_ConnectionString - catalog: nservicebus - - name: Setup SQL Server persistence - uses: Particular/install-sql-server-action@v3.0.0 - if: matrix.test-category == 'SqlServerPersistence' || matrix.test-category == 'PrimarySqlServerAcceptance' with: connection-string-env-var: ServiceControl_Persistence_SqlServer_ConnectionString catalog: ServiceControl enable-full-text-search: true + - name: Point the SQL Server transport tests at the same server + if: matrix.test-category == 'SqlServer' + run: echo "ServiceControl_TransportTests_SQL_ConnectionString=$Env:ServiceControl_Persistence_SqlServer_ConnectionString" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append - name: Setup PostgreSQL uses: Particular/setup-postgres-action@v3.0.0 if: matrix.test-category == 'PostgreSql' - with: - connection-string-name: ServiceControl_TransportTests_PostgreSQL_ConnectionString - registry-username: ${{ secrets.DOCKERHUB_USERNAME }} - registry-password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Setup PostgreSQL persistence - uses: Particular/setup-postgres-action@v3.0.0 - if: matrix.test-category == 'PostgreSqlPersistence' || matrix.test-category == 'PrimaryPostgreSqlAcceptance' with: connection-string-name: ServiceControl_Persistence_PostgreSql_ConnectionString registry-username: ${{ secrets.DOCKERHUB_USERNAME }} registry-password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Point the PostgreSQL transport tests at the same server + if: matrix.test-category == 'PostgreSql' + run: echo "ServiceControl_TransportTests_PostgreSQL_ConnectionString=$Env:ServiceControl_Persistence_PostgreSql_ConnectionString" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append - name: Setup RabbitMQ uses: Particular/setup-rabbitmq-action@v2.0.0 - if: startsWith(matrix.test-category, 'RabbitMQ') + if: matrix.test-category == 'RabbitMQ' with: connection-string-name: ServiceControl_TransportTests_RabbitMQ_ConnectionString registry-username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -153,7 +160,7 @@ jobs: azure-credentials: ${{ secrets.AZURE_ACI_CREDENTIALS }} tag: ServiceControl - name: Run tests - run: ./tools/run-tests.ps1 -Projects $Env:TEST_PROJECTS + run: ./tools/run-tests.ps1 -Projects $Env:TEST_PROJECTS -MaxParallel ${{ matrix.max-parallel || 1 }} env: TEST_PROJECTS: ${{ steps.select.outputs.test-projects }} ServiceControl_TESTS_FILTER: ${{ matrix.test-category }} diff --git a/README.md b/README.md index 8db5a58c80..dbdd477cb5 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,6 @@ Running all tests all the times takes a lot of resources. Tests are filtered bas Non-transport-specific: - `DefaultCore` -- `DefaultAudit` - `DefaultMonitoring` Transports: @@ -58,26 +57,25 @@ Transports: - `AzureStorageQueues` - `IBMMQ` - `MSMQ` -- `PostgreSql` -- `RabbitMQClassicConventional` -- `RabbitMQClassicDirect` -- `RabbitMQQuorumConventional` -- `RabbitMQQuorumDirect` -- `SqlServer` +- `RabbitMQ` (all four classic/quorum and conventional/direct routing combinations) - `SQS` +Transports and persisters sharing a database server: + +- `PostgreSql` (transport, persistence and acceptance tests) +- `SqlServer` (transport, persistence and acceptance tests) + Persisters: -- `PostgreSqlPersistence` -- `PrimaryRavenAcceptance` -- `PrimaryRavenPersistence` -- `SqlServerPersistence` +- `Raven` (persistence, acceptance and audit tests) > [!NOTE] > If no variable is defined all tests will be executed. Each category is declared once, by the `` property in the test project. CI reads that property to build and run only the projects belonging to the category under test, and the build generates the assembly-level `IncludeInTestCategory` attribute from it, which is what `ServiceControl_TESTS_FILTER` matches against at run time. +A category can span several projects. Where those projects need the same infrastructure, sharing a category means CI provisions it once, compiles the union of their build closures once, and runs their assemblies concurrently on a single runner. That is why, for example, all four RabbitMQ routing topologies are one category rather than four. + ### Adding a test project > [!IMPORTANT] diff --git a/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj b/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj index 0f29ca729f..028c471d49 100644 --- a/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj +++ b/src/ServiceControl.AcceptanceTests.PostgreSql/ServiceControl.AcceptanceTests.PostgreSql.csproj @@ -2,7 +2,7 @@ net10.0 - PrimaryPostgreSqlAcceptance + PostgreSql diff --git a/src/ServiceControl.AcceptanceTests.RavenDB/ServiceControl.AcceptanceTests.RavenDB.csproj b/src/ServiceControl.AcceptanceTests.RavenDB/ServiceControl.AcceptanceTests.RavenDB.csproj index 93d3a06ba3..bc922ad126 100644 --- a/src/ServiceControl.AcceptanceTests.RavenDB/ServiceControl.AcceptanceTests.RavenDB.csproj +++ b/src/ServiceControl.AcceptanceTests.RavenDB/ServiceControl.AcceptanceTests.RavenDB.csproj @@ -2,7 +2,7 @@ net10.0 - PrimaryRavenAcceptance + Raven diff --git a/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj b/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj index d8ffdd841b..be951002d9 100644 --- a/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj +++ b/src/ServiceControl.AcceptanceTests.SqlServer/ServiceControl.AcceptanceTests.SqlServer.csproj @@ -2,7 +2,7 @@ net10.0 - PrimarySqlServerAcceptance + SqlServer diff --git a/src/ServiceControl.Audit.AcceptanceTests.RavenDB/ServiceControl.Audit.AcceptanceTests.RavenDB.csproj b/src/ServiceControl.Audit.AcceptanceTests.RavenDB/ServiceControl.Audit.AcceptanceTests.RavenDB.csproj index a71d5defe6..a22d55be05 100644 --- a/src/ServiceControl.Audit.AcceptanceTests.RavenDB/ServiceControl.Audit.AcceptanceTests.RavenDB.csproj +++ b/src/ServiceControl.Audit.AcceptanceTests.RavenDB/ServiceControl.Audit.AcceptanceTests.RavenDB.csproj @@ -2,7 +2,7 @@ net10.0 - DefaultAudit + Raven diff --git a/src/ServiceControl.Audit.AcceptanceTests/ServiceControl.Audit.AcceptanceTests.csproj b/src/ServiceControl.Audit.AcceptanceTests/ServiceControl.Audit.AcceptanceTests.csproj index 2388115142..afab38a784 100644 --- a/src/ServiceControl.Audit.AcceptanceTests/ServiceControl.Audit.AcceptanceTests.csproj +++ b/src/ServiceControl.Audit.AcceptanceTests/ServiceControl.Audit.AcceptanceTests.csproj @@ -2,7 +2,7 @@ net10.0 - DefaultAudit + Raven diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ServiceControl.Audit.Persistence.Tests.RavenDB.csproj b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ServiceControl.Audit.Persistence.Tests.RavenDB.csproj index 26a098607b..2c798be958 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ServiceControl.Audit.Persistence.Tests.RavenDB.csproj +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/ServiceControl.Audit.Persistence.Tests.RavenDB.csproj @@ -2,7 +2,7 @@ net10.0 - DefaultAudit + Raven diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs index 8d8c613ca3..0cf229eb72 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs @@ -35,7 +35,7 @@ public static async Task GetInstance(CancellationToken cancell var dbPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Tests", "AuditData"); var logPath = Path.Combine(TestContext.CurrentContext.WorkDirectory, "Logs", "Audit"); var logsMode = "Operations"; - var serverUrl = $"http://localhost:{PortUtility.FindAvailablePort(33334)}"; + var serverUrl = $"http://localhost:{PortUtility.GetAssignedOrAvailablePort(33334)}"; var databaseConfiguration = new DatabaseConfiguration("audit", 60, true, TimeSpan.FromMinutes(5), 120000, 5, 5, new ServerConfiguration(dbPath, serverUrl, logPath, logsMode), TimeSpan.FromSeconds(60)); var serverConfig = databaseConfiguration.ServerConfiguration; diff --git a/src/ServiceControl.Audit.Persistence.Tests/ServiceControl.Audit.Persistence.Tests.csproj b/src/ServiceControl.Audit.Persistence.Tests/ServiceControl.Audit.Persistence.Tests.csproj index ece853badc..ac37ef4bce 100644 --- a/src/ServiceControl.Audit.Persistence.Tests/ServiceControl.Audit.Persistence.Tests.csproj +++ b/src/ServiceControl.Audit.Persistence.Tests/ServiceControl.Audit.Persistence.Tests.csproj @@ -2,7 +2,7 @@ net10.0 - DefaultAudit + Raven diff --git a/src/ServiceControl.Audit.UnitTests/ServiceControl.Audit.UnitTests.csproj b/src/ServiceControl.Audit.UnitTests/ServiceControl.Audit.UnitTests.csproj index 44a4d9efa0..c4ee2eb333 100644 --- a/src/ServiceControl.Audit.UnitTests/ServiceControl.Audit.UnitTests.csproj +++ b/src/ServiceControl.Audit.UnitTests/ServiceControl.Audit.UnitTests.csproj @@ -2,7 +2,7 @@ net10.0 - DefaultAudit + Raven diff --git a/src/ServiceControl.Audit/Infrastructure/EventSourceCreator.cs b/src/ServiceControl.Audit/Infrastructure/EventSourceCreator.cs index 004fd541e4..ab224077ba 100644 --- a/src/ServiceControl.Audit/Infrastructure/EventSourceCreator.cs +++ b/src/ServiceControl.Audit/Infrastructure/EventSourceCreator.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Audit.Infrastructure { + using System; using System.Diagnostics; using System.Runtime.Versioning; @@ -8,10 +9,22 @@ static class EventSourceCreator [SupportedOSPlatform("windows")] public static void Create() { - if (!EventLog.SourceExists(SourceName)) + if (EventLog.SourceExists(SourceName)) + { + return; + } + + try { EventLog.CreateEventSource(SourceName, null); } + // An event source is machine-wide, and the check above cannot be made atomic with the + // create. Anything else running this at the same time, another instance being set up or, + // in CI, another test assembly, can get there first. The filter keeps that case distinct + // from an ArgumentException that means the source name itself is unusable. + catch (ArgumentException) when (EventLog.SourceExists(SourceName)) + { + } } public const string SourceName = "ServiceControl.Audit"; diff --git a/src/ServiceControl.Persistence.Tests.PostgreSql/ServiceControl.Persistence.Tests.PostgreSql.csproj b/src/ServiceControl.Persistence.Tests.PostgreSql/ServiceControl.Persistence.Tests.PostgreSql.csproj index a7cfd08509..5401253c40 100644 --- a/src/ServiceControl.Persistence.Tests.PostgreSql/ServiceControl.Persistence.Tests.PostgreSql.csproj +++ b/src/ServiceControl.Persistence.Tests.PostgreSql/ServiceControl.Persistence.Tests.PostgreSql.csproj @@ -2,7 +2,7 @@ net10.0 - PostgreSqlPersistence + PostgreSql diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/ServiceControl.Persistence.Tests.RavenDB.csproj b/src/ServiceControl.Persistence.Tests.RavenDB/ServiceControl.Persistence.Tests.RavenDB.csproj index 4b8c1de262..83f29d47a4 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/ServiceControl.Persistence.Tests.RavenDB.csproj +++ b/src/ServiceControl.Persistence.Tests.RavenDB/ServiceControl.Persistence.Tests.RavenDB.csproj @@ -2,7 +2,7 @@ net10.0 - PrimaryRavenPersistence + Raven diff --git a/src/ServiceControl.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs b/src/ServiceControl.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs index b35874bba5..49b3cddb56 100644 --- a/src/ServiceControl.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs +++ b/src/ServiceControl.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs @@ -41,7 +41,7 @@ public static async Task GetInstance(CancellationToken cancell DatabasePath = dbPath, LogPath = logPath, LogsMode = logsMode, - DatabaseMaintenancePort = PortUtility.FindAvailablePort(RavenPersisterSettings.DatabaseMaintenancePortDefault) + DatabaseMaintenancePort = PortUtility.GetAssignedOrAvailablePort(RavenPersisterSettings.DatabaseMaintenancePortDefault) }; // TODO: See if more refactoring can be done between this and the RavenPersisterSettings above diff --git a/src/ServiceControl.Persistence.Tests.SqlServer/ServiceControl.Persistence.Tests.SqlServer.csproj b/src/ServiceControl.Persistence.Tests.SqlServer/ServiceControl.Persistence.Tests.SqlServer.csproj index 7fb36676a0..7fd542c6f8 100644 --- a/src/ServiceControl.Persistence.Tests.SqlServer/ServiceControl.Persistence.Tests.SqlServer.csproj +++ b/src/ServiceControl.Persistence.Tests.SqlServer/ServiceControl.Persistence.Tests.SqlServer.csproj @@ -2,7 +2,7 @@ net10.0 - SqlServerPersistence + SqlServer diff --git a/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/ServiceControl.Transports.RabbitMQClassicConventionalRoutingTests.csproj b/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/ServiceControl.Transports.RabbitMQClassicConventionalRoutingTests.csproj index 6fd23a7ba6..4aa51233bf 100644 --- a/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/ServiceControl.Transports.RabbitMQClassicConventionalRoutingTests.csproj +++ b/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/ServiceControl.Transports.RabbitMQClassicConventionalRoutingTests.csproj @@ -2,7 +2,7 @@ net10.0 - RabbitMQClassicConventional + RabbitMQ diff --git a/src/ServiceControl.Transports.RabbitMQClassicDirectRouting.Tests/ServiceControl.Transports.RabbitMQClassicDirectRouting.Tests.csproj b/src/ServiceControl.Transports.RabbitMQClassicDirectRouting.Tests/ServiceControl.Transports.RabbitMQClassicDirectRouting.Tests.csproj index 2164ec1d26..8f5bb59f35 100644 --- a/src/ServiceControl.Transports.RabbitMQClassicDirectRouting.Tests/ServiceControl.Transports.RabbitMQClassicDirectRouting.Tests.csproj +++ b/src/ServiceControl.Transports.RabbitMQClassicDirectRouting.Tests/ServiceControl.Transports.RabbitMQClassicDirectRouting.Tests.csproj @@ -2,7 +2,7 @@ net10.0 - RabbitMQClassicDirect + RabbitMQ diff --git a/src/ServiceControl.Transports.RabbitMQQuorumConventionalRouting.Tests/ServiceControl.Transports.RabbitMQQuorumConventionalRouting.Tests.csproj b/src/ServiceControl.Transports.RabbitMQQuorumConventionalRouting.Tests/ServiceControl.Transports.RabbitMQQuorumConventionalRouting.Tests.csproj index be4eea22f0..8f5bb59f35 100644 --- a/src/ServiceControl.Transports.RabbitMQQuorumConventionalRouting.Tests/ServiceControl.Transports.RabbitMQQuorumConventionalRouting.Tests.csproj +++ b/src/ServiceControl.Transports.RabbitMQQuorumConventionalRouting.Tests/ServiceControl.Transports.RabbitMQQuorumConventionalRouting.Tests.csproj @@ -2,7 +2,7 @@ net10.0 - RabbitMQQuorumConventional + RabbitMQ diff --git a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj index 5f5b275050..4aa51233bf 100644 --- a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj +++ b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj @@ -2,7 +2,7 @@ net10.0 - RabbitMQQuorumDirect + RabbitMQ diff --git a/src/ServiceControl/Infrastructure/Installers/EventSourceCreator.cs b/src/ServiceControl/Infrastructure/Installers/EventSourceCreator.cs index a40ed882aa..e765c5a7b1 100644 --- a/src/ServiceControl/Infrastructure/Installers/EventSourceCreator.cs +++ b/src/ServiceControl/Infrastructure/Installers/EventSourceCreator.cs @@ -1,5 +1,6 @@ namespace ServiceBus.Management.Infrastructure.Installers { + using System; using System.Diagnostics; using System.Runtime.Versioning; @@ -8,10 +9,22 @@ static class EventSourceCreator [SupportedOSPlatform("windows")] public static void Create() { - if (!EventLog.SourceExists(SourceName)) + if (EventLog.SourceExists(SourceName)) + { + return; + } + + try { EventLog.CreateEventSource(SourceName, null); } + // An event source is machine-wide, and the check above cannot be made atomic with the + // create. Anything else running this at the same time, another instance being set up or, + // in CI, another test assembly, can get there first. The filter keeps that case distinct + // from an ArgumentException that means the source name itself is unusable. + catch (ArgumentException) when (EventLog.SourceExists(SourceName)) + { + } } public const string SourceName = "ServiceControl"; diff --git a/src/TestHelper/PortUtility.cs b/src/TestHelper/PortUtility.cs index 7fc4648031..21c1aa4169 100644 --- a/src/TestHelper/PortUtility.cs +++ b/src/TestHelper/PortUtility.cs @@ -1,10 +1,34 @@ namespace TestHelper { + using System; + using System.Globalization; using System.Linq; using System.Net.NetworkInformation; public static class PortUtility { + /// + /// The port an embedded server should bind, when the test runner has assigned one. + /// + public const string AssignedPortVariableName = "ServiceControl_TESTS_RAVENDB_PORT"; + + /// + /// Returns the port assigned by the test runner, or probes for a free one when running alone. + /// + /// + /// Concurrent test processes cannot each probe: only inspects + /// the listeners active at that instant, so processes starting together all see the same port + /// free and all but one then fail to bind. + /// + public static int GetAssignedOrAvailablePort(int startPort) + { + var assignedPort = Environment.GetEnvironmentVariable(AssignedPortVariableName); + + return string.IsNullOrWhiteSpace(assignedPort) + ? FindAvailablePort(startPort) + : int.Parse(assignedPort, CultureInfo.InvariantCulture); + } + public static int FindAvailablePort(int startPort) { var activeTcpListeners = IPGlobalProperties diff --git a/tools/run-tests.ps1 b/tools/run-tests.ps1 index 5927f0d073..5d47ac6085 100644 --- a/tools/run-tests.ps1 +++ b/tools/run-tests.ps1 @@ -4,6 +4,10 @@ # # This is a scoped replacement for Particular/run-tests-action, which has no way to be told which # projects to run. It should fold back into that action once it grows a 'projects' input. +# +# -MaxParallel runs several assemblies at once. CI jobs that merge categories sharing infrastructure +# use it so the job costs the slowest assembly rather than the sum of all of them. Output is buffered +# per run and replayed on completion, because interleaved dotnet test output is unreadable. param( [Parameter(Mandatory)] @@ -11,6 +15,9 @@ param( [string]$TargetPlatform = 'x64', + [ValidateRange(1, 16)] + [int]$MaxParallel = 1, + [switch]$ReportWarnings ) @@ -23,10 +30,12 @@ if ($projectPaths.Count -eq 0) { } Write-Output "Target Platform = $TargetPlatform" +Write-Output "Max parallel test runs = $MaxParallel" $reportWarningsValue = if ($ReportWarnings) { 'true' } else { 'false' } $isUnix = $PSVersionTable.Platform -eq 'Unix' -$exitCode = 0 + +$runs = [Collections.Generic.List[object]]::new() foreach ($project in $projectPaths) { $frameworks = @( @@ -44,16 +53,94 @@ foreach ($project in $projectPaths) { continue } - Write-Output "::group::Running $(Split-Path $project -Leaf) ($framework)" + $runs.Add([pscustomobject]@{ + Label = "$(Split-Path $project -Leaf) ($framework)" + Project = $project + Framework = $framework + }) + } +} + +if ($runs.Count -eq 0) { + throw 'No test projects were runnable on this platform.' +} + +# RavenDB.Embedded binds a fixed port, and the tests otherwise pick one by probing the active +# listeners, which concurrent processes all do at the same instant and all resolve to the same port. +# Hand each run its own instead. Left unset when running one at a time, because probing copes better +# with a port that something else on the machine already holds. +$assignPorts = $MaxParallel -gt 1 +$nextPort = 33334 +$portSpacing = 10 + +$exitCode = 0 - dotnet test $project --configuration Release --no-build --framework $framework --logger "GitHubActions;report-warnings=$reportWarningsValue" -- RunConfiguration.TreatNoTestsAsError=true "RunConfiguration.TargetPlatform=$TargetPlatform" +function Complete-Run($run) { + Write-Output "::group::Running $($run.Label)" + foreach ($stream in @($run.OutFile, $run.ErrFile)) { + if ((Test-Path $stream) -and (Get-Item $stream).Length -gt 0) { + Get-Content -Path $stream | Write-Output + } + Remove-Item -Path $stream -Force -ErrorAction SilentlyContinue + } + Write-Output '::endgroup::' - Write-Output '::endgroup::' + if ($run.Process.ExitCode -ne 0) { + Write-Output "::error::$($run.Label) exit code = $($run.Process.ExitCode)" + $script:exitCode = 1 + } +} - if ($LASTEXITCODE -ne 0) { - Write-Output "::error::Exit code = $LASTEXITCODE" - $exitCode = 1 +$pending = [Collections.Generic.Queue[object]]::new($runs) +$active = [Collections.Generic.List[object]]::new() + +while ($pending.Count -gt 0 -or $active.Count -gt 0) { + while ($active.Count -lt $MaxParallel -and $pending.Count -gt 0) { + $run = $pending.Dequeue() + $run | Add-Member -NotePropertyName OutFile -NotePropertyValue ([IO.Path]::GetTempFileName()) + $run | Add-Member -NotePropertyName ErrFile -NotePropertyValue ([IO.Path]::GetTempFileName()) + + $arguments = @( + 'test', $run.Project + '--configuration', 'Release' + '--no-build' + '--framework', $run.Framework + '--logger', "GitHubActions;report-warnings=$reportWarningsValue" + '--' + 'RunConfiguration.TreatNoTestsAsError=true' + "RunConfiguration.TargetPlatform=$TargetPlatform" + ) + + if ($assignPorts) { + # Set immediately before spawning, so the child inherits this run's value. Safe because + # spawning is serialised here even though the runs themselves are not. + $Env:ServiceControl_TESTS_RAVENDB_PORT = $nextPort + $nextPort += $portSpacing } + + Write-Output "Starting $($run.Label)" + + $run | Add-Member -NotePropertyName Process -NotePropertyValue ( + Start-Process -FilePath 'dotnet' -ArgumentList $arguments -NoNewWindow -PassThru ` + -RedirectStandardOutput $run.OutFile -RedirectStandardError $run.ErrFile) + $active.Add($run) + } + + $finished = $active | Where-Object { $_.Process.HasExited } + + if (-not $finished) { + Start-Sleep -Milliseconds 500 + continue + } + + foreach ($run in @($finished)) { + # Bounded on purpose. The parameterless WaitForExit() also waits for the redirected streams to + # reach EOF, and on Linux Start-Process pumps them through a pipe, so a test that leaves behind + # a child holding the inherited handle blocks it forever. HasExited has already told us the + # test process itself is done; this only gives the pump a moment to drain. + [void]$run.Process.WaitForExit(5000) + Complete-Run $run + [void]$active.Remove($run) } } diff --git a/tools/select-test-projects.ps1 b/tools/select-test-projects.ps1 index d1f5704f6f..8ae2dc89aa 100644 --- a/tools/select-test-projects.ps1 +++ b/tools/select-test-projects.ps1 @@ -5,6 +5,10 @@ # Writes the selected project paths to $GITHUB_OUTPUT as 'test-projects', and generates an MSBuild # traversal project so that `dotnet build` can build the whole selection in one graph. # +# A category can span several test projects that share infrastructure, so that CI provisions its +# container once and compiles the union of their closures once. Their assemblies then run concurrently, +# via the -MaxParallel switch on run-tests.ps1. +# # Use -List to print every category and its projects without writing any files. [CmdletBinding(DefaultParameterSetName = 'Select')]