Add configured and live audit instance counts to the usage report

src/Particular.LicensingComponent.Contracts/AuditServiceMetadata.cs

@@ -1,3 +1,7 @@
 namespace Particular.LicensingComponent.Contracts;
 
-public record AuditServiceMetadata(Dictionary<string, int> Versions, Dictionary<string, int> Transports);
+public record AuditServiceMetadata(Dictionary<string, int> Versions, Dictionary<string, int> Transports)
+{
+    public int? ConfiguredInstances { get; init; }
+    public int? LiveInstances { get; init; }
+}

src/Particular.LicensingComponent.Contracts/RemoteInstanceInformation.cs

@@ -11,4 +11,5 @@ public class RemoteInstanceInformation
     public SemanticVersion? SemanticVersion { get; set; }
     public TimeSpan Retention { get; set; }
     public string? Transport { get; set; }
+    public bool RespondedAsAuditInstance { get; set; }
 }

src/Particular.LicensingComponent.UnitTests/AuditQuery_Tests.cs

@@ -73,6 +73,33 @@ public async Task Should_return_audit_remotes()
         }
     }
 
+    [Test]
+    public async Task Should_flag_only_remotes_whose_configuration_has_an_audit_queue_as_audit_instances()
+    {
+        var configurationApi = new ConfigurationApi_ReturningRemotes(
+            new RemoteConfiguration { ApiUri = "http://audit:44444/api/", Status = "online", Version = "6.2.0", Configuration = JsonNode.Parse("""{"transport":{"transport_type":"RabbitMQ.QuorumConventionalRouting","audit_queue":"audit"}}""") },
+            new RemoteConfiguration { ApiUri = "http://primary:33333/api/", Status = "online", Version = "6.2.0", Configuration = JsonNode.Parse("""{"transport":{"transport_type":"RabbitMQ.QuorumConventionalRouting","error_queue":"error"}}""") },
+            new RemoteConfiguration { ApiUri = "http://decommissioned:44444/api/", Status = "unavailable", Version = "Unknown" });
+        var auditQuery = new AuditQuery(NullLogger<AuditQuery>.Instance, new FakeEndpointApi(), new FakeAuditCountApi(), configurationApi);
+
+        var remotes = await auditQuery.GetAuditRemotes();
+
+        Assert.That(remotes.Select(remote => remote.RespondedAsAuditInstance), Is.EqualTo(new[] { true, false, false }));
+    }
+
+    [Test]
+    public async Task Should_return_each_remote_with_only_its_own_queues()
+    {
+        var configurationApi = new ConfigurationApi_ReturningRemotes(
+            new RemoteConfiguration { ApiUri = "http://audit-a:44444/api/", Status = "online", Version = "6.2.0", Configuration = JsonNode.Parse("""{"transport":{"audit_queue":"audit.a","audit_log_queue":"audit.a.log"}}""") },
+            new RemoteConfiguration { ApiUri = "http://audit-b:44444/api/", Status = "online", Version = "6.2.0", Configuration = JsonNode.Parse("""{"transport":{"audit_queue":"audit.b","audit_log_queue":"audit.b.log"}}""") });
+        var auditQuery = new AuditQuery(NullLogger<AuditQuery>.Instance, new FakeEndpointApi(), new FakeAuditCountApi(), configurationApi);
+
+        var remotes = await auditQuery.GetAuditRemotes();
+
+        Assert.That(remotes.Select(remote => remote.Queues), Is.EqualTo(new[] { new[] { "audit.a", "audit.a.log" }, new[] { "audit.b", "audit.b.log" } }));
+    }
+
     [Test]
     public async Task Should_return_successful_audit_connection_if_instances_exist_and_are_online()
     {
@@ -167,6 +194,15 @@ public Task<RemoteConfiguration[]> GetRemoteConfigs(CancellationToken cancellati
         public Task<RootUrls> GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException();
     }
 
+    class ConfigurationApi_ReturningRemotes(params RemoteConfiguration[] remotes) : IConfigurationApi
+    {
+        public Task<object> GetConfig(CancellationToken cancellationToken = default) => throw new NotImplementedException();
+
+        public Task<RemoteConfiguration[]> GetRemoteConfigs(CancellationToken cancellationToken = default) => Task.FromResult(remotes);
+
+        public Task<RootUrls> GetUrls(string baseUrl, CancellationToken cancellationToken = default) => throw new NotImplementedException();
+    }
+
     class ConfigurationApi_Configurable : IConfigurationApi
     {
         public Task<object> GetConfig(CancellationToken cancellationToken = default) => throw new NotImplementedException();

src/Particular.LicensingComponent.UnitTests/AuditThroughputCollectorHostedService_Tests.cs

@@ -245,6 +245,33 @@ await Task.Run(async () =>
         }
     }
 
+    [Test]
+    public async Task Should_store_how_many_audit_instances_are_configured_and_how_many_are_live()
+    {
+        var liveAuditInstance = new RemoteInstanceInformation { Status = "online", VersionString = "6.2.0", SemanticVersion = new SemanticVersion(6, 2, 0), RespondedAsAuditInstance = true };
+        var primaryInstanceListedAsRemote = new RemoteInstanceInformation { Status = "online", VersionString = "6.2.0", SemanticVersion = new SemanticVersion(6, 2, 0) };
+        var auditInstanceWithoutVersionHeader = new RemoteInstanceInformation { Status = "online", VersionString = "Missing", RespondedAsAuditInstance = true };
+        var unreachableRemote = new RemoteInstanceInformation { Status = "unavailable", VersionString = "Unknown" };
+        var auditQuery = new AuditQuery_WithRemotes([liveAuditInstance, primaryInstanceListedAsRemote, auditInstanceWithoutVersionHeader, unreachableRemote]);
+
+        using var auditThroughputCollectorHostedService = new AuditThroughputCollectorHostedService(
+            NullLogger<AuditThroughputCollectorHostedService>.Instance, configuration.ThroughputSettings, DataStore,
+            auditQuery, new FakeTimeProvider())
+        { DelayStart = TimeSpan.Zero };
+
+        await auditThroughputCollectorHostedService.StartAsync(CancellationToken.None);
+        await auditQuery.KnownEndpointsRequested.WaitAsync(TimeSpan.FromSeconds(30));
+        await auditThroughputCollectorHostedService.StopAsync(CancellationToken.None);
+
+        var auditServiceMetadata = await DataStore.GetAuditServiceMetadata();
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(auditServiceMetadata.ConfiguredInstances, Is.EqualTo(4));
+            Assert.That(auditServiceMetadata.LiveInstances, Is.EqualTo(1));
+        }
+    }
+
     class AuditQuery_NoAuditRemotes : IAuditQuery
     {
         public SemanticVersion MinAuditCountsVersion => new(4, 29, 0);
@@ -423,4 +450,30 @@ public Task<ConnectionSettingsTestResult> TestAuditConnection(CancellationToken
         readonly DateOnly throughputDate;
         readonly long throughputCount;
     }
+
+    class AuditQuery_WithRemotes(List<RemoteInstanceInformation> remotes) : IAuditQuery
+    {
+        public Task KnownEndpointsRequested => knownEndpointsRequested.Task;
+
+        public SemanticVersion MinAuditCountsVersion => new(4, 29, 0);
+        public Func<RemoteInstanceInformation, bool> ValidRemoteInstances => _ => true;
+
+        public Task<List<RemoteInstanceInformation>> GetAuditRemotes(CancellationToken cancellationToken = default) =>
+            Task.FromResult(remotes);
+
+        public Task<IEnumerable<ServiceControlEndpoint>> GetKnownEndpoints(CancellationToken cancellationToken = default)
+        {
+            knownEndpointsRequested.TrySetResult();
+
+            return Task.FromResult<IEnumerable<ServiceControlEndpoint>>([]);
+        }
+
+        public Task<IEnumerable<AuditCount>> GetAuditCountForEndpoint(string endpointUrlName, CancellationToken cancellationToken = default) =>
+            Task.FromResult<IEnumerable<AuditCount>>([]);
+
+        public Task<ConnectionSettingsTestResult> TestAuditConnection(CancellationToken cancellationToken = default) =>
+            throw new NotImplementedException();
+
+        readonly TaskCompletionSource knownEndpointsRequested = new(TaskCreationOptions.RunContinuationsAsynchronously);
+    }
 }

src/Particular.LicensingComponent.UnitTests/ThroughputCollector/ThroughputCollector_Report_EnvironmentInformation_Tests.cs

@@ -196,4 +196,36 @@ await DataStore.CreateBuilder()
         Assert.That(report.ReportData.ScopeType, Is.Not.Null, $"Missing ScopeType from report");
         Assert.That(report.ReportData.ScopeType, Is.EqualTo(expectedScopeType), $"Invalid ScopeType on report");
     }
+
+    [Test]
+    public async Task Should_include_audit_instance_counts_in_environment_data()
+    {
+        await DataStore.SaveAuditServiceMetadata(new AuditServiceMetadata([], []) { ConfiguredInstances = 50, LiveInstances = 2 });
+
+        var report = await ThroughputCollector.GenerateThroughputReport("", null);
+
+        var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData;
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(environmentData, Does.ContainKey("Audit.ConfiguredInstances").WithValue("50"));
+            Assert.That(environmentData, Does.ContainKey("Audit.LiveInstances").WithValue("2"));
+        }
+    }
+
+    [Test]
+    public async Task Should_leave_audit_instance_counts_out_of_environment_data_when_none_are_stored()
+    {
+        await DataStore.SaveAuditServiceMetadata(new AuditServiceMetadata(
+            new Dictionary<string, int> { ["6.2.0"] = 1 },
+            new Dictionary<string, int> { ["RabbitMQ.QuorumConventionalRouting"] = 1 }));
+
+        var report = await ThroughputCollector.GenerateThroughputReport("", null);
+
+        var environmentData = report.ReportData.EnvironmentInformation.EnvironmentData;
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(environmentData, Does.Not.ContainKey("Audit.ConfiguredInstances"));
+            Assert.That(environmentData, Does.Not.ContainKey("Audit.LiveInstances"));
+        }
+    }
 }

src/Particular.LicensingComponent/AuditThroughput/AuditQuery.cs

@@ -47,12 +47,12 @@ public async Task<List<RemoteInstanceInformation>> GetAuditRemotes(CancellationT
 
                 if (remotes.Any())
                 {
-                    List<string> queues = [];
-
                     foreach (var remote in remotes)
                     {
+                        List<string> queues = [];
                         string? retention = null;
                         string? transportTypeUsed = null;
+                        var respondedAsAuditInstance = false;
                         if (remote.Configuration != null)
                         {
                             retention = remote.Configuration.AsObject().TryGetPropertyValue("data_retention", out var dataRetention) &&
@@ -71,6 +71,7 @@ public async Task<List<RemoteInstanceInformation>> GetAuditRemotes(CancellationT
                                 if (transport?.AsObject().TryGetPropertyValue("audit_queue", out var auditQueue) == true)
                                 {
                                     queues.Add(auditQueue!.GetValue<string>());
+                                    respondedAsAuditInstance = true;
                                 }
                                 if (transport?.AsObject().TryGetPropertyValue("audit_log_queue", out var auditLogQueue) == true)
                                 {
@@ -90,7 +91,8 @@ public async Task<List<RemoteInstanceInformation>> GetAuditRemotes(CancellationT
                             Status = remote.Status,
                             Retention = TimeSpan.TryParse(retention, out var ts) ? ts : TimeSpan.Zero,
                             Queues = queues,
-                            Transport = transportTypeUsed ?? ""
+                            Transport = transportTypeUsed ?? "",
+                            RespondedAsAuditInstance = respondedAsAuditInstance
                         };
 
                         remoteInstance.SemanticVersion = SemanticVersion.TryParse(remoteInstance.VersionString ?? string.Empty, out var v) ? v : null;

src/Particular.LicensingComponent/AuditThroughput/AuditThroughputCollectorHostedService.cs

@@ -155,7 +155,13 @@ async Task SaveAuditInstanceData(List<RemoteInstanceInformation>? auditRemotes,
                 .GroupBy(s => s.Transport!)
                 .ToDictionary(g => g.Key, g => g.Count());
 
-            await dataStore.SaveAuditServiceMetadata(new AuditServiceMetadata(versions, transports), cancellationToken);
+            var auditServiceMetadata = new AuditServiceMetadata(versions, transports)
+            {
+                ConfiguredInstances = auditRemotes.Count,
+                LiveInstances = auditRemotes.Count(remote => remote.RespondedAsAuditInstance && remote.SemanticVersion is not null)
+            };
+
+            await dataStore.SaveAuditServiceMetadata(auditServiceMetadata, cancellationToken);
         }
     }
 }

src/Particular.LicensingComponent/ThroughputCollector.cs

@@ -1,6 +1,7 @@
 namespace Particular.LicensingComponent;
 
 using System;
+using System.Globalization;
 using System.Runtime.CompilerServices;
 using System.Threading;
 using AuditThroughput;
@@ -186,6 +187,16 @@ public async Task<SignedReport> GenerateThroughputReport(string spVersion, DateT
         report.EnvironmentInformation.EnvironmentData[EnvironmentDataType.AuditEnabled.ToString()] = systemHasAuditEnabled.ToString();
         report.EnvironmentInformation.EnvironmentData[EnvironmentDataType.MonitoringEnabled.ToString()] = systemHasMonitoringEnabled.ToString();
 
+        if (auditServiceMetadata.ConfiguredInstances is { } configuredAuditInstances)
+        {
+            report.EnvironmentInformation.EnvironmentData["Audit.ConfiguredInstances"] = configuredAuditInstances.ToString(CultureInfo.InvariantCulture);
+        }
+
+        if (auditServiceMetadata.LiveInstances is { } liveAuditInstances)
+        {
+            report.EnvironmentInformation.EnvironmentData["Audit.LiveInstances"] = liveAuditInstances.ToString(CultureInfo.InvariantCulture);
+        }
+
         foreach (var environmentDataProvider in environmentDataProviders)
         {
             EnvironmentDatum[] environmentData;

src/ServiceControl.Persistence.Tests.RavenDB/Throughput/AuditServiceMetadataCompatibilityTests.cs

@@ -0,0 +1,38 @@
+namespace ServiceControl.Persistence.Tests.RavenDB.Throughput;
+
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using NUnit.Framework;
+using ServiceControl.Persistence.RavenDB.Throughput;
+
+[TestFixture]
+class AuditServiceMetadataCompatibilityTests : RavenPersistenceTestBase
+{
+    [Test]
+    public async Task Audit_service_metadata_saved_before_the_instance_counts_existed_reads_back_without_them()
+    {
+        var throughputDatabase = ServiceProvider.GetRequiredService<ThroughputDatabaseConfiguration>().Name;
+
+        using (var session = DocumentStore.OpenAsyncSession(throughputDatabase))
+        {
+            var metadataWithoutCounts = new AuditServiceMetadataBeforeInstanceCounts(
+                new Dictionary<string, int> { ["6.2.0"] = 2 },
+                new Dictionary<string, int> { ["RabbitMQ.QuorumConventionalRouting"] = 2 });
+
+            await session.StoreAsync(metadataWithoutCounts, "AuditServiceMetadata");
+            await session.SaveChangesAsync();
+        }
+
+        var auditServiceMetadata = await LicensingDataStore.GetAuditServiceMetadata();
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(auditServiceMetadata.Versions, Is.EquivalentTo(new Dictionary<string, int> { ["6.2.0"] = 2 }));
+            Assert.That(auditServiceMetadata.ConfiguredInstances, Is.Null);
+            Assert.That(auditServiceMetadata.LiveInstances, Is.Null);
+        }
+    }
+
+    record AuditServiceMetadataBeforeInstanceCounts(Dictionary<string, int> Versions, Dictionary<string, int> Transports);
+}

src/ServiceControl.Persistence.Tests/EFCore/LicensingDataStoreEFTests.cs

@@ -1,9 +1,14 @@
 namespace ServiceControl.Persistence.Tests;
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using EFCore.DbContexts;
+using EFCore.Entities;
+using EFCore.Infrastructure;
+using Microsoft.Extensions.DependencyInjection;
 using NUnit.Framework;
 using Particular.LicensingComponent.Contracts;
 
@@ -203,6 +208,26 @@ await LicensingDataStore.SaveEndpoint(
         }
     }
 
+    [Test]
+    public async Task Audit_service_metadata_saved_before_the_instance_counts_existed_reads_back_without_them()
+    {
+        await using (var scope = ServiceProvider.CreateAsyncScope())
+        {
+            var dbContext = scope.ServiceProvider.GetRequiredService<ServiceControlDbContext>();
+            dbContext.Settings.Add(new SettingEntity { Key = SettingKeys.AuditServiceMetadata, Value = """{"Versions":{"6.2.0":2},"Transports":{"RabbitMQ.QuorumConventionalRouting":2}}""" });
+            await dbContext.SaveChangesAsync();
+        }
+
+        var auditServiceMetadata = await LicensingDataStore.GetAuditServiceMetadata();
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(auditServiceMetadata.Versions, Is.EquivalentTo(new Dictionary<string, int> { ["6.2.0"] = 2 }));
+            Assert.That(auditServiceMetadata.ConfiguredInstances, Is.Null);
+            Assert.That(auditServiceMetadata.LiveInstances, Is.Null);
+        }
+    }
+
     Task SaveEndpoint(string name, ThroughputSource source) =>
         LicensingDataStore.SaveEndpoint(new Endpoint(name, source) { SanitizedName = name });
 

src/ServiceControl.Persistence.Tests/Throughput/AuditServiceMetadataTests.cs

@@ -30,6 +30,20 @@ public async Task Should_retrieve_saved_audit_service_metadata()
         }
     }
 
+    [Test]
+    public async Task Should_retrieve_saved_audit_instance_counts()
+    {
+        await LicensingDataStore.SaveAuditServiceMetadata(new AuditServiceMetadata([], []) { ConfiguredInstances = 50, LiveInstances = 2 });
+
+        var retrievedAuditServiceMetadata = await LicensingDataStore.GetAuditServiceMetadata();
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(retrievedAuditServiceMetadata.ConfiguredInstances, Is.EqualTo(50));
+            Assert.That(retrievedAuditServiceMetadata.LiveInstances, Is.EqualTo(2));
+        }
+    }
+
     [Test]
     public async Task Should_update_existing_audit_service_metadata_if_already_exists()
     {