Add the missing usage report keys for the primary instance, its storage and its transport

src/ServiceControl.AcceptanceTests/Licensing/When_reporting_the_environment.cs

@@ -16,6 +16,11 @@ namespace ServiceControl.AcceptanceTests.Licensing
     using Particular.LicensingComponent.Contracts;
     using Particular.LicensingComponent.MonitoringThroughput;
     using Particular.LicensingComponent.Shared;
+    using Recoverability.MessageRedirects;
+    using ServiceBus.Management.Infrastructure.Settings;
+    using ServiceControl.Monitoring;
+    using ServiceControl.Persistence;
+    using Conventions = NServiceBus.AcceptanceTesting.Customization.Conventions;
 
     class When_reporting_the_environment : AcceptanceTest
     {
@@ -86,18 +91,137 @@ await Define<Context>()
             "Storage.FullTextSearch",
             "Storage.BodyStorage.Type",
             "Storage.BodyStorage.Auth",
+            "Storage.Auth",
+            "Storage.QueryTimeoutSeconds",
+            "Storage.FreeSpaceThresholdPercent",
+            "Transport.Type",
+            "Transport.Auth",
+            "Transport.CertificateValidation",
             "Security.Authentication",
             "Security.RoleBasedAuthorization",
+            "Security.TokenValidation",
+            "Security.ClaimMapping",
+            "Security.ServicePulseOfflineAccess",
             "Security.Https",
+            "Security.HttpsHardening",
+            "Security.Cors",
+            "Security.ForwardedHeaders",
             "Features.IntegratedServicePulse",
             "Features.MessageEditing",
             "Features.ExternalIntegrationsPublishing",
             "Features.ForwardErrorMessages",
+            "Features.ErrorIngestion",
+            "Features.ConfigurationValidation",
             "Features.EmailNotifications",
+            "ServicePulse.MonitoringUrl",
+            "ServicePulse.DefaultRoute",
+            "ServicePulse.ShowPendingRetry",
+            "Host.VirtualDirectory",
+            "Host.ShutdownTimeoutSeconds",
+            "Logging.Providers",
+            "Logging.Level",
+            "Telemetry.OtlpMetrics",
+            "Limits.ExternalIntegrationsBatchSize",
+            "Ingestion.Error.MaxConcurrency",
+            "Ingestion.Error.BatchSize",
+            "Ingestion.Error.MaxParallelWriters",
+            "Ingestion.Error.BatchTimeoutMs",
+            "Ingestion.Error.RestartAfterFailureSeconds",
+            "Heartbeats.TrackInstancesDefault",
+            "Heartbeats.TrackInstancesOverrides",
+            "Heartbeats.KnownInstances",
+            "Heartbeats.MonitoredInstances",
+            "Heartbeats.GracePeriodSeconds",
+            "Recoverability.Redirects",
+            "Recoverability.RetryHistoryDepth",
+            "Licensing.ReportMasks",
             "Retention.ErrorHours",
             "Retention.EventsHours"
         ];
 
+        [Test]
+        public async Task Should_count_the_choices_made_in_servicepulse_without_revealing_them()
+        {
+            JsonDocument report = null;
+
+            await Define<Context>()
+                .WithEndpoint<MonitoringInstance>()
+                .WithEndpoint<ScalingOut>()
+                .Do("Wait for the first heartbeat of the endpoint that scales out", async _ =>
+                {
+                    var endpoints = await this.TryGetMany<EndpointsView>("/api/endpoints", endpoint => endpoint.Name == ScalingOutEndpoint && endpoint.Monitored);
+
+                    return endpoints.HasResult;
+                })
+                .Do("Stop tracking instances by default", async _ =>
+                {
+                    await this.Patch("/api/endpointssettings", new { track_instances = false });
+
+                    var settings = await this.TryGetMany<SettingsData>("/api/endpointssettings",
+                        setting => setting.Name == string.Empty && !setting.TrackInstances);
+
+                    return settings.HasResult;
+                })
+                .Do("Keep tracking the endpoint that scales out", async _ =>
+                {
+                    await this.Patch($"/api/endpointssettings/{ScalingOutEndpoint}", new { track_instances = true });
+
+                    var settings = await this.TryGetMany<SettingsData>("/api/endpointssettings",
+                        setting => setting.Name == ScalingOutEndpoint && setting.TrackInstances);
+
+                    return settings.HasResult;
+                })
+                .Do("Redirect a retired queue", async _ =>
+                {
+                    await this.Post("/api/redirects", new RedirectRequest { fromphysicaladdress = RetiredQueue, tophysicaladdress = ReplacementQueue });
+
+                    var redirects = await this.TryGetMany<MessageRedirectFromJson>("/api/redirects");
+
+                    return redirects.HasResult;
+                })
+                .Do("Mask the customer's name in the report", async _ =>
+                {
+                    await this.Post("/api/licensing/settings/masks/update", new[] { MaskedWord });
+
+                    return true;
+                })
+                .Do("Wait for the throughput data to be recorded", async _ =>
+                {
+                    var available = await this.TryGet<ReportGenerationState>(
+                        "/api/licensing/report/available", state => state.ReportCanBeGenerated);
+
+                    return available.HasResult;
+                })
+                .Do("Download the report", async _ =>
+                {
+                    var archive = await this.DownloadData("/api/licensing/report/file?spVersion=1.2.3");
+
+                    report = ReadReport(archive);
+
+                    return true;
+                })
+                .Done(_ => true)
+                .Run();
+
+            var data = report.RootElement
+                .GetProperty("ReportData")
+                .GetProperty("EnvironmentInformation")
+                .GetProperty("EnvironmentData")
+                .EnumerateObject()
+                .ToDictionary(entry => entry.Name, entry => entry.Value.GetString());
+
+            using (Assert.EnterMultipleScope())
+            {
+                Assert.That(data["Heartbeats.TrackInstancesDefault"], Is.EqualTo("Disabled"));
+                Assert.That(data["Heartbeats.TrackInstancesOverrides"], Is.EqualTo("1"));
+                Assert.That(int.Parse(data["Heartbeats.MonitoredInstances"]), Is.GreaterThanOrEqualTo(1));
+                Assert.That(int.Parse(data["Heartbeats.KnownInstances"]), Is.GreaterThanOrEqualTo(int.Parse(data["Heartbeats.MonitoredInstances"])));
+                Assert.That(data["Recoverability.Redirects"], Is.EqualTo("1"));
+                Assert.That(data["Licensing.ReportMasks"], Is.EqualTo("1"));
+                Assert.That(string.Join("|", data.Values), Does.Not.Contain(RetiredQueue).And.Not.Contain(ReplacementQueue).And.Not.Contain(MaskedWord).And.Not.Contain(ScalingOutEndpoint));
+            }
+        }
+
         static JsonDocument ReadReport(byte[] archive)
         {
             using var zip = new ZipArchive(new MemoryStream(archive), ZipArchiveMode.Read);
@@ -107,12 +231,23 @@ static JsonDocument ReadReport(byte[] archive)
         }
 
         const string SalesEndpoint = "Particular.Sales";
+        const string RetiredQueue = "Contoso.Billing.Legacy@machine1";
+        const string ReplacementQueue = "Contoso.Billing@machine2";
+        const string MaskedWord = "Contoso";
 
         class Context : ScenarioContext, ISequenceContext
         {
             public int Step { get; set; }
         }
 
+        static string ScalingOutEndpoint => Conventions.EndpointNamingConvention(typeof(ScalingOut));
+
+        class ScalingOut : EndpointConfigurationBuilder
+        {
+            public ScalingOut() =>
+                EndpointSetup<DefaultServerWithoutAudit>(c => c.SendHeartbeatTo(Settings.DEFAULT_INSTANCE_NAME));
+        }
+
         class MonitoringInstance : EndpointConfigurationBuilder
         {
             public MonitoringInstance() =>

src/ServiceControl.Infrastructure/OpenIdConnectSettings.cs

@@ -11,6 +11,10 @@ namespace ServiceControl.Infrastructure;
 /// </summary>
 public class OpenIdConnectSettings
 {
+    public const string DefaultRolesClaim = "roles";
+    public const string DefaultSubjectIdClaim = "sub";
+    public const string DefaultSubjectNameClaim = "preferred_username";
+
     readonly ILogger logger = LoggerUtil.CreateStaticLogger<OpenIdConnectSettings>();
 
     /// <summary>
@@ -33,15 +37,15 @@ public OpenIdConnectSettings(SettingsRootNamespace rootNamespace, bool validateC
         ValidateIssuerSigningKey = SettingsReader.Read(rootNamespace, "Authentication.ValidateIssuerSigningKey", true);
         RequireHttpsMetadata = SettingsReader.Read(rootNamespace, "Authentication.RequireHttpsMetadata", true);
 
-        RolesClaim = SettingsReader.Read(rootNamespace, "Authentication.RolesClaim", "roles");
+        RolesClaim = SettingsReader.Read(rootNamespace, "Authentication.RolesClaim", DefaultRolesClaim);
         RoleBasedAuthorizationEnabled = SettingsReader.Read(rootNamespace, "Authentication.RoleBasedAuthorizationEnabled", false);
 
         // Claims that identify the principal in the authorization audit log. The handler treats both
         // as required — a missing or empty value is a sign that the IdP isn't emitting the expected
         // claim and the operator needs to fix the configuration, so the handler will throw rather
         // than substitute a placeholder.
-        SubjectIdClaim = SettingsReader.Read(rootNamespace, "Authentication.SubjectIdClaim", "sub");
-        SubjectNameClaim = SettingsReader.Read(rootNamespace, "Authentication.SubjectNameClaim", "preferred_username");
+        SubjectIdClaim = SettingsReader.Read(rootNamespace, "Authentication.SubjectIdClaim", DefaultSubjectIdClaim);
+        SubjectNameClaim = SettingsReader.Read(rootNamespace, "Authentication.SubjectNameClaim", DefaultSubjectNameClaim);
 
         // ServicePulse settings are only relevant for the primary ServiceControl instance
         // which serves the OIDC configuration endpoint that ServicePulse uses for login

src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlPersistence.cs

@@ -5,6 +5,7 @@ namespace ServiceControl.Persistence.EFCore.PostgreSql;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;
 using Microsoft.Extensions.DependencyInjection;
+using Particular.LicensingComponent.Contracts;
 using ServiceControl.Persistence.EFCore.Abstractions;
 using ServiceControl.Persistence.EFCore.DbContexts;
 using ServiceControl.Persistence.EFCore.Infrastructure;
@@ -17,6 +18,7 @@ public void AddPersistence(IServiceCollection services)
         ConfigureDbContext(services);
         RegisterDataStores(services, settings);
 
+        services.AddSingleton<IEnvironmentDataProvider, PostgreSqlStorageAuthenticationEnvironmentDataProvider>();
         services.AddSingleton<IFailedMessageIngestionSqlDialect, PostgreSqlFailedMessageIngestionSqlDialect>();
         services.AddSingleton<IRetryBatchSqlDialect, PostgreSqlRetryBatchSqlDialect>();
         services.AddSingleton<IFullTextSearchDialect, PostgreSqlFullTextSearchDialect>();

src/ServiceControl.Persistence.EFCore.PostgreSql/PostgreSqlStorageAuthenticationEnvironmentDataProvider.cs

@@ -0,0 +1,25 @@
+namespace ServiceControl.Persistence.EFCore.PostgreSql;
+
+using Npgsql;
+using Particular.LicensingComponent.Contracts;
+using static Particular.LicensingComponent.Contracts.EnvironmentDatum;
+
+class PostgreSqlStorageAuthenticationEnvironmentDataProvider(PostgreSqlPersisterSettings settings) : IEnvironmentDataProvider
+{
+    public IEnumerable<EnvironmentDatum> GetData() =>
+    [
+        Value("Storage.Auth", Authentication)
+    ];
+
+    string Authentication()
+    {
+        var builder = new NpgsqlConnectionStringBuilder(settings.ConnectionString);
+
+        if (!string.IsNullOrEmpty(builder.SslCertificate))
+        {
+            return "ClientCertificate";
+        }
+
+        return !string.IsNullOrEmpty(builder.Password) || !string.IsNullOrEmpty(builder.Passfile) ? "Password" : "None";
+    }
+}

src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerPersistence.cs

@@ -5,6 +5,7 @@ namespace ServiceControl.Persistence.EFCore.SqlServer;
 using Microsoft.EntityFrameworkCore.Infrastructure;
 using Microsoft.EntityFrameworkCore.Migrations;
 using Microsoft.Extensions.DependencyInjection;
+using Particular.LicensingComponent.Contracts;
 using ServiceControl.Persistence.EFCore.Abstractions;
 using ServiceControl.Persistence.EFCore.DbContexts;
 using ServiceControl.Persistence.EFCore.Infrastructure;
@@ -17,6 +18,7 @@ public void AddPersistence(IServiceCollection services)
         ConfigureDbContext(services);
         RegisterDataStores(services, settings);
 
+        services.AddSingleton<IEnvironmentDataProvider, SqlServerStorageAuthenticationEnvironmentDataProvider>();
         services.AddSingleton<IFailedMessageIngestionSqlDialect, SqlServerFailedMessageIngestionSqlDialect>();
         services.AddSingleton<IRetryBatchSqlDialect, SqlServerRetryBatchSqlDialect>();
         services.AddSingleton<IFullTextSearchDialect, SqlServerFullTextSearchDialect>();

src/ServiceControl.Persistence.EFCore.SqlServer/SqlServerStorageAuthenticationEnvironmentDataProvider.cs

@@ -0,0 +1,25 @@
+namespace ServiceControl.Persistence.EFCore.SqlServer;
+
+using Microsoft.Data.SqlClient;
+using Particular.LicensingComponent.Contracts;
+using static Particular.LicensingComponent.Contracts.EnvironmentDatum;
+
+class SqlServerStorageAuthenticationEnvironmentDataProvider(SqlServerPersisterSettings settings) : IEnvironmentDataProvider
+{
+    public IEnumerable<EnvironmentDatum> GetData() =>
+    [
+        Value("Storage.Auth", Authentication)
+    ];
+
+    string Authentication()
+    {
+        var builder = new SqlConnectionStringBuilder(settings.ConnectionString);
+
+        if (builder.Authentication is SqlAuthenticationMethod.NotSpecified)
+        {
+            return builder.IntegratedSecurity ? "Integrated" : "SqlPassword";
+        }
+
+        return builder.Authentication is SqlAuthenticationMethod.SqlPassword ? "SqlPassword" : "EntraId";
+    }
+}

src/ServiceControl.Persistence.EFCore/Abstractions/EFPersistenceConfigurationBase.cs

@@ -7,11 +7,11 @@ namespace ServiceControl.Persistence.EFCore.Abstractions;
 public abstract class EFPersistenceConfigurationBase : PersistenceConfiguration, IPersistenceConfiguration
 {
     const string ConnectionStringKey = "Database/ConnectionString";
-    const string SchemaKey = "Database/Schema";
-    const string CommandTimeoutKey = "Database/CommandTimeout";
+    internal const string SchemaKey = "Database/Schema";
+    internal const string CommandTimeoutKey = "Database/CommandTimeout";
     const string BodyStorageTypeKey = "MessageBody/StorageType";
     const string FileSystemStoragePathKey = "MessageBody/FileSystem/StoragePath";
-    const string FileSystemDataSpaceRemainingThresholdKey = "MessageBody/FileSystem/DataSpaceRemainingThreshold";
+    internal const string FileSystemDataSpaceRemainingThresholdKey = "MessageBody/FileSystem/DataSpaceRemainingThreshold";
     const string AzureConnectionStringKey = "MessageBody/Azure/ConnectionString";
     const string AzureServiceUriKey = "MessageBody/Azure/ServiceUri";
     const string AzureManagedIdentityClientIdKey = "MessageBody/Azure/ManagedIdentityClientId";
@@ -23,11 +23,11 @@ public abstract class EFPersistenceConfigurationBase : PersistenceConfiguration,
     const string S3ServiceUrlKey = "MessageBody/S3/ServiceUrl";
     const string S3AccessKeyIdKey = "MessageBody/S3/AccessKeyId";
     const string S3SecretAccessKeyKey = "MessageBody/S3/SecretAccessKey";
-    const string MinBodySizeForCompressionKey = "MessageBody/MinCompressionSize";
+    internal const string MinBodySizeForCompressionKey = "MessageBody/MinCompressionSize";
     const string MaxBodySizeToStoreKey = "MaxBodySizeToStore";
     const string ErrorRetentionPeriodKey = "ErrorRetentionPeriod";
     const string EventsRetentionPeriodKey = "EventsRetentionPeriod";
-    const string SubscriptionCacheDurationKey = "SubscriptionCacheDuration";
+    internal const string SubscriptionCacheDurationKey = "SubscriptionCacheDuration";
     const string ExternalIntegrationsDispatchingBatchSizeKey = "ExternalIntegrationsDispatchingBatchSize";
 
     public bool SupportsMaintenanceMode => false;

src/ServiceControl.Persistence.EFCore/Implementation/EFEnvironmentDataProvider.cs

@@ -4,6 +4,8 @@ namespace ServiceControl.Persistence.EFCore.Implementation;
 using Abstractions;
 using Infrastructure;
 using Particular.LicensingComponent.Contracts;
+using ServiceControl.Configuration;
+using ServiceControl.Infrastructure;
 using static Particular.LicensingComponent.Contracts.EnvironmentDatum;
 
 class EFEnvironmentDataProvider(EFPersisterSettings settings, IDatabaseHostingProbe hostingProbe) : IEnvironmentDataProvider
@@ -24,10 +26,28 @@ public IEnumerable<EnvironmentDatum> GetData()
             Value("Storage.FullTextSearch", () => settings.EnableFullTextSearchOnBodies ? "Enabled" : "Disabled"),
             Value("Storage.BodyStorage.Type", () => BodyStorageType(settings.BodyStorage)),
             Value("Storage.BodyStorage.Auth", () => BodyStorageAuth(settings.BodyStorage)),
-            Value("Limits.MaxBodySizeToStore", () => settings.BodyStorage.MaxBodySizeToStore.ToString(CultureInfo.InvariantCulture))
+            Value("Limits.MaxBodySizeToStore", () => settings.BodyStorage.MaxBodySizeToStore.ToString(CultureInfo.InvariantCulture)),
+            Value("Storage.Schema", () => settings.Schema is null ? "Default" : "Custom"),
+            Value("Storage.CommandTimeoutSeconds", () => WhenConfigured(EFPersistenceConfigurationBase.CommandTimeoutKey, () => Number(settings.CommandTimeout))),
+            Value("Storage.QueryTimeoutSeconds", () => WhenConfigured(QueryTimeLimit.SettingName, () => Seconds(settings.QueryTimeout))),
+            Value("Storage.SubscriptionCacheSeconds", () => WhenConfigured(EFPersistenceConfigurationBase.SubscriptionCacheDurationKey, () => Seconds(settings.SubscriptionCacheDuration))),
+            Value("Storage.BodyStorage.MinCompressionBytes", () => WhenConfigured(EFPersistenceConfigurationBase.MinBodySizeForCompressionKey, () => Number(settings.BodyStorage.MinCompressionSize))),
+            Value("Storage.FreeSpaceThresholdPercent", FreeSpaceThreshold)
         ];
     }
 
+    string FreeSpaceThreshold() => settings.BodyStorage is FileSystemBodyStorageSettings fileSystem
+        ? WhenConfigured(EFPersistenceConfigurationBase.FileSystemDataSpaceRemainingThresholdKey, () => Number(fileSystem.DataSpaceRemainingThreshold))
+        : "NotApplicable";
+
+    static string WhenConfigured(string key, Func<string> readValue) =>
+        SettingsReader.TryRead<string>(SettingsNamespace, key, out _) ? readValue() : "Default";
+
+    static string Number(int value) => value.ToString(CultureInfo.InvariantCulture);
+
+    static string Seconds(TimeSpan value) =>
+        Math.Round(value.TotalSeconds, MidpointRounding.AwayFromZero).ToString("F0", CultureInfo.InvariantCulture);
+
     static string BodyStorageType(BodyStorageSettings bodyStorage) => bodyStorage switch
     {
         FileSystemBodyStorageSettings => nameof(Abstractions.BodyStorageType.FileSystem),
@@ -47,4 +67,6 @@ public IEnumerable<EnvironmentDatum> GetData()
         S3BodyStorageSettings s3 => s3.Credentials is null ? "IamRole" : "StaticCredentials",
         _ => "NotApplicable"
     };
+
+    static readonly SettingsRootNamespace SettingsNamespace = new("ServiceControl");
 }