diff --git a/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj b/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj index 5d44f04283..9d6f20686e 100644 --- a/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj +++ b/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj @@ -51,6 +51,11 @@ $(DefineConstants);WINDOWS_UWP + + + $(DefineConstants);WIN_UI + + diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs index f15fd04dda..4fb87cc30e 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs @@ -9,8 +9,10 @@ using Microsoft.Testing.Platform.Logging; using Microsoft.Testing.Platform.Messages; using Microsoft.Testing.Platform.Services; +using Microsoft.Testing.Platform.Telemetry; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Helpers; using Microsoft.VisualStudio.TestPlatform.ObjectModel; @@ -41,8 +43,7 @@ protected override Task SynchronizedDiscoverTestsAsync(VSTestDiscoverTestExecuti Debugger.Launch(); } - new MSTestDiscoverer().DiscoverTests(request.AssemblyPaths, request.DiscoveryContext, request.MessageLogger, request.DiscoverySink, _configuration, isMTP: true); - return Task.CompletedTask; + return new MSTestDiscoverer(new TestSourceHandler(), CreateTelemetrySender()).DiscoverTestsAsync(request.AssemblyPaths, request.DiscoveryContext, request.MessageLogger, request.DiscoverySink, _configuration, isMTP: true); } /// @@ -55,7 +56,7 @@ protected override async Task SynchronizedRunTestsAsync(VSTestRunTestExecutionRe Debugger.Launch(); } - MSTestExecutor testExecutor = new(cancellationToken); + MSTestExecutor testExecutor = new(cancellationToken, CreateTelemetrySender()); await testExecutor.RunTestsAsync(request.AssemblyPaths, request.RunContext, request.FrameworkHandle, _configuration, isMTP: true).ConfigureAwait(false); } @@ -103,5 +104,19 @@ private static TestMethodIdentifierProperty GetMethodIdentifierPropertyFromManag // Or alternatively, does VSTest object model expose the assembly full name somewhere? return new TestMethodIdentifierProperty(assemblyFullName: string.Empty, @namespace, typeName, methodName, arity, parameterTypes, returnTypeFullName: string.Empty); } + + [SuppressMessage("ApiDesign", "RS0030:Do not use banned APIs", Justification = "We can use MTP from this folder")] + private Func, Task>? CreateTelemetrySender() + { + ITelemetryInformation telemetryInformation = ServiceProvider.GetTelemetryInformation(); + if (!telemetryInformation.IsEnabled) + { + return null; + } + + ITelemetryCollector telemetryCollector = ServiceProvider.GetTelemetryCollector(); + + return (eventName, metrics) => telemetryCollector.LogEventAsync(eventName, metrics, CancellationToken.None); + } } #endif diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs index 5b46c15750..a98e00c83a 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs @@ -20,14 +20,27 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; internal sealed class MSTestDiscoverer : ITestDiscoverer { private readonly ITestSourceHandler _testSourceHandler; +#if !WINDOWS_UWP && !WIN_UI + private readonly Func, Task>? _telemetrySender; +#endif + // The parameterless constructor is required by VSTest, which instantiates the + // discoverer via reflection. The internal constructor exists for tests and for the + // MTP bridge (MSTestBridgedTestFramework) which injects a telemetry sender. public MSTestDiscoverer() : this(new TestSourceHandler()) { } - internal /* for testing purposes */ MSTestDiscoverer(ITestSourceHandler testSourceHandler) - => _testSourceHandler = testSourceHandler; + internal MSTestDiscoverer(ITestSourceHandler testSourceHandler, Func, Task>? telemetrySender = null) + { + _testSourceHandler = testSourceHandler; +#if !WINDOWS_UWP && !WIN_UI + _telemetrySender = telemetrySender; +#else + _ = telemetrySender; +#endif + } /// /// Discovers the tests available from the provided source. Not supported for .xap source. @@ -39,9 +52,12 @@ public MSTestDiscoverer() [System.Security.SecurityCritical] [SuppressMessage("Microsoft.Design", "CA1062:Validate arguments of public methods", MessageId = "0", Justification = "Discovery context can be null.")] public void DiscoverTests(IEnumerable sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink) - => DiscoverTests(sources, discoveryContext, logger, discoverySink, null, isMTP: false); + // VSTest's ITestDiscoverer is a synchronous interface. The telemetry sender is null in + // this code path (only the MTP bridge supplies one), so the awaited send below completes + // synchronously and GetAwaiter().GetResult() does not actually block on I/O. + => DiscoverTestsAsync(sources, discoveryContext, logger, discoverySink, configuration: null, isMTP: false).GetAwaiter().GetResult(); - internal void DiscoverTests(IEnumerable sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink, IConfiguration? configuration, bool isMTP) + internal async Task DiscoverTestsAsync(IEnumerable sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink, IConfiguration? configuration, bool isMTP) { if (sources is null) { @@ -58,9 +74,31 @@ internal void DiscoverTests(IEnumerable sources, IDiscoveryContext disco throw new ArgumentNullException(nameof(discoverySink)); } - if (MSTestDiscovererHelpers.InitializeDiscovery(sources, discoveryContext, logger, configuration, _testSourceHandler)) + // Initialize telemetry collection if not already set (e.g. first call in the session). +#if !WINDOWS_UWP && !WIN_UI + if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) + { + _ = MSTestTelemetryDataCollector.EnsureInitialized(); + } +#endif + + try + { + if (MSTestDiscovererHelpers.InitializeDiscovery(sources, discoveryContext, logger, configuration, _testSourceHandler)) + { + new UnitTestDiscoverer(_testSourceHandler).DiscoverTests(sources, logger, discoverySink, discoveryContext, isMTP); + } + } + finally { - new UnitTestDiscoverer(_testSourceHandler).DiscoverTests(sources, logger, discoverySink, discoveryContext, isMTP); +#if !WINDOWS_UWP && !WIN_UI + // Send the discovery telemetry event ('mstest/discovery'). This always runs at the + // end of discovery — for discover-only sessions it is the only event; for sessions + // where a run follows, MSTestExecutor will send a separate 'mstest/sessionexit' event + // carrying assertion usage. Keeping the two events distinct avoids settings/attribute + // duplication and lets each event be self-contained. + await MSTestTelemetryDataCollector.SendDiscoveryTelemetryAndResetAsync(_telemetrySender).ConfigureAwait(false); +#endif } } } diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs index 903d9f9313..f85044c5f8 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs @@ -20,6 +20,9 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; internal sealed class MSTestExecutor : ITestExecutor { private readonly CancellationToken _cancellationToken; +#if !WINDOWS_UWP && !WIN_UI + private readonly Func, Task>? _telemetrySender; +#endif /// /// Token for canceling the test run. @@ -35,10 +38,15 @@ public MSTestExecutor() _cancellationToken = CancellationToken.None; } - internal MSTestExecutor(CancellationToken cancellationToken) + internal MSTestExecutor(CancellationToken cancellationToken, Func, Task>? telemetrySender = null) { TestExecutionManager = new TestExecutionManager(); _cancellationToken = cancellationToken; +#if !WINDOWS_UWP && !WIN_UI + _telemetrySender = telemetrySender; +#else + _ = telemetrySender; +#endif } /// @@ -119,12 +127,27 @@ internal async Task RunTestsAsync(IEnumerable? tests, IRunContext? run Ensure.NotEmpty(tests); - if (!MSTestDiscovererHelpers.InitializeDiscovery(from test in tests select test.Source, runContext, frameworkHandle, configuration, new TestSourceHandler())) + // Initialize telemetry collection if not already set +#if !WINDOWS_UWP && !WIN_UI + if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) { - return; + _ = MSTestTelemetryDataCollector.EnsureInitialized(); } +#endif + + try + { + if (!MSTestDiscovererHelpers.InitializeDiscovery(from test in tests select test.Source, runContext, frameworkHandle, configuration, new TestSourceHandler())) + { + return; + } - await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(tests, runContext, frameworkHandle, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(tests, runContext, frameworkHandle, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + } + finally + { + await SendTelemetryAsync().ConfigureAwait(false); + } } internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? runContext, IFrameworkHandle? frameworkHandle, IConfiguration? configuration, bool isMTP) @@ -147,14 +170,29 @@ internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? run Ensure.NotEmpty(sources); - TestSourceHandler testSourceHandler = new(); - if (!MSTestDiscovererHelpers.InitializeDiscovery(sources, runContext, frameworkHandle, configuration, testSourceHandler)) + // Initialize telemetry collection if not already set +#if !WINDOWS_UWP && !WIN_UI + if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) { - return; + _ = MSTestTelemetryDataCollector.EnsureInitialized(); } +#endif + + try + { + TestSourceHandler testSourceHandler = new(); + if (!MSTestDiscovererHelpers.InitializeDiscovery(sources, runContext, frameworkHandle, configuration, testSourceHandler)) + { + return; + } - sources = testSourceHandler.GetTestSources(sources); - await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(sources, runContext, frameworkHandle, testSourceHandler, isMTP, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + sources = testSourceHandler.GetTestSources(sources); + await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(sources, runContext, frameworkHandle, testSourceHandler, isMTP, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + } + finally + { + await SendTelemetryAsync().ConfigureAwait(false); + } } /// @@ -163,6 +201,14 @@ internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? run public void Cancel() => _testRunCancellationToken?.Cancel(); +#if !WINDOWS_UWP && !WIN_UI + private Task SendTelemetryAsync() + => MSTestTelemetryDataCollector.SendExecutionTelemetryAndResetAsync(_telemetrySender); +#else + private static Task SendTelemetryAsync() + => Task.CompletedTask; +#endif + private async Task RunTestsFromRightContextAsync(IFrameworkHandle frameworkHandle, Func runTestsAction) { ApartmentState? requestedApartmentState = MSTestSettings.RunConfigurationSettings.ExecutionApartmentState; diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs index 1aefac9304..60df656e33 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs @@ -50,6 +50,17 @@ internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflec return null; } + // Track class-level attributes for telemetry (read Current per call so a session reset + // between TypeEnumerator construction and use cannot cause writes to land on an + // orphaned collector). +#if !WINDOWS_UWP && !WIN_UI + if (Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.MSTestTelemetryDataCollector.Current is { } telemetryDataCollector) + { + Attribute[] classAttributes = ReflectHelper.GetCustomAttributesCached(_type); + telemetryDataCollector.TrackDiscoveredClass(classAttributes); + } +#endif + // If test class is valid, then get the tests return GetTests(warnings); } @@ -151,6 +162,9 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, bool classDisables }; Attribute[] attributes = reflectionOperations.GetCustomAttributesCached(method); +#if !WINDOWS_UWP && !WIN_UI + Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.MSTestTelemetryDataCollector.Current?.TrackDiscoveredMethod(attributes); +#endif TestMethodAttribute? testMethodAttribute = null; List? workItemIds = null; diff --git a/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.Configuration.cs b/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.Configuration.cs index f200893c1b..01aa0e45e8 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.Configuration.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.Configuration.cs @@ -59,6 +59,18 @@ internal static void PopulateSettings(IDiscoveryContext? context, IMessageLogger CurrentSettings = settings; RunConfigurationSettings = runConfigurationSettings; + + // Track configuration source for telemetry. +#if !WINDOWS_UWP && !WIN_UI + if (MSTestTelemetryDataCollector.Current is { } telemetry) + { + telemetry.ConfigurationSource = configuration?["mstest"] is not null + ? "testconfig.json" + : !StringEx.IsNullOrEmpty(context?.RunSettings?.SettingsXml) + ? "runsettings" + : "none"; + } +#endif } private static void SetGlobalSettings(string runsettingsXml, MSTestSettings settings, IMessageLogger? logger) diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs new file mode 100644 index 0000000000..af1c732475 --- /dev/null +++ b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs @@ -0,0 +1,516 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#if !WINDOWS_UWP && !WIN_UI +using System.Security.Cryptography; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; + +/// +/// Collects and aggregates telemetry data about MSTest usage within a test session. +/// Captures settings, attribute usage, custom/inherited types, and assertion API usage. +/// +/// +/// This collector relies on static state () that lives in the +/// AppDomain in which the adapter executes. On .NET Framework runs that opt into the +/// adapter's child-AppDomain isolation, code that runs inside the child AppDomain (for +/// example, attribute discovery via the adapter's enumerators) sees its own +/// snapshot, which is initially null. In that case telemetry from +/// the isolated AppDomain is silently dropped (the Current?.Track* call sites are +/// null-safe). This is an intentional, graceful degradation: the .NET Framework AppDomain +/// scenario is rare and the effort to marshal counters across AppDomain boundaries via +/// is not justified for best-effort usage telemetry. +/// +internal sealed class MSTestTelemetryDataCollector +{ + private readonly ConcurrentDictionary _attributeCounts = new(); +#if NET9_0_OR_GREATER + private readonly Lock _customTypesGate = new(); +#else + private readonly object _customTypesGate = new(); +#endif + private readonly HashSet _customTestMethodTypes = []; + private readonly HashSet _customTestClassTypes = []; + +#pragma warning disable IDE0032 // Use auto property - Volatile.Read/Write requires a ref to a field + private static MSTestTelemetryDataCollector? s_current; + private static int s_discoveryEventEmitted; + + // Volatile because ConfigurationSource is written from the discovery thread (e.g. settings + // load) and read from whichever thread runs SendDiscoveryTelemetryAndResetAsync — without + // a memory barrier the reader could in principle observe a stale null. + private volatile string? _configurationSource; +#pragma warning restore IDE0032 // Use auto property + + /// + /// Gets or sets the current telemetry data collector for the session. + /// Set at session start, cleared at session close. + /// + internal static MSTestTelemetryDataCollector? Current + { + get => Volatile.Read(ref s_current); + set => Volatile.Write(ref s_current, value); + } + + internal static MSTestTelemetryDataCollector EnsureInitialized() + { + MSTestTelemetryDataCollector? collector = Current; + if (collector is not null) + { + return collector; + } + + collector = new MSTestTelemetryDataCollector(); + MSTestTelemetryDataCollector? existingCollector = Interlocked.CompareExchange(ref s_current, collector, null); + + return existingCollector ?? collector; + } + + /// + /// Checks whether telemetry collection is opted out via environment variables. + /// Mirrors the same checks as Microsoft.Testing.Platform's TelemetryManager. + /// + /// true if telemetry is opted out; false otherwise. + internal static bool IsTelemetryOptedOut() + { + string? telemetryOptOut = Environment.GetEnvironmentVariable("TESTINGPLATFORM_TELEMETRY_OPTOUT"); + if (telemetryOptOut is "1" or "true") + { + return true; + } + + string? cliTelemetryOptOut = Environment.GetEnvironmentVariable("DOTNET_CLI_TELEMETRY_OPTOUT"); + + return cliTelemetryOptOut is "1" or "true"; + } + + /// + /// Gets or sets the configuration source used for this session. + /// + internal string? ConfigurationSource + { + get => _configurationSource; + set => _configurationSource = value; + } + + /// + /// Records the attributes found on a test method during discovery. Safe to call concurrently + /// from multiple discovery threads — counters use a + /// and the custom-type sets are protected by an internal lock. + /// + /// The cached attributes from the method. + internal void TrackDiscoveredMethod(Attribute[] attributes) + { + foreach (Attribute attribute in attributes) + { + Type attributeType = attribute.GetType(); + string attributeName = attributeType.Name; + + // Track custom/inherited TestMethodAttribute types (store anonymized hash) + if (attribute is TestMethodAttribute && attributeType != typeof(TestMethodAttribute)) + { + AddCustomType(_customTestMethodTypes, AnonymizeString(attributeType.FullName ?? attributeName)); + } + + // Track custom/inherited TestClassAttribute types (store anonymized hash) + if (attribute is TestClassAttribute && attributeType != typeof(TestClassAttribute)) + { + AddCustomType(_customTestClassTypes, AnonymizeString(attributeType.FullName ?? attributeName)); + } + + // Track attribute usage counts by base type name (only known MSTest attributes) + string? trackingName = attribute switch + { + TestMethodAttribute => nameof(TestMethodAttribute), + TestClassAttribute => nameof(TestClassAttribute), + DataRowAttribute => nameof(DataRowAttribute), + DynamicDataAttribute => nameof(DynamicDataAttribute), + TimeoutAttribute => nameof(TimeoutAttribute), + IgnoreAttribute => nameof(IgnoreAttribute), + DoNotParallelizeAttribute => nameof(DoNotParallelizeAttribute), + RetryBaseAttribute => nameof(RetryBaseAttribute), + ConditionBaseAttribute => nameof(ConditionBaseAttribute), + TestCategoryAttribute => nameof(TestCategoryAttribute), +#if !WIN_UI + DeploymentItemAttribute => nameof(DeploymentItemAttribute), +#endif + _ => null, + }; + + if (trackingName is not null) + { + _attributeCounts.AddOrUpdate(trackingName, 1, static (_, count) => count + 1); + } + } + } + + /// + /// Records the attributes found on a test class during discovery. Safe to call concurrently + /// from multiple discovery threads — counters use a + /// and the custom-type sets are protected by an internal lock. + /// + /// The cached attributes from the class. + internal void TrackDiscoveredClass(Attribute[] attributes) + { + foreach (Attribute attribute in attributes) + { + Type attributeType = attribute.GetType(); + + // Track custom/inherited TestClassAttribute types (store anonymized hash) + if (attribute is TestClassAttribute && attributeType != typeof(TestClassAttribute)) + { + AddCustomType(_customTestClassTypes, AnonymizeString(attributeType.FullName ?? attributeType.Name)); + } + + string? trackingName = attribute switch + { + TestClassAttribute => nameof(TestClassAttribute), + ParallelizeAttribute => nameof(ParallelizeAttribute), + DoNotParallelizeAttribute => nameof(DoNotParallelizeAttribute), + _ => null, + }; + + if (trackingName is not null) + { + _attributeCounts.AddOrUpdate(trackingName, 1, static (_, count) => count + 1); + } + } + } + + private void AddCustomType(HashSet set, string value) + { + lock (_customTypesGate) + { + set.Add(value); + } + } + + /// + /// Builds the discovery telemetry metrics dictionary (settings + discovery-time data). + /// Sent at the end of MTP discover-only sessions (e.g. dotnet test --list-tests). + /// + /// A dictionary of telemetry key-value pairs for the discovery event. + internal Dictionary BuildDiscoveryMetrics() + { + Dictionary metrics = []; + AddDiscoveryMetrics(metrics); + return metrics; + } + + /// + /// Builds the execution telemetry metrics dictionary. Sent at the end of an MSTest run + /// (MTP run mode or VSTest run mode). Always carries assertion usage. Also carries the + /// settings/attribute/config payload UNLESS a discovery event has already been emitted + /// during this process — that avoids duplicating the discovery payload across two events + /// when a host (such as a future MTP host) chooses to call both discover and run within + /// the same session. + /// + /// Drained assertion call counts captured during execution. + /// When true, also include the discovery metrics + /// (settings, config_source, attribute_usage, custom_test_*_types). False when the discovery + /// event already shipped these in this process. + /// A dictionary of telemetry key-value pairs for the sessionexit event. + internal Dictionary BuildExecutionMetrics(Dictionary assertionCounts, bool includeDiscoveryPayload) + { + Dictionary metrics = []; + + if (includeDiscoveryPayload) + { + AddDiscoveryMetrics(metrics); + } + + if (assertionCounts.Count > 0) + { + metrics["mstest.assertion_usage"] = SerializeDictionary(assertionCounts); + } + + return metrics; + } + + private void AddDiscoveryMetrics(Dictionary metrics) + { + // Settings + AddSettingsMetrics(metrics); + + // Configuration source (runsettings, testconfig.json, or none) + if (ConfigurationSource is { } configSource) + { + metrics["mstest.config_source"] = configSource; + } + + // Attribute usage (aggregated counts as JSON; serializer enforces ordinal sort) + if (!_attributeCounts.IsEmpty) + { + metrics["mstest.attribute_usage"] = SerializeDictionary(_attributeCounts); + } + + // Custom/inherited types (anonymized names; serializer enforces ordinal sort) + // Take a snapshot under the lock that protects the HashSet to avoid concurrent + // modification while serializing. + string[]? customMethodTypesSnapshot = SnapshotCustomTypes(_customTestMethodTypes); + if (customMethodTypesSnapshot is { Length: > 0 }) + { + metrics["mstest.custom_test_method_types"] = SerializeCollection(customMethodTypesSnapshot); + } + + string[]? customClassTypesSnapshot = SnapshotCustomTypes(_customTestClassTypes); + if (customClassTypesSnapshot is { Length: > 0 }) + { + metrics["mstest.custom_test_class_types"] = SerializeCollection(customClassTypesSnapshot); + } + } + + private string[]? SnapshotCustomTypes(HashSet set) + { + lock (_customTypesGate) + { + return set.Count == 0 ? null : [.. set]; + } + } + + private static string SerializeCollection(IEnumerable values) + { + StringBuilder builder = new("["); + bool isFirst = true; + + foreach (string value in values.OrderBy(static x => x, StringComparer.Ordinal)) + { + if (!isFirst) + { + builder.Append(','); + } + + AppendJsonString(builder, value); + isFirst = false; + } + + builder.Append(']'); + return builder.ToString(); + } + + private static string SerializeDictionary(IEnumerable> values) + { + StringBuilder builder = new("{"); + bool isFirst = true; + + foreach (KeyValuePair value in values.OrderBy(x => x.Key, StringComparer.Ordinal)) + { + if (!isFirst) + { + builder.Append(','); + } + + AppendJsonString(builder, value.Key); + builder.Append(':'); + builder.Append(value.Value.ToString(CultureInfo.InvariantCulture)); + isFirst = false; + } + + builder.Append('}'); + return builder.ToString(); + } + + private static void AppendJsonString(StringBuilder builder, string value) + { + builder.Append('"'); + + foreach (char character in value) + { + switch (character) + { + case '"': + builder.Append("\\\""); + break; + case '\\': + builder.Append("\\\\"); + break; + case '\b': + builder.Append("\\b"); + break; + case '\f': + builder.Append("\\f"); + break; + case '\n': + builder.Append("\\n"); + break; + case '\r': + builder.Append("\\r"); + break; + case '\t': + builder.Append("\\t"); + break; + default: + if (char.IsControl(character)) + { + builder.Append("\\u"); + builder.Append(((int)character).ToString("x4", CultureInfo.InvariantCulture)); + } + else + { + builder.Append(character); + } + + break; + } + } + + builder.Append('"'); + } + + private static void AddSettingsMetrics(Dictionary metrics) + { + MSTestSettings settings = MSTestSettings.CurrentSettings; + + // Parallelization + metrics["mstest.setting.parallelization_enabled"] = AsTelemetryBool(!settings.DisableParallelization); + if (settings.ParallelizationScope is not null) + { + metrics["mstest.setting.parallelization_scope"] = settings.ParallelizationScope.Value.ToString(); + } + + if (settings.ParallelizationWorkers is not null) + { + // Cast to double so AppInsightsProvider routes this through the metric channel + // instead of stringifying it as a property — see AppInsightsProvider.SendLoopAsync. + metrics["mstest.setting.parallelization_workers"] = (double)settings.ParallelizationWorkers.Value; + } + + // Timeouts (cast to double for the same reason as parallelization_workers above). + metrics["mstest.setting.test_timeout"] = (double)settings.TestTimeout; + metrics["mstest.setting.assembly_initialize_timeout"] = (double)settings.AssemblyInitializeTimeout; + metrics["mstest.setting.assembly_cleanup_timeout"] = (double)settings.AssemblyCleanupTimeout; + metrics["mstest.setting.class_initialize_timeout"] = (double)settings.ClassInitializeTimeout; + metrics["mstest.setting.class_cleanup_timeout"] = (double)settings.ClassCleanupTimeout; + metrics["mstest.setting.test_initialize_timeout"] = (double)settings.TestInitializeTimeout; + metrics["mstest.setting.test_cleanup_timeout"] = (double)settings.TestCleanupTimeout; + metrics["mstest.setting.cooperative_cancellation"] = AsTelemetryBool(settings.CooperativeCancellationTimeout); + + // Behavior + metrics["mstest.setting.map_inconclusive_to_failed"] = AsTelemetryBool(settings.MapInconclusiveToFailed); + metrics["mstest.setting.map_not_runnable_to_failed"] = AsTelemetryBool(settings.MapNotRunnableToFailed); + metrics["mstest.setting.treat_discovery_warnings_as_errors"] = AsTelemetryBool(settings.TreatDiscoveryWarningsAsErrors); + metrics["mstest.setting.consider_empty_data_source_as_inconclusive"] = AsTelemetryBool(settings.ConsiderEmptyDataSourceAsInconclusive); + metrics["mstest.setting.order_tests_by_name"] = AsTelemetryBool(settings.OrderTestsByNameInClass); + metrics["mstest.setting.capture_debug_traces"] = AsTelemetryBool(settings.CaptureDebugTraces); + } + + // MTP's telemetry providers (e.g. AppInsightsProvider) reject raw boolean values and assert + // that they should be sent as their lowercase string form. This mirrors + // Microsoft.Testing.Platform.Telemetry.TelemetryExtensions.AsTelemetryBool, which we can't + // reference from here because that type lives in a different assembly. + private static string AsTelemetryBool(bool value) => value ? "true" : "false"; + + private static string AnonymizeString(string value) + { +#if NET + byte[] hash = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(value)); + return Convert.ToHexString(hash); +#else + using var sha256 = SHA256.Create(); + byte[] hash = sha256.ComputeHash(System.Text.Encoding.UTF8.GetBytes(value)); + return BitConverter.ToString(hash).Replace("-", string.Empty); +#endif + } + + /// + /// Sends the accumulated discovery telemetry via the provided sender delegate and clears the + /// discovery state by resetting the current collector. Safe to call when no sender is available + /// (the collector is still cleared so state does not leak across sessions). + /// + /// Optional delegate to send telemetry. If null, telemetry is silently discarded. + internal static async Task SendDiscoveryTelemetryAndResetAsync(Func, Task>? telemetrySender) + { + try + { + MSTestTelemetryDataCollector? collector = Current; + if (collector is null || telemetrySender is null) + { + return; + } + + // Defense in depth: re-check opt-out at send time in case the env var was set after + // EnsureInitialized but before this point. + if (IsTelemetryOptedOut()) + { + return; + } + + Dictionary metrics = collector.BuildDiscoveryMetrics(); + if (metrics.Count > 0) + { + await telemetrySender("dotnet/testingplatform/mstest/discovery", metrics).ConfigureAwait(false); + + // Mark that the discovery payload (settings + attribute_usage + custom_test_*_types + // + config_source) has shipped in this process so a subsequent execution event in + // the same session does not duplicate it. + Interlocked.Exchange(ref s_discoveryEventEmitted, 1); + } + } + catch (Exception) + { + // Telemetry should never cause test failures + } + finally + { + // Clear the current collector so a subsequent execution accumulates settings/config + // anew (settings are static-per-process so re-population is cheap and keeps each + // event self-contained). + Current = null; + } + } + + /// + /// Sends the accumulated execution telemetry via the provided sender delegate, drains the + /// static assertion counters, and clears the current collector. Safe to call when no sender + /// is available (the counters and collector are still drained/cleared so state does not leak + /// across sessions). + /// + /// Optional delegate to send telemetry. If null, telemetry is silently discarded. + internal static async Task SendExecutionTelemetryAndResetAsync(Func, Task>? telemetrySender) + { + try + { + // Always drain the static assertion counters so they don't leak across sessions, + // even when no sender is wired (VSTest mode) or no collector was initialized + // (e.g. telemetry opted out before EnsureInitialized was called). + Dictionary assertionCounts = TelemetryCollector.DrainAssertionCallCounts(); + + MSTestTelemetryDataCollector? collector = Current; + if (collector is null || telemetrySender is null) + { + return; + } + + // Defense in depth: re-check opt-out at send time in case the env var was set after + // EnsureInitialized but before this point. + if (IsTelemetryOptedOut()) + { + return; + } + + // If the discovery event already shipped the settings/attribute payload during this + // process, do not duplicate it in the sessionexit event. The flag is reset below in + // the finally block so each process can still ship a fresh payload after a full + // session reset. + bool includeDiscoveryPayload = Interlocked.CompareExchange(ref s_discoveryEventEmitted, 0, 0) == 0; + + Dictionary metrics = collector.BuildExecutionMetrics(assertionCounts, includeDiscoveryPayload); + if (metrics.Count > 0) + { + await telemetrySender("dotnet/testingplatform/mstest/sessionexit", metrics).ConfigureAwait(false); + } + } + catch (Exception) + { + // Telemetry should never cause test failures + } + finally + { + Current = null; + Interlocked.Exchange(ref s_discoveryEventEmitted, 0); + } + } +} +#endif diff --git a/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsProvider.cs b/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsProvider.cs index 2b94ff37b7..4724dc2c02 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsProvider.cs +++ b/src/Platform/Microsoft.Testing.Extensions.Telemetry/AppInsightsProvider.cs @@ -63,7 +63,17 @@ internal sealed partial class AppInsightsProvider : TelemetryProperties.HostProperties.RuntimeIdentifierPropertyName, TelemetryProperties.HostProperties.ApplicationModePropertyName, TelemetryProperties.HostProperties.ExitCodePropertyName, - TelemetryProperties.HostProperties.ExtensionsPropertyName + TelemetryProperties.HostProperties.ExtensionsPropertyName, + + // MSTest session telemetry (see MSTestTelemetryDataCollector). These carry aggregated + // counts, anonymized SHA256 type hashes inside JSON envelopes, and well-known enum/string + // values - none of them contain unhashed user-identifying data. + "mstest.config_source", + "mstest.attribute_usage", + "mstest.custom_test_method_types", + "mstest.custom_test_class_types", + "mstest.assertion_usage", + "mstest.setting.parallelization_scope", ]; #endif diff --git a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj index 376a5686ba..9aabda5ddc 100644 --- a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj +++ b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj @@ -58,6 +58,7 @@ This package provides the core platform and the .NET implementation of the proto + diff --git a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.Numerics.cs b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.Numerics.cs index 95d917ae41..aa42fa6abf 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.Numerics.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.Numerics.cs @@ -81,7 +81,10 @@ private static void ReportAssertAreEqualFailed(T expected, T actual, T delta, #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreEqual(float expected, float actual, float delta, [InterpolatedStringHandlerArgument(nameof(expected), nameof(actual), nameof(delta))] ref AssertNonGenericAreEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(expectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + message.ComputeAssertion(expectedExpression, actualExpression); + } /// /// Tests whether the specified floats are equal and throws an exception @@ -117,6 +120,8 @@ public static void AreEqual(float expected, float actual, float delta, [Interpol /// public static void AreEqual(float expected, float actual, float delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -128,7 +133,10 @@ public static void AreEqual(float expected, float actual, float delta, string? m #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreNotEqual(float notExpected, float actual, float delta, [InterpolatedStringHandlerArgument(nameof(notExpected), nameof(actual), nameof(delta))] ref AssertNonGenericAreNotEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(notExpectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + message.ComputeAssertion(notExpectedExpression, actualExpression); + } /// /// Tests whether the specified floats are unequal and throws an exception @@ -164,6 +172,8 @@ public static void AreNotEqual(float notExpected, float actual, float delta, [In /// public static void AreNotEqual(float notExpected, float actual, float delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); @@ -196,7 +206,10 @@ private static bool AreNotEqualFailing(float notExpected, float actual, float de #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreEqual(decimal expected, decimal actual, decimal delta, [InterpolatedStringHandlerArgument(nameof(expected), nameof(actual), nameof(delta))] ref AssertNonGenericAreEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(expectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + message.ComputeAssertion(expectedExpression, actualExpression); + } /// /// Tests whether the specified decimals are equal and throws an exception @@ -232,6 +245,8 @@ public static void AreEqual(decimal expected, decimal actual, decimal delta, [In /// public static void AreEqual(decimal expected, decimal actual, decimal delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -243,7 +258,10 @@ public static void AreEqual(decimal expected, decimal actual, decimal delta, str #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreNotEqual(decimal notExpected, decimal actual, decimal delta, [InterpolatedStringHandlerArgument(nameof(notExpected), nameof(actual), nameof(delta))] ref AssertNonGenericAreNotEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(notExpectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + message.ComputeAssertion(notExpectedExpression, actualExpression); + } /// /// Tests whether the specified decimals are unequal and throws an exception @@ -279,6 +297,8 @@ public static void AreNotEqual(decimal notExpected, decimal actual, decimal delt /// public static void AreNotEqual(decimal notExpected, decimal actual, decimal delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); @@ -293,7 +313,10 @@ private static bool AreNotEqualFailing(decimal notExpected, decimal actual, deci #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreEqual(long expected, long actual, long delta, [InterpolatedStringHandlerArgument(nameof(expected), nameof(actual), nameof(delta))] ref AssertNonGenericAreEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(expectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + message.ComputeAssertion(expectedExpression, actualExpression); + } /// /// Tests whether the specified longs are equal and throws an exception @@ -329,6 +352,8 @@ public static void AreEqual(long expected, long actual, long delta, [Interpolate /// public static void AreEqual(long expected, long actual, long delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -340,7 +365,10 @@ public static void AreEqual(long expected, long actual, long delta, string? mess #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreNotEqual(long notExpected, long actual, long delta, [InterpolatedStringHandlerArgument(nameof(notExpected), nameof(actual), nameof(delta))] ref AssertNonGenericAreNotEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(notExpectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + message.ComputeAssertion(notExpectedExpression, actualExpression); + } /// /// Tests whether the specified longs are unequal and throws an exception @@ -376,6 +404,8 @@ public static void AreNotEqual(long notExpected, long actual, long delta, [Inter /// public static void AreNotEqual(long notExpected, long actual, long delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); @@ -390,7 +420,10 @@ private static bool AreNotEqualFailing(long notExpected, long actual, long delta #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreEqual(double expected, double actual, double delta, [InterpolatedStringHandlerArgument(nameof(expected), nameof(actual), nameof(delta))] ref AssertNonGenericAreEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(expectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + message.ComputeAssertion(expectedExpression, actualExpression); + } /// /// Tests whether the specified doubles are equal and throws an exception @@ -425,6 +458,8 @@ public static void AreEqual(double expected, double actual, double delta, [Inter /// public static void AreEqual(double expected, double actual, double delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -436,7 +471,10 @@ public static void AreEqual(double expected, double actual, double delta, string #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreNotEqual(double notExpected, double actual, double delta, [InterpolatedStringHandlerArgument(nameof(notExpected), nameof(actual), nameof(delta))] ref AssertNonGenericAreNotEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(notExpectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + message.ComputeAssertion(notExpectedExpression, actualExpression); + } /// /// Tests whether the specified doubles are unequal and throws an exception @@ -472,6 +510,8 @@ public static void AreNotEqual(double notExpected, double actual, double delta, /// public static void AreNotEqual(double notExpected, double actual, double delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.String.cs b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.String.cs index aafc366e5c..74299f9e38 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.String.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.String.cs @@ -80,7 +80,10 @@ public static void AreEqual(string? expected, string? actual, bool ignoreCase, s #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreEqual(string? expected, string? actual, bool ignoreCase, [InterpolatedStringHandlerArgument(nameof(expected), nameof(actual), nameof(ignoreCase))] ref AssertNonGenericAreEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(expectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + message.ComputeAssertion(expectedExpression, actualExpression); + } /// #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 @@ -88,6 +91,7 @@ public static void AreEqual(string? expected, string? actual, bool ignoreCase, #pragma warning restore IDE0060 // Remove unused parameter CultureInfo culture, [InterpolatedStringHandlerArgument(nameof(expected), nameof(actual), nameof(ignoreCase), nameof(culture))] ref AssertNonGenericAreEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); CheckParameterNotNull(culture, "Assert.AreEqual", nameof(culture)); message.ComputeAssertion(expectedExpression, actualExpression); } @@ -127,6 +131,8 @@ public static void AreEqual(string? expected, string? actual, bool ignoreCase, /// public static void AreEqual(string? expected, string? actual, bool ignoreCase, CultureInfo culture, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + CheckParameterNotNull(culture, "Assert.AreEqual", nameof(culture)); if (!AreEqualFailing(expected, actual, ignoreCase, culture)) { @@ -175,7 +181,10 @@ public static void AreNotEqual(string? notExpected, string? actual, bool ignoreC #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreNotEqual(string? notExpected, string? actual, bool ignoreCase, [InterpolatedStringHandlerArgument(nameof(notExpected), nameof(actual), nameof(ignoreCase))] ref AssertNonGenericAreNotEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(notExpectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + message.ComputeAssertion(notExpectedExpression, actualExpression); + } /// #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 @@ -183,6 +192,7 @@ public static void AreNotEqual(string? notExpected, string? actual, bool ignoreC #pragma warning restore IDE0060 // Remove unused parameter CultureInfo culture, [InterpolatedStringHandlerArgument(nameof(notExpected), nameof(actual), nameof(ignoreCase), nameof(culture))] ref AssertNonGenericAreNotEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); CheckParameterNotNull(culture, "Assert.AreNotEqual", nameof(culture)); message.ComputeAssertion(notExpectedExpression, actualExpression); } @@ -223,6 +233,8 @@ public static void AreNotEqual(string? notExpected, string? actual, bool ignoreC /// public static void AreNotEqual(string? notExpected, string? actual, bool ignoreCase, CultureInfo culture, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + CheckParameterNotNull(culture, "Assert.AreNotEqual", "culture"); if (!AreNotEqualFailing(notExpected, actual, ignoreCase, culture)) { diff --git a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs index 3142d9644f..59710dcab7 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs @@ -81,13 +81,19 @@ public static void AreEqual(T? expected, T? actual, string? message = "", [Ca #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreEqual(T? expected, T? actual, [InterpolatedStringHandlerArgument(nameof(expected), nameof(actual))] ref AssertAreEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(expectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + message.ComputeAssertion(expectedExpression, actualExpression); + } /// #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreEqual(T? expected, T? actual, IEqualityComparer? comparer, [InterpolatedStringHandlerArgument(nameof(expected), nameof(actual), nameof(comparer))] ref AssertAreEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(expectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + message.ComputeAssertion(expectedExpression, actualExpression); + } /// /// Tests whether the specified values are equal and throws an exception @@ -126,6 +132,8 @@ public static void AreEqual(T? expected, T? actual, IEqualityComparer? com /// public static void AreEqual(T? expected, T? actual, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (!AreEqualFailing(expected, actual, comparer)) { return; @@ -297,13 +305,19 @@ public static void AreNotEqual(T? notExpected, T? actual, string? message = " #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreNotEqual(T? notExpected, T? actual, [InterpolatedStringHandlerArgument(nameof(notExpected), nameof(actual))] ref AssertAreNotEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(notExpectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + message.ComputeAssertion(notExpectedExpression, actualExpression); + } /// #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreNotEqual(T? notExpected, T? actual, IEqualityComparer comparer, [InterpolatedStringHandlerArgument(nameof(notExpected), nameof(actual), nameof(comparer))] ref AssertAreNotEqualInterpolatedStringHandler message, [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(notExpectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + message.ComputeAssertion(notExpectedExpression, actualExpression); + } /// /// Tests whether the specified values are unequal and throws an exception @@ -342,6 +356,8 @@ public static void AreNotEqual(T? notExpected, T? actual, IEqualityComparer public static void AreNotEqual(T? notExpected, T? actual, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (!AreNotEqualFailing(notExpected, actual, comparer)) { return; diff --git a/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs b/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs index dced467094..2a3eaf72dc 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs @@ -138,7 +138,10 @@ internal void ComputeAssertion(string notExpectedExpression, string actualExpres #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreSame(T? expected, T? actual, [InterpolatedStringHandlerArgument(nameof(expected), nameof(actual))] ref AssertAreSameInterpolatedStringHandler message, [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(expectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreSame"); + message.ComputeAssertion(expectedExpression, actualExpression); + } #pragma warning disable RS0027 // API with optional parameter(s) should have the most parameters amongst its public overloads @@ -174,6 +177,8 @@ public static void AreSame(T? expected, T? actual, [InterpolatedStringHandler /// public static void AreSame(T? expected, T? actual, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreSame"); + if (!IsAreSameFailing(expected, actual)) { return; @@ -216,7 +221,10 @@ private static void ReportAssertAreSameFailed(T? expected, T? actual, string? #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void AreNotSame(T? notExpected, T? actual, [InterpolatedStringHandlerArgument(nameof(notExpected), nameof(actual))] ref AssertAreNotSameInterpolatedStringHandler message, [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(notExpectedExpression, actualExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.AreNotSame"); + message.ComputeAssertion(notExpectedExpression, actualExpression); + } /// /// Tests whether the specified objects refer to different objects and @@ -251,6 +259,8 @@ public static void AreNotSame(T? notExpected, T? actual, [InterpolatedStringH /// public static void AreNotSame(T? notExpected, T? actual, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotSame"); + if (IsAreNotSameFailing(notExpected, actual)) { ReportAssertAreNotSameFailed(notExpected, actual, message, notExpectedExpression, actualExpression); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs b/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs index ecdbfa1407..d06dca873a 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs @@ -154,6 +154,8 @@ public static T ContainsSingle(IEnumerable collection, string? message = " /// The item that matches the predicate. public static T ContainsSingle(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.ContainsSingle"); + T firstMatch = default!; int matchCount = 0; @@ -209,6 +211,8 @@ public static T ContainsSingle(Func predicate, IEnumerable collec /// The item that matches the predicate. public static object? ContainsSingle(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.ContainsSingle"); + object? firstMatch = null; int matchCount = 0; @@ -273,6 +277,8 @@ public static T ContainsSingle(Func predicate, IEnumerable collec /// public static void Contains(T expected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + if (!collection.Contains(expected)) { string userMessage = BuildUserMessageForExpectedExpressionAndCollectionExpression(message, expectedExpression, collectionExpression); @@ -296,6 +302,8 @@ public static void Contains(T expected, IEnumerable collection, string? me /// public static void Contains(object? expected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(collection, "Assert.Contains", "collection"); foreach (object? item in collection) @@ -328,6 +336,8 @@ public static void Contains(object? expected, IEnumerable collection, string? me /// public static void Contains(T expected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + if (!collection.Contains(expected, comparer)) { string userMessage = BuildUserMessageForExpectedExpressionAndCollectionExpression(message, expectedExpression, collectionExpression); @@ -352,6 +362,8 @@ public static void Contains(T expected, IEnumerable collection, IEqualityC /// public static void Contains(object? expected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(collection, "Assert.Contains", "collection"); CheckParameterNotNull(comparer, "Assert.Contains", "comparer"); @@ -384,6 +396,8 @@ public static void Contains(object? expected, IEnumerable collection, IEqualityC /// public static void Contains(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + if (!collection.Any(predicate)) { string userMessage = BuildUserMessageForPredicateExpressionAndCollectionExpression(message, predicateExpression, collectionExpression); @@ -407,6 +421,8 @@ public static void Contains(Func predicate, IEnumerable collectio /// public static void Contains(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(collection, "Assert.Contains", "collection"); CheckParameterNotNull(predicate, "Assert.Contains", "predicate"); @@ -486,6 +502,8 @@ public static void Contains(string substring, string value, string? message = "" /// public static void Contains(string substring, string value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(substring))] string substringExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(value, "Assert.Contains", "value"); CheckParameterNotNull(substring, "Assert.Contains", "substring"); @@ -522,6 +540,8 @@ public static void Contains(string substring, string value, StringComparison com /// public static void DoesNotContain(T notExpected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + if (collection.Contains(notExpected)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndCollectionExpression(message, notExpectedExpression, collectionExpression); @@ -545,6 +565,8 @@ public static void DoesNotContain(T notExpected, IEnumerable collection, s /// public static void DoesNotContain(object? notExpected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(collection, "Assert.DoesNotContain", "collection"); foreach (object? item in collection) @@ -575,6 +597,8 @@ public static void DoesNotContain(object? notExpected, IEnumerable collection, s /// public static void DoesNotContain(T notExpected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + if (collection.Contains(notExpected, comparer)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndCollectionExpression(message, notExpectedExpression, collectionExpression); @@ -599,6 +623,8 @@ public static void DoesNotContain(T notExpected, IEnumerable collection, I /// public static void DoesNotContain(object? notExpected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(collection, "Assert.DoesNotContain", "collection"); CheckParameterNotNull(comparer, "Assert.DoesNotContain", "comparer"); @@ -629,6 +655,8 @@ public static void DoesNotContain(object? notExpected, IEnumerable collection, I /// public static void DoesNotContain(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + if (collection.Any(predicate)) { string userMessage = BuildUserMessageForPredicateExpressionAndCollectionExpression(message, predicateExpression, collectionExpression); @@ -652,6 +680,8 @@ public static void DoesNotContain(Func predicate, IEnumerable col /// public static void DoesNotContain(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(collection, "Assert.DoesNotContain", "collection"); CheckParameterNotNull(predicate, "Assert.DoesNotContain", "predicate"); @@ -729,6 +759,8 @@ public static void DoesNotContain(string substring, string value, string? messag /// public static void DoesNotContain(string substring, string value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(substring))] string substringExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(value, "Assert.DoesNotContain", "value"); CheckParameterNotNull(substring, "Assert.DoesNotContain", "substring"); @@ -772,6 +804,8 @@ public static void DoesNotContain(string substring, string value, StringComparis public static void IsInRange(T minValue, T maxValue, T value, string? message = "", [CallerArgumentExpression(nameof(minValue))] string minValueExpression = "", [CallerArgumentExpression(nameof(maxValue))] string maxValueExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : struct, IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsInRange"); + if (maxValue.CompareTo(minValue) < 0) { throw new ArgumentOutOfRangeException(nameof(maxValue), FrameworkMessages.IsInRangeMaxValueMustBeGreaterThanOrEqualMinValue); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Count.cs b/src/TestFramework/TestFramework/Assertions/Assert.Count.cs index 4ad00d8d02..c106e77bc5 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Count.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Count.cs @@ -183,7 +183,10 @@ public void AppendFormatted(object? value, int alignment = 0, string? format = n #pragma warning disable IDE0060 // Remove unused parameter public static void IsNotEmpty(IEnumerable collection, [InterpolatedStringHandlerArgument(nameof(collection))] ref AssertIsNotEmptyInterpolatedStringHandler message, [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(collectionExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.IsNotEmpty"); + message.ComputeAssertion(collectionExpression); + } /// /// Tests that the collection is not empty. @@ -197,6 +200,8 @@ public static void IsNotEmpty(IEnumerable collection, [InterpolatedStringH /// public static void IsNotEmpty(IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotEmpty"); + if (collection.Any()) { return; @@ -217,6 +222,8 @@ public static void IsNotEmpty(IEnumerable collection, string? message = "" /// public static void IsNotEmpty(IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotEmpty"); + if (collection.Cast().Any()) { return; @@ -243,7 +250,10 @@ public static void IsNotEmpty(IEnumerable collection, string? message = "", [Cal #pragma warning disable IDE0060 // Remove unused parameter public static void HasCount(int expected, IEnumerable collection, [InterpolatedStringHandlerArgument(nameof(expected), nameof(collection))] ref AssertCountInterpolatedStringHandler message, [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion("HasCount", collectionExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.HasCount"); + message.ComputeAssertion("HasCount", collectionExpression); + } /// /// Tests whether the collection has the expected count/length. @@ -289,7 +299,10 @@ public static void HasCount(int expected, IEnumerable collection, string? messag #pragma warning disable IDE0060 // Remove unused parameter public static void IsEmpty(IEnumerable collection, [InterpolatedStringHandlerArgument(nameof(collection))] ref AssertCountInterpolatedStringHandler message, [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion("IsEmpty", collectionExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.IsEmpty"); + message.ComputeAssertion("IsEmpty", collectionExpression); + } /// /// Tests that the collection is empty. @@ -320,6 +333,10 @@ public static void IsEmpty(IEnumerable collection, string? message = "", [Caller private static void HasCount(string assertionName, int expected, IEnumerable collection, string? message, string collectionExpression) { + // assertionName is one of a small fixed set ("HasCount", "IsEmpty"); use a cached prefixed + // string instead of allocating "Assert." + assertionName on every call. + TelemetryCollector.TrackAssertionCall(GetTrackedAssertionName(assertionName)); + int actualCount = collection.Count(); if (actualCount == expected) { @@ -333,6 +350,14 @@ private static void HasCount(string assertionName, int expected, IEnumerable< private static void HasCount(string assertionName, int expected, IEnumerable collection, string? message, string collectionExpression) => HasCount(assertionName, expected, collection.Cast(), message, collectionExpression); + private static string GetTrackedAssertionName(string assertionName) + => assertionName switch + { + "HasCount" => "Assert.HasCount", + "IsEmpty" => "Assert.IsEmpty", + _ => string.Concat("Assert.", assertionName), + }; + [DoesNotReturn] private static void ReportAssertCountFailed(string assertionName, int expectedCount, int actualCount, string userMessage) { diff --git a/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs b/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs index 9ebe0a817c..bdd47da320 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs @@ -72,6 +72,8 @@ public static void EndsWith([NotNull] string? expectedSuffix, [NotNull] string? /// public static void EndsWith([NotNull] string? expectedSuffix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(expectedSuffix))] string expectedSuffixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.EndsWith"); + CheckParameterNotNull(value, "Assert.EndsWith", "value"); CheckParameterNotNull(expectedSuffix, "Assert.EndsWith", "expectedSuffix"); if (!value.EndsWith(expectedSuffix, comparisonType)) @@ -146,6 +148,8 @@ public static void DoesNotEndWith([NotNull] string? notExpectedSuffix, [NotNull] /// public static void DoesNotEndWith([NotNull] string? notExpectedSuffix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(notExpectedSuffix))] string notExpectedSuffixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotEndWith"); + CheckParameterNotNull(value, "Assert.DoesNotEndWith", "value"); CheckParameterNotNull(notExpectedSuffix, "Assert.DoesNotEndWith", "notExpectedSuffix"); if (value.EndsWith(notExpectedSuffix, comparisonType)) diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs b/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs index cbe187dc13..e377044d5a 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs @@ -22,5 +22,8 @@ public sealed partial class Assert /// [DoesNotReturn] public static void Fail(string message = "") - => ThrowAssertFailed("Assert.Fail", BuildUserMessage(message)); + { + TelemetryCollector.TrackAssertionCall("Assert.Fail"); + ThrowAssertFailed("Assert.Fail", BuildUserMessage(message)); + } } diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs b/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs index ba7f415a54..0d1d634c01 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs @@ -44,6 +44,8 @@ public sealed partial class Assert public static void IsGreaterThan(T lowerBound, T value, string? message = "", [CallerArgumentExpression(nameof(lowerBound))] string lowerBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsGreaterThan"); + if (value.CompareTo(lowerBound) > 0) { return; @@ -89,6 +91,8 @@ public static void IsGreaterThan(T lowerBound, T value, string? message = "", public static void IsGreaterThanOrEqualTo(T lowerBound, T value, string? message = "", [CallerArgumentExpression(nameof(lowerBound))] string lowerBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsGreaterThanOrEqualTo"); + if (value.CompareTo(lowerBound) >= 0) { return; @@ -134,6 +138,8 @@ public static void IsGreaterThanOrEqualTo(T lowerBound, T value, string? mess public static void IsLessThan(T upperBound, T value, string? message = "", [CallerArgumentExpression(nameof(upperBound))] string upperBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsLessThan"); + if (value.CompareTo(upperBound) < 0) { return; @@ -179,6 +185,8 @@ public static void IsLessThan(T upperBound, T value, string? message = "", [C public static void IsLessThanOrEqualTo(T upperBound, T value, string? message = "", [CallerArgumentExpression(nameof(upperBound))] string upperBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsLessThanOrEqualTo"); + if (value.CompareTo(upperBound) <= 0) { return; @@ -216,6 +224,8 @@ public static void IsLessThanOrEqualTo(T upperBound, T value, string? message public static void IsPositive(T value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : struct, IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsPositive"); + var zero = default(T); // Handle special case for floating point NaN values @@ -270,6 +280,8 @@ public static void IsPositive(T value, string? message = "", [CallerArgumentE public static void IsNegative(T value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : struct, IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsNegative"); + var zero = default(T); // Handle special case for floating point NaN values diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs b/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs index 6923b71fbf..ecd743b0c9 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs @@ -23,6 +23,8 @@ public sealed partial class Assert [DoesNotReturn] public static void Inconclusive(string message = "") { + TelemetryCollector.TrackAssertionCall("Assert.Inconclusive"); + string userMessage = BuildUserMessage(message); throw new AssertInconclusiveException( FormatAssertionFailed("Assert.Inconclusive", userMessage)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs index 7961cda03e..5ac767cca9 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs @@ -285,6 +285,8 @@ internal void ComputeAssertion(string valueExpression) /// public static void IsExactInstanceOfType([NotNull] object? value, [NotNull] Type? expectedType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsExactInstanceOfType"); + if (IsExactInstanceOfTypeFailing(value, expectedType)) { ReportAssertIsExactInstanceOfTypeFailed(value, expectedType, message, valueExpression); @@ -296,7 +298,10 @@ public static void IsExactInstanceOfType([NotNull] object? value, [NotNull] Type public static void IsExactInstanceOfType([NotNull] object? value, [NotNull] Type? expectedType, [InterpolatedStringHandlerArgument(nameof(value), nameof(expectedType))] ref AssertIsExactInstanceOfTypeInterpolatedStringHandler message, [CallerArgumentExpression(nameof(value))] string valueExpression = "") #pragma warning restore IDE0060 // Remove unused parameter #pragma warning disable CS8777 // Parameter must have a non-null value when exiting. - Deliberately keeping [NotNull] annotation while using soft assertions. Within an AssertScope, the postcondition is not enforced (same as all other assertion postconditions in scoped mode). - => message.ComputeAssertion(valueExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.IsExactInstanceOfType"); + message.ComputeAssertion(valueExpression); + } #pragma warning restore CS8777 // Parameter must have a non-null value when exiting. /// @@ -316,6 +321,7 @@ public static T IsExactInstanceOfType([NotNull] object? value, [InterpolatedS #pragma warning restore IDE0060 // Remove unused parameter #pragma warning disable CS8777 // Parameter must have a non-null value when exiting. - Deliberately keeping [NotNull] annotation while using soft assertions. Within an AssertScope, the postcondition is not enforced (same as all other assertion postconditions in scoped mode). { + TelemetryCollector.TrackAssertionCall("Assert.IsExactInstanceOfType"); message.ComputeAssertion(valueExpression); return (T)value!; } @@ -372,6 +378,8 @@ private static void ReportAssertIsExactInstanceOfTypeFailed(object? value, Type? /// public static void IsNotExactInstanceOfType(object? value, [NotNull] Type? wrongType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotExactInstanceOfType"); + if (IsNotExactInstanceOfTypeFailing(value, wrongType)) { ReportAssertIsNotExactInstanceOfTypeFailed(value, wrongType, message, valueExpression); @@ -383,7 +391,10 @@ public static void IsNotExactInstanceOfType(object? value, [NotNull] Type? wrong public static void IsNotExactInstanceOfType(object? value, [NotNull] Type? wrongType, [InterpolatedStringHandlerArgument(nameof(value), nameof(wrongType))] ref AssertIsNotExactInstanceOfTypeInterpolatedStringHandler message, [CallerArgumentExpression(nameof(value))] string valueExpression = "") #pragma warning restore IDE0060 // Remove unused parameter #pragma warning disable CS8777 // Parameter must have a non-null value when exiting. - Not sure how to express the semantics to the compiler, but the implementation guarantees that. - => message.ComputeAssertion(valueExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.IsNotExactInstanceOfType"); + message.ComputeAssertion(valueExpression); + } #pragma warning restore CS8777 // Parameter must have a non-null value when exiting. /// @@ -398,7 +409,10 @@ public static void IsNotExactInstanceOfType(object? value, string? message = #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void IsNotExactInstanceOfType(object? value, [InterpolatedStringHandlerArgument(nameof(value))] ref AssertGenericIsNotExactInstanceOfTypeInterpolatedStringHandler message, [CallerArgumentExpression(nameof(value))] string valueExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(valueExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.IsNotExactInstanceOfType"); + message.ComputeAssertion(valueExpression); + } private static bool IsNotExactInstanceOfTypeFailing(object? value, [NotNullWhen(false)] Type? wrongType) => wrongType is null || diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs index a9a9231217..872c70da1f 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs @@ -286,6 +286,8 @@ internal void ComputeAssertion(string valueExpression) /// public static void IsInstanceOfType([NotNull] object? value, [NotNull] Type? expectedType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsInstanceOfType"); + if (IsInstanceOfTypeFailing(value, expectedType)) { ReportAssertIsInstanceOfTypeFailed(value, expectedType, message, valueExpression); @@ -297,7 +299,10 @@ public static void IsInstanceOfType([NotNull] object? value, [NotNull] Type? exp public static void IsInstanceOfType([NotNull] object? value, [NotNull] Type? expectedType, [InterpolatedStringHandlerArgument(nameof(value), nameof(expectedType))] ref AssertIsInstanceOfTypeInterpolatedStringHandler message, [CallerArgumentExpression(nameof(value))] string valueExpression = "") #pragma warning restore IDE0060 // Remove unused parameter #pragma warning disable CS8777 // Parameter must have a non-null value when exiting. - Deliberately keeping [NotNull] annotation while using soft assertions. Within an AssertScope, the postcondition is not enforced (same as all other assertion postconditions in scoped mode). - => message.ComputeAssertion(valueExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.IsInstanceOfType"); + message.ComputeAssertion(valueExpression); + } #pragma warning restore CS8777 // Parameter must have a non-null value when exiting. /// @@ -318,6 +323,7 @@ public static T IsInstanceOfType([NotNull] object? value, [InterpolatedString #pragma warning restore IDE0060 // Remove unused parameter #pragma warning disable CS8777 // Parameter must have a non-null value when exiting. - Deliberately keeping [NotNull] annotation while using soft assertions. Within an AssertScope, the postcondition is not enforced (same as all other assertion postconditions in scoped mode). { + TelemetryCollector.TrackAssertionCall("Assert.IsInstanceOfType"); message.ComputeAssertion(valueExpression); return (T)value!; } @@ -375,6 +381,8 @@ private static void ReportAssertIsInstanceOfTypeFailed(object? value, Type? expe /// public static void IsNotInstanceOfType(object? value, [NotNull] Type? wrongType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotInstanceOfType"); + if (IsNotInstanceOfTypeFailing(value, wrongType)) { ReportAssertIsNotInstanceOfTypeFailed(value, wrongType, message, valueExpression); @@ -386,7 +394,10 @@ public static void IsNotInstanceOfType(object? value, [NotNull] Type? wrongType, public static void IsNotInstanceOfType(object? value, [NotNull] Type? wrongType, [InterpolatedStringHandlerArgument(nameof(value), nameof(wrongType))] ref AssertIsNotInstanceOfTypeInterpolatedStringHandler message, [CallerArgumentExpression(nameof(value))] string valueExpression = "") #pragma warning restore IDE0060 // Remove unused parameter #pragma warning disable CS8777 // Parameter must have a non-null value when exiting. - Not sure how to express the semantics to the compiler, but the implementation guarantees that. - => message.ComputeAssertion(valueExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.IsNotInstanceOfType"); + message.ComputeAssertion(valueExpression); + } #pragma warning restore CS8777 // Parameter must have a non-null value when exiting. /// @@ -402,7 +413,10 @@ public static void IsNotInstanceOfType(object? value, string? message = "", [ #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void IsNotInstanceOfType(object? value, [InterpolatedStringHandlerArgument(nameof(value))] AssertGenericIsNotInstanceOfTypeInterpolatedStringHandler message, [CallerArgumentExpression(nameof(value))] string valueExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(valueExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.IsNotInstanceOfType"); + message.ComputeAssertion(valueExpression); + } private static bool IsNotInstanceOfTypeFailing(object? value, [NotNullWhen(false)] Type? wrongType) => wrongType is null || diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs index 1a9a7c9162..033347aa30 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs @@ -128,7 +128,10 @@ internal void ComputeAssertion(string valueExpression) #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void IsNull(object? value, [InterpolatedStringHandlerArgument(nameof(value))] ref AssertIsNullInterpolatedStringHandler message, [CallerArgumentExpression(nameof(value))] string valueExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(valueExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.IsNull"); + message.ComputeAssertion(valueExpression); + } /// /// Tests whether the specified object is null and throws an exception @@ -150,6 +153,8 @@ public static void IsNull(object? value, [InterpolatedStringHandlerArgument(name /// public static void IsNull(object? value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNull"); + if (IsNullFailing(value)) { ReportAssertIsNullFailed(value, message, valueExpression); @@ -179,7 +184,10 @@ private static void ReportAssertIsNullFailed(object? value, string? message, str public static void IsNotNull([NotNull] object? value, [InterpolatedStringHandlerArgument(nameof(value))] ref AssertIsNotNullInterpolatedStringHandler message, [CallerArgumentExpression(nameof(value))] string valueExpression = "") #pragma warning restore IDE0060 // Remove unused parameter #pragma warning disable CS8777 // Parameter must have a non-null value when exiting. - Deliberately keeping [NotNull] annotation while using soft assertions. Within an AssertScope, the postcondition is not enforced (same as all other assertion postconditions in scoped mode). - => message.ComputeAssertion(valueExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.IsNotNull"); + message.ComputeAssertion(valueExpression); + } #pragma warning restore CS8777 // Parameter must have a non-null value when exiting. /// @@ -202,6 +210,8 @@ public static void IsNotNull([NotNull] object? value, [InterpolatedStringHandler /// public static void IsNotNull([NotNull] object? value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotNull"); + if (IsNotNullFailing(value)) { ReportAssertIsNotNullFailed(message, valueExpression, nameof(value)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs index 4f24f6200f..88647fff5a 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs @@ -128,7 +128,10 @@ internal void ComputeAssertion(string conditionExpression) #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void IsTrue([DoesNotReturnIf(false)] bool? condition, [InterpolatedStringHandlerArgument(nameof(condition))] ref AssertIsTrueInterpolatedStringHandler message, [CallerArgumentExpression(nameof(condition))] string conditionExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(conditionExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.IsTrue"); + message.ComputeAssertion(conditionExpression); + } /// /// Tests whether the specified condition is true and throws an exception @@ -150,6 +153,8 @@ public static void IsTrue([DoesNotReturnIf(false)] bool? condition, [Interpolate /// public static void IsTrue([DoesNotReturnIf(false)] bool? condition, string? message = "", [CallerArgumentExpression(nameof(condition))] string conditionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsTrue"); + if (IsTrueFailing(condition)) { ReportAssertIsTrueFailed(condition, message, conditionExpression); @@ -179,7 +184,10 @@ private static void ReportAssertIsTrueFailed(bool? condition, string? message, s #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static void IsFalse([DoesNotReturnIf(true)] bool? condition, [InterpolatedStringHandlerArgument(nameof(condition))] ref AssertIsFalseInterpolatedStringHandler message, [CallerArgumentExpression(nameof(condition))] string conditionExpression = "") #pragma warning restore IDE0060 // Remove unused parameter - => message.ComputeAssertion(conditionExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.IsFalse"); + message.ComputeAssertion(conditionExpression); + } /// /// Tests whether the specified condition is false and throws an exception @@ -201,6 +209,8 @@ public static void IsFalse([DoesNotReturnIf(true)] bool? condition, [Interpolate /// public static void IsFalse([DoesNotReturnIf(true)] bool? condition, string? message = "", [CallerArgumentExpression(nameof(condition))] string conditionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsFalse"); + if (IsFalseFailing(condition)) { ReportAssertIsFalseFailed(condition, message, conditionExpression); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs b/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs index 23b69859c7..c36ff1ef52 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs @@ -39,6 +39,8 @@ public sealed partial class Assert /// public static void MatchesRegex([NotNull] Regex? pattern, [NotNull] string? value, string? message = "", [CallerArgumentExpression(nameof(pattern))] string patternExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.MatchesRegex"); + CheckParameterNotNull(value, "Assert.MatchesRegex", "value"); CheckParameterNotNull(pattern, "Assert.MatchesRegex", "pattern"); @@ -115,6 +117,8 @@ public static void MatchesRegex([NotNull] string? pattern, [NotNull] string? val /// public static void DoesNotMatchRegex([NotNull] Regex? pattern, [NotNull] string? value, string? message = "", [CallerArgumentExpression(nameof(pattern))] string patternExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotMatchRegex"); + CheckParameterNotNull(value, "Assert.DoesNotMatchRegex", "value"); CheckParameterNotNull(pattern, "Assert.DoesNotMatchRegex", "pattern"); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs b/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs index caf58c2a10..c478b3cb44 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs @@ -72,6 +72,8 @@ public static void StartsWith([NotNull] string? expectedPrefix, [NotNull] string /// public static void StartsWith([NotNull] string? expectedPrefix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(expectedPrefix))] string expectedPrefixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.StartsWith"); + CheckParameterNotNull(value, "Assert.StartsWith", "value"); CheckParameterNotNull(expectedPrefix, "Assert.StartsWith", "expectedPrefix"); if (!value.StartsWith(expectedPrefix, comparisonType)) @@ -144,6 +146,8 @@ public static void DoesNotStartWith([NotNull] string? notExpectedPrefix, [NotNul /// public static void DoesNotStartWith([NotNull] string? notExpectedPrefix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(notExpectedPrefix))] string notExpectedPrefixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotStartWith"); + CheckParameterNotNull(value, "Assert.DoesNotStartWith", "value"); CheckParameterNotNull(notExpectedPrefix, "Assert.DoesNotStartWith", "notExpectedPrefix"); if (value.StartsWith(notExpectedPrefix, comparisonType)) diff --git a/src/TestFramework/TestFramework/Assertions/Assert.That.cs b/src/TestFramework/TestFramework/Assertions/Assert.That.cs index 0894543cc7..382a53f397 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.That.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.That.cs @@ -26,6 +26,8 @@ public static partial class AssertExtensions /// Thrown if the evaluated condition is . public static void That(Expression> condition, string? message = null, [CallerArgumentExpression(nameof(condition))] string? conditionExpression = null) { + TelemetryCollector.TrackAssertionCall("Assert.That"); + if (condition == null) { throw new ArgumentNullException(nameof(condition)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs b/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs index 1f83c35a38..124365b353 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs @@ -227,14 +227,20 @@ public static TException Throws(Func action, Func(Action action, [InterpolatedStringHandlerArgument(nameof(action))] ref AssertNonStrictThrowsInterpolatedStringHandler message, [CallerArgumentExpression(nameof(action))] string actionExpression = "") #pragma warning restore IDE0060 // Remove unused parameter where TException : Exception - => message.ComputeAssertion(actionExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.Throws"); + return message.ComputeAssertion(actionExpression); + } /// #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static TException Throws(Func action, [InterpolatedStringHandlerArgument(nameof(action))] ref AssertNonStrictThrowsInterpolatedStringHandler message, [CallerArgumentExpression(nameof(action))] string actionExpression = "") #pragma warning restore IDE0060 // Remove unused parameter where TException : Exception - => message.ComputeAssertion(actionExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.Throws"); + return message.ComputeAssertion(actionExpression); + } /// /// Asserts that the delegate throws an exception of type @@ -307,18 +313,26 @@ public static TException ThrowsExactly(Func action, Func(Action action, [InterpolatedStringHandlerArgument(nameof(action))] ref AssertThrowsExactlyInterpolatedStringHandler message, [CallerArgumentExpression(nameof(action))] string actionExpression = "") #pragma warning restore IDE0060 // Remove unused parameter where TException : Exception - => message.ComputeAssertion(actionExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.ThrowsExactly"); + return message.ComputeAssertion(actionExpression); + } /// #pragma warning disable IDE0060 // Remove unused parameter - https://github.com/dotnet/roslyn/issues/76578 public static TException ThrowsExactly(Func action, [InterpolatedStringHandlerArgument(nameof(action))] ref AssertThrowsExactlyInterpolatedStringHandler message, [CallerArgumentExpression(nameof(action))] string actionExpression = "") #pragma warning restore IDE0060 // Remove unused parameter where TException : Exception - => message.ComputeAssertion(actionExpression); + { + TelemetryCollector.TrackAssertionCall("Assert.ThrowsExactly"); + return message.ComputeAssertion(actionExpression); + } private static TException ThrowsException(Action action, bool isStrictType, string? message, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(GetTrackedThrowsName(assertMethodName)); + _ = action ?? throw new ArgumentNullException(nameof(action)); _ = message ?? throw new ArgumentNullException(nameof(message)); @@ -339,6 +353,8 @@ private static TException ThrowsException(Action action, bool isStri private static TException ThrowsException(Action action, bool isStrictType, Func messageBuilder, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(GetTrackedThrowsName(assertMethodName)); + _ = action ?? throw new ArgumentNullException(nameof(action)); _ = messageBuilder ?? throw new ArgumentNullException(nameof(messageBuilder)); @@ -475,6 +491,8 @@ public static Task ThrowsExactlyAsync(Func action, private static async Task ThrowsExceptionAsync(Func action, bool isStrictType, string? message, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(GetTrackedThrowsName(assertMethodName)); + _ = action ?? throw new ArgumentNullException(nameof(action)); _ = message ?? throw new ArgumentNullException(nameof(message)); @@ -495,6 +513,8 @@ private static async Task ThrowsExceptionAsync(Func ThrowsExceptionAsync(Func action, bool isStrictType, Func messageBuilder, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(GetTrackedThrowsName(assertMethodName)); + _ = action ?? throw new ArgumentNullException(nameof(action)); _ = messageBuilder ?? throw new ArgumentNullException(nameof(messageBuilder)); @@ -673,4 +693,17 @@ public static ThrowsExceptionState CreateNoExceptionState() public static ThrowsExceptionState CreateNotFailingState(Exception exception) => new(ThrowsFailureKind.NotFailing, exception); } + + // assertMethodName comes from [CallerMemberName] for the Throws/ThrowsExactly/ThrowsAsync/ + // ThrowsExactlyAsync helpers — a small fixed set. Use a switch to avoid allocating a fresh + // "Assert." + name string on every call. + private static string GetTrackedThrowsName(string assertMethodName) + => assertMethodName switch + { + "Throws" => "Assert.Throws", + "ThrowsExactly" => "Assert.ThrowsExactly", + "ThrowsAsync" => "Assert.ThrowsAsync", + "ThrowsExactlyAsync" => "Assert.ThrowsExactlyAsync", + _ => string.Concat("Assert.", assertMethodName), + }; } diff --git a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equality.cs b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equality.cs index 3ace0a74dc..bccbe1cf9a 100644 --- a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equality.cs +++ b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equality.cs @@ -58,6 +58,8 @@ public static void AreEqual(ICollection? expected, ICollection? actual) /// public static void AreEqual(ICollection? expected, ICollection? actual, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreEqual"); + string reason = string.Empty; if (!AreCollectionsEqual(expected, actual, new ObjectComparer(), ref reason)) { @@ -112,6 +114,8 @@ public static void AreNotEqual(ICollection? notExpected, ICollection? actual) /// public static void AreNotEqual(ICollection? notExpected, ICollection? actual, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreNotEqual"); + string reason = string.Empty; if (AreCollectionsEqual(notExpected, actual, new ObjectComparer(), ref reason)) { @@ -170,6 +174,8 @@ public static void AreEqual(ICollection? expected, ICollection? actual, [NotNull /// public static void AreEqual(ICollection? expected, ICollection? actual, [NotNull] IComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreEqual"); + string reason = string.Empty; if (!AreCollectionsEqual(expected, actual, comparer, ref reason)) { @@ -228,6 +234,8 @@ public static void AreNotEqual(ICollection? notExpected, ICollection? actual, [N /// public static void AreNotEqual(ICollection? notExpected, ICollection? actual, [NotNull] IComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreNotEqual"); + string reason = string.Empty; if (AreCollectionsEqual(notExpected, actual, comparer, ref reason)) { diff --git a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equivalence.cs b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equivalence.cs index 022604dd90..a4218b4b86 100644 --- a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equivalence.cs +++ b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Equivalence.cs @@ -118,6 +118,8 @@ public static void AreEquivalent( [NotNullIfNotNull(nameof(actual))] IEnumerable? expected, [NotNullIfNotNull(nameof(expected))] IEnumerable? actual, [NotNull] IEqualityComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreEquivalent"); + Assert.CheckParameterNotNull(comparer, "Assert.AreCollectionsEqual", "comparer"); // Check whether one is null while the other is not. @@ -284,6 +286,8 @@ public static void AreNotEquivalent( [NotNullIfNotNull(nameof(actual))] IEnumerable? notExpected, [NotNullIfNotNull(nameof(notExpected))] IEnumerable? actual, [NotNull] IEqualityComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreNotEquivalent"); + Assert.CheckParameterNotNull(comparer, "Assert.AreCollectionsEqual", "comparer"); // Check whether one is null while the other is not. diff --git a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Membership.cs b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Membership.cs index 189ffadae1..5203a843eb 100644 --- a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Membership.cs +++ b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Membership.cs @@ -50,6 +50,8 @@ public static void Contains([NotNull] ICollection? collection, object? element) /// public static void Contains([NotNull] ICollection? collection, object? element, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.Contains"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.Contains", "collection"); foreach (object? current in collection) @@ -101,6 +103,8 @@ public static void DoesNotContain([NotNull] ICollection? collection, object? ele /// public static void DoesNotContain([NotNull] ICollection? collection, object? element, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.DoesNotContain"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.DoesNotContain", "collection"); foreach (object? current in collection) @@ -141,6 +145,8 @@ public static void AllItemsAreNotNull([NotNull] ICollection? collection) /// public static void AllItemsAreNotNull([NotNull] ICollection? collection, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AllItemsAreNotNull"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.AllItemsAreNotNull", "collection"); foreach (object? current in collection) { @@ -183,6 +189,8 @@ public static void AllItemsAreUnique([NotNull] ICollection? collection) /// public static void AllItemsAreUnique([NotNull] ICollection? collection, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AllItemsAreUnique"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.AllItemsAreUnique", "collection"); message = Assert.ReplaceNulls(message); diff --git a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Subset.cs b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Subset.cs index 7d5fff305b..4c9f2f9d19 100644 --- a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Subset.cs +++ b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Subset.cs @@ -54,6 +54,8 @@ public static void IsSubsetOf([NotNull] ICollection? subset, [NotNull] ICollecti /// public static void IsSubsetOf([NotNull] ICollection? subset, [NotNull] ICollection? superset, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.IsSubsetOf"); + Assert.CheckParameterNotNull(subset, "CollectionAssert.IsSubsetOf", "subset"); Assert.CheckParameterNotNull(superset, "CollectionAssert.IsSubsetOf", "superset"); Tuple> isSubsetValue = IsSubsetOfHelper(subset, superset); @@ -114,6 +116,8 @@ public static void IsNotSubsetOf([NotNull] ICollection? subset, [NotNull] IColle /// public static void IsNotSubsetOf([NotNull] ICollection? subset, [NotNull] ICollection? superset, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.IsNotSubsetOf"); + Assert.CheckParameterNotNull(subset, "CollectionAssert.IsNotSubsetOf", "subset"); Assert.CheckParameterNotNull(superset, "CollectionAssert.IsNotSubsetOf", "superset"); Tuple> isSubsetValue = IsSubsetOfHelper(subset, superset); diff --git a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Type.cs b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Type.cs index eeae04b966..72dc9fd796 100644 --- a/src/TestFramework/TestFramework/Assertions/CollectionAssert.Type.cs +++ b/src/TestFramework/TestFramework/Assertions/CollectionAssert.Type.cs @@ -57,6 +57,8 @@ public static void AllItemsAreInstancesOfType([NotNull] ICollection? collection, public static void AllItemsAreInstancesOfType( [NotNull] ICollection? collection, [NotNull] Type? expectedType, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AllItemsAreInstancesOfType"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.AllItemsAreInstancesOfType", "collection"); Assert.CheckParameterNotNull(expectedType, "CollectionAssert.AllItemsAreInstancesOfType", "expectedType"); int i = 0; diff --git a/src/TestFramework/TestFramework/Assertions/StringAssert.cs b/src/TestFramework/TestFramework/Assertions/StringAssert.cs index 59d3f35fe2..f3bebb718f 100644 --- a/src/TestFramework/TestFramework/Assertions/StringAssert.cs +++ b/src/TestFramework/TestFramework/Assertions/StringAssert.cs @@ -116,6 +116,8 @@ public static void Contains([NotNull] string? value, [NotNull] string? substring /// public static void Contains([NotNull] string? value, [NotNull] string? substring, StringComparison comparisonType, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.Contains"); + Assert.CheckParameterNotNull(value, "StringAssert.Contains", "value"); Assert.CheckParameterNotNull(substring, "StringAssert.Contains", "substring"); if (value.IndexOf(substring, comparisonType) < 0) @@ -213,6 +215,8 @@ public static void StartsWith([NotNull] string? value, [NotNull] string? substri /// public static void StartsWith([NotNull] string? value, [NotNull] string? substring, StringComparison comparisonType, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.StartsWith"); + Assert.CheckParameterNotNull(value, "StringAssert.StartsWith", "value"); Assert.CheckParameterNotNull(substring, "StringAssert.StartsWith", "substring"); if (!value.StartsWith(substring, comparisonType)) @@ -310,6 +314,8 @@ public static void EndsWith([NotNull] string? value, [NotNull] string? substring /// public static void EndsWith([NotNull] string? value, [NotNull] string? substring, StringComparison comparisonType, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.EndsWith"); + Assert.CheckParameterNotNull(value, "StringAssert.EndsWith", "value"); Assert.CheckParameterNotNull(substring, "StringAssert.EndsWith", "substring"); if (!value.EndsWith(substring, comparisonType)) @@ -364,6 +370,8 @@ public static void Matches([NotNull] string? value, [NotNull] Regex? pattern) /// public static void Matches([NotNull] string? value, [NotNull] Regex? pattern, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.Matches"); + Assert.CheckParameterNotNull(value, "StringAssert.Matches", "value"); Assert.CheckParameterNotNull(pattern, "StringAssert.Matches", "pattern"); @@ -415,6 +423,8 @@ public static void DoesNotMatch([NotNull] string? value, [NotNull] Regex? patter /// public static void DoesNotMatch([NotNull] string? value, [NotNull] Regex? pattern, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.DoesNotMatch"); + Assert.CheckParameterNotNull(value, "StringAssert.DoesNotMatch", "value"); Assert.CheckParameterNotNull(pattern, "StringAssert.DoesNotMatch", "pattern"); diff --git a/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs b/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs new file mode 100644 index 0000000000..faa1bf3fc2 --- /dev/null +++ b/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Collects aggregated telemetry data about MSTest API usage within a test session. +/// This data is used to understand which APIs are heavily used or unused to guide future investment. +/// +internal static class TelemetryCollector +{ + // Lazily evaluated opt-out flag. Mirrors the env-var checks performed by the adapter + // (and by Microsoft.Testing.Platform's TelemetryManager) so the assertion hot path can + // short-circuit when the user has opted out. Lazy guarantees the env-var lookup + // happens at most once per process. + private static readonly Lazy IsEnabled = new(IsTelemetryEnabledFromEnvironment, LazyThreadSafetyMode.ExecutionAndPublication); + + private static ConcurrentDictionary s_assertionCallCounts = new(); + + /// + /// Records that an assertion method was called. This is on the hot path of every assertion, + /// so it is aggressively inlined and short-circuits when the user has opted out of telemetry + /// via TESTINGPLATFORM_TELEMETRY_OPTOUT or DOTNET_CLI_TELEMETRY_OPTOUT. + /// Any unexpected exception (e.g. ) is swallowed so + /// telemetry never alters user-visible test behavior. + /// + /// The full name of the assertion (e.g. "Assert.AreEqual", "CollectionAssert.Contains"). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void TrackAssertionCall(string assertionName) + { + if (!IsEnabled.Value) + { + return; + } + + try + { + s_assertionCallCounts.AddOrUpdate(assertionName, 1, static (_, count) => count + 1); + } + catch + { + // Telemetry must never affect test outcomes. + } + } + + /// + /// Gets a snapshot of all assertion call counts and resets the counters. + /// This is thread-safe but best-effort: it atomically swaps the dictionary and copies the old one. + /// In-flight calls to that race with the swap may be lost. + /// This is acceptable for telemetry where approximate counts are sufficient. + /// + /// A dictionary mapping assertion names to their (best-effort) call counts. + internal static Dictionary DrainAssertionCallCounts() + { + ConcurrentDictionary old = Interlocked.Exchange(ref s_assertionCallCounts, new ConcurrentDictionary()); + + // Use the explicit Dictionary(IEnumerable) ctor so we get a stable + // snapshot of the swapped-out instance. A collection-expression spread would be + // semantically equivalent but the explicit ctor reads more clearly here. +#pragma warning disable IDE0028 // Simplify collection initialization + return new Dictionary(old); +#pragma warning restore IDE0028 + } + + private static bool IsTelemetryEnabledFromEnvironment() + { + try + { + string? telemetryOptOut = Environment.GetEnvironmentVariable("TESTINGPLATFORM_TELEMETRY_OPTOUT"); + if (string.Equals(telemetryOptOut, "1", StringComparison.Ordinal) || + string.Equals(telemetryOptOut, "true", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + string? cliTelemetryOptOut = Environment.GetEnvironmentVariable("DOTNET_CLI_TELEMETRY_OPTOUT"); + return !string.Equals(cliTelemetryOptOut, "1", StringComparison.Ordinal) + && !string.Equals(cliTelemetryOptOut, "true", StringComparison.OrdinalIgnoreCase); + } + catch + { + // If we cannot read environment variables (e.g. partial-trust scenarios), treat + // that as opted out — telemetry must never affect test behavior. + return false; + } + } +} diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs new file mode 100644 index 0000000000..b09a2e72cb --- /dev/null +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs @@ -0,0 +1,302 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Acceptance.IntegrationTests; +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; +using Microsoft.Testing.Platform.Helpers; + +namespace MSTest.Acceptance.IntegrationTests; + +[TestClass] +public sealed class TelemetryTests : AcceptanceTestBase +{ + private const string MTPAssetName = "TelemetryMTPProject"; + private const string TestResultsFolderName = "TestResults"; + + #region MTP mode - Run + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task MTP_RunTests_SendsTelemetryWithSettingsAndAttributes(string tfm) + { + string diagPath = Path.Combine(AssetFixture.MTPProjectPath, "bin", "Release", tfm, TestResultsFolderName); + string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); + + var testHost = TestHost.LocateFrom(AssetFixture.MTPProjectPath, MTPAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--diagnostic", + disableTelemetry: false, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + // In MTP run mode, MTP only invokes the executor (no separate discovery call), so the + // single sessionexit event carries the full picture: settings, config_source, attribute + // usage, and assertion usage. + string diagContentsPattern = +""" +.+ Send telemetry event: dotnet/testingplatform/mstest/sessionexit +.+mstest\.setting\.parallelization_enabled +"""; + string diagFilePath = await AssertDiagnosticReportAsync(testHostResult, diagPathPattern, diagContentsPattern); + + string content = await File.ReadAllTextAsync(diagFilePath, TestContext.CancellationToken); + Assert.IsTrue(Regex.IsMatch(content, "mstest\\.attribute_usage"), $"Expected attribute_usage in telemetry.\n{content}"); + Assert.IsTrue(Regex.IsMatch(content, "mstest\\.config_source"), $"Expected config_source in telemetry.\n{content}"); + Assert.IsTrue(Regex.IsMatch(content, "mstest\\.assertion_usage"), $"Expected assertion_usage in telemetry.\n{content}"); + + // Regression guard: discovery + execution data must ship in a single sessionexit event, + // not split across two (an earlier iteration of this code did the latter). + MatchCollection sessionExitEvents = Regex.Matches(content, "Send telemetry event: dotnet/testingplatform/mstest/sessionexit"); + Assert.HasCount(1, sessionExitEvents, $"Expected exactly one MSTest sessionexit telemetry event, found {sessionExitEvents.Count}.\n{content}"); + } + + #endregion + + #region MTP mode - Discovery only + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task MTP_DiscoverTests_SendsTelemetryEvent(string tfm) + { + string diagPath = Path.Combine(AssetFixture.MTPProjectPath, "bin", "Release", tfm, TestResultsFolderName); + string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); + + var testHost = TestHost.LocateFrom(AssetFixture.MTPProjectPath, MTPAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--list-tests --diagnostic", + disableTelemetry: false, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + string diagContentsPattern = +""" +.+ Send telemetry event: dotnet/testingplatform/mstest/discovery[\s\S]+?mstest\.attribute_usage +"""; + await AssertDiagnosticReportAsync(testHostResult, diagPathPattern, diagContentsPattern); + } + + #endregion + + #region MTP mode - Telemetry disabled + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task MTP_WhenTelemetryDisabled_DoesNotSendMSTestEvent(string tfm) + { + string diagPath = Path.Combine(AssetFixture.MTPProjectPath, "bin", "Release", tfm, TestResultsFolderName); + string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); + + var testHost = TestHost.LocateFrom(AssetFixture.MTPProjectPath, MTPAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--diagnostic", + new Dictionary + { + { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" }, + }, + disableTelemetry: false, + TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + string diagContentsPattern = +""" +.+ Microsoft.Testing.Platform.Telemetry.TelemetryManager DEBUG Telemetry is 'DISABLED' +"""; + string diagFilePath = await AssertDiagnosticReportAsync(testHostResult, diagPathPattern, diagContentsPattern); + + string content = await File.ReadAllTextAsync(diagFilePath, TestContext.CancellationToken); + Assert.IsFalse( + Regex.IsMatch(content, "Send telemetry event: dotnet/testingplatform/mstest/sessionexit"), + "MSTest sessionexit telemetry event should not be sent when telemetry is disabled."); + Assert.IsFalse( + Regex.IsMatch(content, "Send telemetry event: dotnet/testingplatform/mstest/discovery"), + "MSTest discovery telemetry event should not be sent when telemetry is disabled."); + } + + #endregion + + #region VSTest mode - Run + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task VSTest_RunTests_Succeeds(string tfm) + { + DotnetMuxerResult testResult = await DotnetCli.RunAsync( + $"test -c Release {AssetFixture.VSTestProjectPath} --framework {tfm}", + workingDirectory: AssetFixture.VSTestProjectPath, + failIfReturnValueIsNotZero: false, + cancellationToken: TestContext.CancellationToken); + + Assert.AreEqual(0, testResult.ExitCode, $"dotnet test failed:\n{testResult.StandardOutput}\n{testResult.StandardError}"); + testResult.AssertOutputContains("Passed!"); + } + + #endregion + + #region VSTest mode - Discovery only + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task VSTest_DiscoverTests_Succeeds(string tfm) + { + DotnetMuxerResult testResult = await DotnetCli.RunAsync( + $"test -c Release {AssetFixture.VSTestProjectPath} --framework {tfm} --list-tests", + workingDirectory: AssetFixture.VSTestProjectPath, + failIfReturnValueIsNotZero: false, + cancellationToken: TestContext.CancellationToken); + + Assert.AreEqual(0, testResult.ExitCode, $"dotnet test --list-tests failed:\n{testResult.StandardOutput}\n{testResult.StandardError}"); + testResult.AssertOutputContains("PassingTest"); + testResult.AssertOutputContains("DataDrivenTest"); + testResult.AssertOutputContains("TestWithTimeout"); + } + + #endregion + + #region Helpers + + private static async Task AssertDiagnosticReportAsync(TestHostResult testHostResult, string diagPathPattern, string diagContentsPattern, string level = "Trace", string flushType = "async") + { + string outputPattern = $""" +Diagnostic file \(level '{level}' with {flushType} flush\): {diagPathPattern} +"""; + testHostResult.AssertOutputMatchesRegex(outputPattern); + Match match = Regex.Match(testHostResult.StandardOutput, diagPathPattern); + Assert.IsTrue(match.Success, $"{testHostResult}\n{diagPathPattern}"); + + (bool success, string content) = await CheckDiagnosticContentsMatchAsync(match.Value, diagContentsPattern); + Assert.IsTrue(success, $"{content}\n{diagContentsPattern}"); + + return match.Value; + } + + private static async Task<(bool IsMatch, string Content)> CheckDiagnosticContentsMatchAsync(string path, string pattern) + { + using var reader = new StreamReader(path); + string content = await reader.ReadToEndAsync(); + + return (Regex.IsMatch(content, pattern), content); + } + + #endregion + + public sealed class TestAssetFixture() : TestAssetFixtureBase() + { + private const string AssetId = nameof(TelemetryTests); + + public string MTPProjectPath => GetAssetPath(AssetId); + + public string VSTestProjectPath => Path.Combine(GetAssetPath(AssetId), "vstest"); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() + => (AssetId, MTPAssetName, + SourceCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$MicrosoftNETTestSdkVersion$", MicrosoftNETTestSdkVersion)); + + private const string SourceCode = """ +#file TelemetryMTPProject.csproj + + + + Exe + true + $TargetFrameworks$ + Preview + + + + + + + + + + + + + +#file UnitTest1.cs +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void PassingTest() + { + } + + [TestMethod] + [DataRow(1)] + [DataRow(2)] + public void DataDrivenTest(int value) + { + Assert.IsTrue(value > 0); + } + + [TestMethod] + [Timeout(30000)] + public void TestWithTimeout() + { + } +} + +#file vstest/TelemetryVSTestProject.csproj + + + + $TargetFrameworks$ + Preview + false + true + + + + + + + + + + +#file vstest/global.json +{ + "test": { + "runner": "VSTest" + } +} + +#file vstest/UnitTest1.cs +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void PassingTest() + { + } + + [TestMethod] + [DataRow(1)] + [DataRow(2)] + public void DataDrivenTest(int value) + { + Assert.IsTrue(value > 0); + } + + [TestMethod] + [Timeout(30000)] + public void TestWithTimeout() + { + } +} +"""; + } + + public TestContext TestContext { get; set; } +}