From 06ad3e104cfb9628a54ea58f1ab810fa285a8d0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Tue, 17 Mar 2026 14:27:25 +0100 Subject: [PATCH 1/9] Add telemetry collection for MSTest usage analytics Add infrastructure to collect aggregated telemetry about MSTest usage: - Track assertion API usage (Assert, CollectionAssert, StringAssert) - Track attribute usage and custom types during discovery - Track MSTest configuration settings and source - Send telemetry via MTP telemetry collector on session exit Key implementation details: - TelemetryCollector: thread-safe static counter in TestFramework using ConcurrentDictionary with atomic swap-and-drain pattern - MSTestTelemetryDataCollector: aggregates discovery and assertion data, builds metrics, anonymizes custom type names via SHA256 - Telemetry is opt-in via MTP telemetry infrastructure (respects TESTINGPLATFORM_TELEMETRY_OPTOUT) - VSTest mode collects but discards data (no telemetry sender available) Fixes applied during review: - Fix race condition in DrainAssertionCallCounts using Interlocked.Exchange - Fix thread safety of Current property using Volatile.Read/Write - Add synchronous SendTelemetryAndReset to avoid deadlock in sync callers - Use IDictionary delegate type for consistency with MTP - Replace bare catch with catch (Exception) - Remove duplicate _serviceProvider field (use inherited ServiceProvider) - Add missing ContainsSingle telemetry tracking - Replace string interpolation with string.Concat in hot paths - Standardize blank lines after TrackAssertionCall calls --- .../MSTestBridgedTestFramework.cs | 19 +- .../VSTestAdapter/MSTestDiscoverer.cs | 22 +- .../VSTestAdapter/MSTestExecutor.cs | 31 +- .../Discovery/AssemblyEnumerator.cs | 2 +- .../Discovery/TypeEnumerator.cs | 13 +- .../MSTestSettings.cs | 12 + .../Telemetry/MSTestTelemetryDataCollector.cs | 288 +++++++++++++++++ .../Microsoft.Testing.Platform.csproj | 1 + .../Assertions/Assert.AreEqual.cs | 24 ++ .../Assertions/Assert.AreSame.cs | 4 + .../Assertions/Assert.Contains.cs | 34 ++ .../TestFramework/Assertions/Assert.Count.cs | 6 + .../Assertions/Assert.EndsWith.cs | 4 + .../TestFramework/Assertions/Assert.Fail.cs | 5 +- .../Assertions/Assert.IComparable.cs | 12 + .../Assertions/Assert.Inconclusive.cs | 2 + .../Assert.IsExactInstanceOfType.cs | 4 + .../Assertions/Assert.IsInstanceOfType.cs | 4 + .../TestFramework/Assertions/Assert.IsNull.cs | 4 + .../TestFramework/Assertions/Assert.IsTrue.cs | 4 + .../Assertions/Assert.Matches.cs | 4 + .../Assertions/Assert.StartsWith.cs | 4 + .../Assertions/Assert.ThrowsException.cs | 8 + .../Assertions/CollectionAssert.cs | 26 ++ .../TestFramework/Assertions/StringAssert.cs | 10 + .../Internal/TelemetryCollector.cs | 34 ++ .../TelemetryTests.cs | 294 ++++++++++++++++++ 27 files changed, 863 insertions(+), 12 deletions(-) create mode 100644 src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs create mode 100644 src/TestFramework/TestFramework/Internal/TelemetryCollector.cs create mode 100644 test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs index 55e200cd73..5e9c52be45 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs @@ -9,6 +9,7 @@ using Microsoft.Testing.Platform.Logging; using Microsoft.Testing.Platform.Messages; using Microsoft.Testing.Platform.Services; +using Microsoft.Testing.Platform.Telemetry; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Helpers; @@ -41,7 +42,7 @@ protected override Task SynchronizedDiscoverTestsAsync(VSTestDiscoverTestExecuti Debugger.Launch(); } - new MSTestDiscoverer().DiscoverTests(request.AssemblyPaths, request.DiscoveryContext, request.MessageLogger, request.DiscoverySink, _configuration); + new MSTestDiscoverer(new TestSourceHandler(), CreateTelemetrySender()).DiscoverTests(request.AssemblyPaths, request.DiscoveryContext, request.MessageLogger, request.DiscoverySink, _configuration); return Task.CompletedTask; } @@ -55,7 +56,7 @@ protected override async Task SynchronizedRunTestsAsync(VSTestRunTestExecutionRe Debugger.Launch(); } - MSTestExecutor testExecutor = new(cancellationToken); + MSTestExecutor testExecutor = new(cancellationToken, CreateTelemetrySender()); await testExecutor.RunTestsAsync(request.AssemblyPaths, request.RunContext, request.FrameworkHandle, _configuration).ConfigureAwait(false); } @@ -103,5 +104,19 @@ private static TestMethodIdentifierProperty GetMethodIdentifierPropertyFromManag // Or alternatively, does VSTest object model expose the assembly full name somewhere? return new TestMethodIdentifierProperty(assemblyFullName: string.Empty, @namespace, typeName, methodName, arity, parameterTypes, returnTypeFullName: string.Empty); } + + [SuppressMessage("ApiDesign", "RS0030:Do not use banned APIs", Justification = "We can use MTP from this folder")] + private Func, Task>? CreateTelemetrySender() + { + ITelemetryInformation telemetryInformation = ServiceProvider.GetTelemetryInformation(); + if (!telemetryInformation.IsEnabled) + { + return null; + } + + ITelemetryCollector telemetryCollector = ServiceProvider.GetTelemetryCollector(); + + return (eventName, metrics) => telemetryCollector.LogEventAsync(eventName, metrics, CancellationToken.None); + } } #endif diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs index 3a3f24bf76..b34271447f 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs @@ -20,14 +20,18 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; internal sealed class MSTestDiscoverer : ITestDiscoverer { private readonly ITestSourceHandler _testSourceHandler; + private readonly Func, Task>? _telemetrySender; public MSTestDiscoverer() : this(new TestSourceHandler()) { } - internal /* for testing purposes */ MSTestDiscoverer(ITestSourceHandler testSourceHandler) - => _testSourceHandler = testSourceHandler; + internal /* for testing purposes */ MSTestDiscoverer(ITestSourceHandler testSourceHandler, Func, Task>? telemetrySender = null) + { + _testSourceHandler = testSourceHandler; + _telemetrySender = telemetrySender; + } /// /// Discovers the tests available from the provided source. Not supported for .xap source. @@ -47,9 +51,19 @@ internal void DiscoverTests(IEnumerable sources, IDiscoveryContext disco Ensure.NotNull(logger); Ensure.NotNull(discoverySink); - if (MSTestDiscovererHelpers.InitializeDiscovery(sources, discoveryContext, logger, configuration, _testSourceHandler)) + // Initialize telemetry collection if not already set (e.g. first call in the session) + MSTestTelemetryDataCollector.Current ??= new MSTestTelemetryDataCollector(); + + try + { + if (MSTestDiscovererHelpers.InitializeDiscovery(sources, discoveryContext, logger, configuration, _testSourceHandler)) + { + new UnitTestDiscoverer(_testSourceHandler).DiscoverTests(sources, logger, discoverySink, discoveryContext); + } + } + finally { - new UnitTestDiscoverer(_testSourceHandler).DiscoverTests(sources, logger, discoverySink, discoveryContext); + MSTestTelemetryDataCollector.SendTelemetryAndReset(_telemetrySender); } } } diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs index 8fcb749311..1b84d52365 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs @@ -20,6 +20,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; internal sealed class MSTestExecutor : ITestExecutor { private readonly CancellationToken _cancellationToken; + private readonly Func, Task>? _telemetrySender; /// /// Token for canceling the test run. @@ -35,10 +36,11 @@ public MSTestExecutor() _cancellationToken = CancellationToken.None; } - internal MSTestExecutor(CancellationToken cancellationToken) + internal MSTestExecutor(CancellationToken cancellationToken, Func, Task>? telemetrySender = null) { TestExecutionManager = new TestExecutionManager(); _cancellationToken = cancellationToken; + _telemetrySender = telemetrySender; } /// @@ -105,12 +107,22 @@ internal async Task RunTestsAsync(IEnumerable? tests, IRunContext? run Ensure.NotNull(frameworkHandle); Ensure.NotNullOrEmpty(tests); + // Initialize telemetry collection if not already set + MSTestTelemetryDataCollector.Current ??= new MSTestTelemetryDataCollector(); + if (!MSTestDiscovererHelpers.InitializeDiscovery(from test in tests select test.Source, runContext, frameworkHandle, configuration, new TestSourceHandler())) { return; } - await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(tests, runContext, frameworkHandle, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + try + { + await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(tests, runContext, frameworkHandle, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + } + finally + { + await SendTelemetryAsync().ConfigureAwait(false); + } } internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? runContext, IFrameworkHandle? frameworkHandle, IConfiguration? configuration) @@ -123,6 +135,9 @@ internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? run Ensure.NotNull(frameworkHandle); Ensure.NotNullOrEmpty(sources); + // Initialize telemetry collection if not already set + MSTestTelemetryDataCollector.Current ??= new MSTestTelemetryDataCollector(); + TestSourceHandler testSourceHandler = new(); if (!MSTestDiscovererHelpers.InitializeDiscovery(sources, runContext, frameworkHandle, configuration, testSourceHandler)) { @@ -130,7 +145,14 @@ internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? run } sources = testSourceHandler.GetTestSources(sources); - await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(sources, runContext, frameworkHandle, testSourceHandler, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + try + { + await RunTestsFromRightContextAsync(frameworkHandle, async testRunToken => await TestExecutionManager.RunTestsAsync(sources, runContext, frameworkHandle, testSourceHandler, testRunToken).ConfigureAwait(false)).ConfigureAwait(false); + } + finally + { + await SendTelemetryAsync().ConfigureAwait(false); + } } /// @@ -139,6 +161,9 @@ internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? run public void Cancel() => _testRunCancellationToken?.Cancel(); + private async Task SendTelemetryAsync() + => await MSTestTelemetryDataCollector.SendTelemetryAndResetAsync(_telemetrySender).ConfigureAwait(false); + private async Task RunTestsFromRightContextAsync(IFrameworkHandle frameworkHandle, Func runTestsAction) { ApartmentState? requestedApartmentState = MSTestSettings.RunConfigurationSettings.ExecutionApartmentState; diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/AssemblyEnumerator.cs index b3ea7a2df6..3920849371 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/AssemblyEnumerator.cs @@ -145,7 +145,7 @@ internal virtual TypeEnumerator GetTypeEnumerator(Type type, string assemblyFile var typeValidator = new TypeValidator(ReflectHelper, discoverInternals); var testMethodValidator = new TestMethodValidator(ReflectHelper, discoverInternals); - return new TypeEnumerator(type, assemblyFileName, ReflectHelper, typeValidator, testMethodValidator); + return new TypeEnumerator(type, assemblyFileName, ReflectHelper, typeValidator, testMethodValidator, MSTestTelemetryDataCollector.Current); } private List DiscoverTestsInType( diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs index b151d6d594..8e66b42b03 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs @@ -19,6 +19,7 @@ internal class TypeEnumerator private readonly TypeValidator _typeValidator; private readonly TestMethodValidator _testMethodValidator; private readonly ReflectHelper _reflectHelper; + private readonly MSTestTelemetryDataCollector? _telemetryDataCollector; /// /// Initializes a new instance of the class. @@ -28,13 +29,15 @@ internal class TypeEnumerator /// An instance to reflection helper for type information. /// The validator for test classes. /// The validator for test methods. - internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflectHelper, TypeValidator typeValidator, TestMethodValidator testMethodValidator) + /// Optional telemetry data collector for tracking API usage. + internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflectHelper, TypeValidator typeValidator, TestMethodValidator testMethodValidator, MSTestTelemetryDataCollector? telemetryDataCollector = null) { _type = type; _assemblyFilePath = assemblyFilePath; _reflectHelper = reflectHelper; _typeValidator = typeValidator; _testMethodValidator = testMethodValidator; + _telemetryDataCollector = telemetryDataCollector; } /// @@ -49,6 +52,13 @@ internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflec return null; } + // Track class-level attributes for telemetry + if (_telemetryDataCollector is not null) + { + Attribute[] classAttributes = _reflectHelper.GetCustomAttributesCached(_type); + _telemetryDataCollector.TrackDiscoveredClass(_type, classAttributes); + } + // If test class is valid, then get the tests return GetTests(warnings); } @@ -143,6 +153,7 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, bool classDisables }; Attribute[] attributes = _reflectHelper.GetCustomAttributesCached(method); + _telemetryDataCollector?.TrackDiscoveredMethod(attributes); TestMethodAttribute? testMethodAttribute = null; // Backward looping for backcompat. This used to be calls to _reflectHelper.GetFirstAttributeOrDefault diff --git a/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.cs b/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.cs index 501d5559cd..2605b54ee0 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.cs @@ -295,6 +295,18 @@ internal static void PopulateSettings(IDiscoveryContext? context, IMessageLogger CurrentSettings = settings; RunConfigurationSettings = runConfigurationSettings; + + // Track configuration source for telemetry +#if !WINDOWS_UWP + if (MSTestTelemetryDataCollector.Current is { } telemetry) + { + telemetry.ConfigurationSource = configuration?["mstest"] is not null + ? "testconfig.json" + : !StringEx.IsNullOrEmpty(context?.RunSettings?.SettingsXml) + ? "runsettings" + : "none"; + } +#endif } /// diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs new file mode 100644 index 0000000000..8d6ad68e2e --- /dev/null +++ b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs @@ -0,0 +1,288 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#if !WINDOWS_UWP +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; + +/// +/// Collects and aggregates telemetry data about MSTest usage within a test session. +/// Captures settings, attribute usage, custom/inherited types, and assertion API usage. +/// +internal sealed class MSTestTelemetryDataCollector +{ + private readonly Dictionary _attributeCounts = []; + private readonly HashSet _customTestMethodTypes = []; + private readonly HashSet _customTestClassTypes = []; + + private static MSTestTelemetryDataCollector? s_current; + + /// + /// Gets or sets the current telemetry data collector for the session. + /// Set at session start, cleared at session close. + /// + internal static MSTestTelemetryDataCollector? Current + { + get => Volatile.Read(ref s_current); + set => Volatile.Write(ref s_current, value); + } + + /// + /// Gets a value indicating whether any data has been collected. + /// + internal bool HasData { get; private set; } + + /// + /// Gets or sets the configuration source used for this session. + /// + internal string? ConfigurationSource { get; set; } + + /// + /// Records the attributes found on a test method during discovery. + /// + /// The cached attributes from the method. + internal void TrackDiscoveredMethod(Attribute[] attributes) + { + HasData = true; + + foreach (Attribute attribute in attributes) + { + Type attributeType = attribute.GetType(); + string attributeName = attributeType.Name; + + // Track custom/inherited TestMethodAttribute types (store anonymized hash) + if (attribute is TestMethodAttribute && attributeType != typeof(TestMethodAttribute)) + { + _customTestMethodTypes.Add(AnonymizeString(attributeType.FullName ?? attributeName)); + } + + // Track custom/inherited TestClassAttribute types (store anonymized hash) + if (attribute is TestClassAttribute && attributeType != typeof(TestClassAttribute)) + { + _customTestClassTypes.Add(AnonymizeString(attributeType.FullName ?? attributeName)); + } + + // Track attribute usage counts by base type name + string trackingName = attribute switch + { + TestMethodAttribute => nameof(TestMethodAttribute), + TestClassAttribute => nameof(TestClassAttribute), + DataRowAttribute => nameof(DataRowAttribute), + DynamicDataAttribute => nameof(DynamicDataAttribute), + TimeoutAttribute => nameof(TimeoutAttribute), + IgnoreAttribute => nameof(IgnoreAttribute), + DoNotParallelizeAttribute => nameof(DoNotParallelizeAttribute), + RetryBaseAttribute => nameof(RetryBaseAttribute), + ConditionBaseAttribute => nameof(ConditionBaseAttribute), + TestCategoryAttribute => nameof(TestCategoryAttribute), + DeploymentItemAttribute => nameof(DeploymentItemAttribute), + _ => attributeName, + }; + + _attributeCounts[trackingName] = _attributeCounts.TryGetValue(trackingName, out long count) + ? count + 1 + : 1; + } + } + + /// + /// Records the attributes found on a test class during discovery. + /// + /// The type of the test class. + /// The cached attributes from the class. + internal void TrackDiscoveredClass(Type classType, Attribute[] attributes) + { + HasData = true; + + foreach (Attribute attribute in attributes) + { + Type attributeType = attribute.GetType(); + + // Track custom/inherited TestClassAttribute types (store anonymized hash) + if (attribute is TestClassAttribute && attributeType != typeof(TestClassAttribute)) + { + _customTestClassTypes.Add(AnonymizeString(attributeType.FullName ?? attributeType.Name)); + } + + string trackingName = attribute switch + { + TestClassAttribute => nameof(TestClassAttribute), + ParallelizeAttribute => nameof(ParallelizeAttribute), + DoNotParallelizeAttribute => nameof(DoNotParallelizeAttribute), + _ => null!, + }; + + if (trackingName is not null) + { + _attributeCounts[trackingName] = _attributeCounts.TryGetValue(trackingName, out long count) + ? count + 1 + : 1; + } + } + } + + /// + /// Builds the telemetry metrics dictionary for sending via the telemetry collector. + /// + /// A dictionary of telemetry key-value pairs. + internal Dictionary BuildMetrics() + { + Dictionary metrics = []; + + // Settings + AddSettingsMetrics(metrics); + + // Configuration source (runsettings, testconfig.json, or none) + if (ConfigurationSource is not null) + { + metrics["mstest.config_source"] = ConfigurationSource; + } + + // Attribute usage (aggregated counts as JSON) + if (_attributeCounts.Count > 0) + { + metrics["mstest.attribute_usage"] = JsonSerializer.Serialize(_attributeCounts, MSTestTelemetryJsonContext.Default.DictionaryStringInt64); + } + + // Custom/inherited types (anonymized names) + if (_customTestMethodTypes.Count > 0) + { + metrics["mstest.custom_test_method_types"] = JsonSerializer.Serialize(_customTestMethodTypes, MSTestTelemetryJsonContext.Default.HashSetString); + } + + if (_customTestClassTypes.Count > 0) + { + metrics["mstest.custom_test_class_types"] = JsonSerializer.Serialize(_customTestClassTypes, MSTestTelemetryJsonContext.Default.HashSetString); + } + + // Assertion usage (drain the static counters) + Dictionary assertionCounts = TelemetryCollector.DrainAssertionCallCounts(); + if (assertionCounts.Count > 0) + { + metrics["mstest.assertion_usage"] = JsonSerializer.Serialize(assertionCounts, MSTestTelemetryJsonContext.Default.DictionaryStringInt64); + } + + return metrics; + } + + private static void AddSettingsMetrics(Dictionary metrics) + { + MSTestSettings settings = MSTestSettings.CurrentSettings; + + // Parallelization + metrics["mstest.setting.parallelization_enabled"] = !settings.DisableParallelization; + if (settings.ParallelizationScope is not null) + { + metrics["mstest.setting.parallelization_scope"] = settings.ParallelizationScope.Value.ToString(); + } + + if (settings.ParallelizationWorkers is not null) + { + metrics["mstest.setting.parallelization_workers"] = settings.ParallelizationWorkers.Value; + } + + // Timeouts + metrics["mstest.setting.test_timeout"] = settings.TestTimeout; + metrics["mstest.setting.assembly_initialize_timeout"] = settings.AssemblyInitializeTimeout; + metrics["mstest.setting.assembly_cleanup_timeout"] = settings.AssemblyCleanupTimeout; + metrics["mstest.setting.class_initialize_timeout"] = settings.ClassInitializeTimeout; + metrics["mstest.setting.class_cleanup_timeout"] = settings.ClassCleanupTimeout; + metrics["mstest.setting.test_initialize_timeout"] = settings.TestInitializeTimeout; + metrics["mstest.setting.test_cleanup_timeout"] = settings.TestCleanupTimeout; + metrics["mstest.setting.cooperative_cancellation"] = settings.CooperativeCancellationTimeout; + + // Behavior + metrics["mstest.setting.map_inconclusive_to_failed"] = settings.MapInconclusiveToFailed; + metrics["mstest.setting.map_not_runnable_to_failed"] = settings.MapNotRunnableToFailed; + metrics["mstest.setting.treat_discovery_warnings_as_errors"] = settings.TreatDiscoveryWarningsAsErrors; + metrics["mstest.setting.consider_empty_data_source_as_inconclusive"] = settings.ConsiderEmptyDataSourceAsInconclusive; + metrics["mstest.setting.order_tests_by_name"] = settings.OrderTestsByNameInClass; + metrics["mstest.setting.capture_debug_traces"] = settings.CaptureDebugTraces; + metrics["mstest.setting.has_test_settings_file"] = settings.TestSettingsFile is not null; + } + + private static string AnonymizeString(string value) + { +#if NET + byte[] hash = SHA256.HashData(Encoding.UTF8.GetBytes(value)); +#else + using SHA256 sha256 = SHA256.Create(); + byte[] hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(value)); +#endif + + return Convert.ToHexString(hash); + } + + /// + /// Sends collected telemetry via the provided sender delegate and resets the current collector. + /// Safe to call even when no sender is available (no-op). + /// + /// Optional delegate to send telemetry. If null, telemetry is silently discarded. + internal static async Task SendTelemetryAndResetAsync(Func, Task>? telemetrySender) + { + try + { + MSTestTelemetryDataCollector? collector = Current; + if (collector is not { HasData: true } || telemetrySender is null) + { + return; + } + + Dictionary metrics = collector.BuildMetrics(); + if (metrics.Count > 0) + { + await telemetrySender("dotnet/testingplatform/mstest/sessionexit", metrics).ConfigureAwait(false); + } + } + catch (Exception) + { + // Telemetry should never cause test failures + } + finally + { + Current = null; + } + } + + /// + /// Synchronous version of for call sites that cannot use async. + /// Resets the current collector regardless of whether telemetry was sent. + /// + /// Optional delegate to send telemetry. If null, telemetry is silently discarded. + internal static void SendTelemetryAndReset(Func, Task>? telemetrySender) + { + try + { + MSTestTelemetryDataCollector? collector = Current; + if (collector is not { HasData: true } || telemetrySender is null) + { + return; + } + + Dictionary metrics = collector.BuildMetrics(); + if (metrics.Count > 0) + { + // Use Task.Run to avoid capturing any SynchronizationContext that could cause deadlocks + Task.Run(() => telemetrySender("dotnet/testingplatform/mstest/sessionexit", metrics)).GetAwaiter().GetResult(); + } + } + catch (Exception) + { + // Telemetry should never cause test failures + } + finally + { + Current = null; + } + } +} + +[JsonSerializable(typeof(Dictionary))] +[JsonSerializable(typeof(HashSet))] +internal sealed partial class MSTestTelemetryJsonContext : JsonSerializerContext; +#endif diff --git a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj index 0c69386a00..869e113dcf 100644 --- a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj +++ b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj @@ -59,6 +59,7 @@ This package provides the core platform and the .NET implementation of the proto + diff --git a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs index 5cd7fe9939..87787bab99 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.AreEqual.cs @@ -483,6 +483,8 @@ public static void AreEqual(T? expected, T? actual, IEqualityComparer? com /// public static void AreEqual(T? expected, T? actual, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (!AreEqualFailing(expected, actual, comparer)) { return; @@ -786,6 +788,8 @@ public static void AreNotEqual(T? notExpected, T? actual, IEqualityComparer public static void AreNotEqual(T? notExpected, T? actual, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (!AreNotEqualFailing(notExpected, actual, comparer)) { return; @@ -835,6 +839,8 @@ public static void AreEqual(float expected, float actual, float delta, [Interpol /// public static void AreEqual(float expected, float actual, float delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -882,6 +888,8 @@ public static void AreNotEqual(float notExpected, float actual, float delta, [In /// public static void AreNotEqual(float notExpected, float actual, float delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); @@ -950,6 +958,8 @@ public static void AreEqual(decimal expected, decimal actual, decimal delta, [In /// public static void AreEqual(decimal expected, decimal actual, decimal delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -997,6 +1007,8 @@ public static void AreNotEqual(decimal notExpected, decimal actual, decimal delt /// public static void AreNotEqual(decimal notExpected, decimal actual, decimal delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); @@ -1047,6 +1059,8 @@ public static void AreEqual(long expected, long actual, long delta, [Interpolate /// public static void AreEqual(long expected, long actual, long delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -1094,6 +1108,8 @@ public static void AreNotEqual(long notExpected, long actual, long delta, [Inter /// public static void AreNotEqual(long notExpected, long actual, long delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); @@ -1143,6 +1159,8 @@ public static void AreEqual(double expected, double actual, double delta, [Inter /// public static void AreEqual(double expected, double actual, double delta, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + if (AreEqualFailing(expected, actual, delta)) { string userMessage = BuildUserMessageForExpectedExpressionAndActualExpression(message, expectedExpression, actualExpression); @@ -1190,6 +1208,8 @@ public static void AreNotEqual(double notExpected, double actual, double delta, /// public static void AreNotEqual(double notExpected, double actual, double delta, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + if (AreNotEqualFailing(notExpected, actual, delta)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression); @@ -1316,6 +1336,8 @@ public static void AreEqual(string? expected, string? actual, bool ignoreCase, /// public static void AreEqual(string? expected, string? actual, bool ignoreCase, CultureInfo culture, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreEqual"); + CheckParameterNotNull(culture, "Assert.AreEqual", "culture"); if (!AreEqualFailing(expected, actual, ignoreCase, culture)) { @@ -1412,6 +1434,8 @@ public static void AreNotEqual(string? notExpected, string? actual, bool ignoreC /// public static void AreNotEqual(string? notExpected, string? actual, bool ignoreCase, CultureInfo culture, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotEqual"); + CheckParameterNotNull(culture, "Assert.AreNotEqual", "culture"); if (!AreNotEqualFailing(notExpected, actual, ignoreCase, culture)) { diff --git a/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs b/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs index 93be91c7fd..581c22d2e1 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.AreSame.cs @@ -172,6 +172,8 @@ public static void AreSame(T? expected, T? actual, [InterpolatedStringHandler /// public static void AreSame(T? expected, T? actual, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreSame"); + if (!IsAreSameFailing(expected, actual)) { return; @@ -238,6 +240,8 @@ public static void AreNotSame(T? notExpected, T? actual, [InterpolatedStringH /// public static void AreNotSame(T? notExpected, T? actual, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(actual))] string actualExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.AreNotSame"); + if (IsAreNotSameFailing(notExpected, actual)) { ThrowAssertAreNotSameFailed(BuildUserMessageForNotExpectedExpressionAndActualExpression(message, notExpectedExpression, actualExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs b/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs index 19fb68e0a6..01b90d7b93 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Contains.cs @@ -154,6 +154,8 @@ public static T ContainsSingle(IEnumerable collection, string? message = " /// The item that matches the predicate. public static T ContainsSingle(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.ContainsSingle"); + T firstMatch = default!; int matchCount = 0; @@ -209,6 +211,8 @@ public static T ContainsSingle(Func predicate, IEnumerable collec /// The item that matches the predicate. public static object? ContainsSingle(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.ContainsSingle"); + object? firstMatch = null; int matchCount = 0; @@ -273,6 +277,8 @@ public static T ContainsSingle(Func predicate, IEnumerable collec /// public static void Contains(T expected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + if (!collection.Contains(expected)) { string userMessage = BuildUserMessageForExpectedExpressionAndCollectionExpression(message, expectedExpression, collectionExpression); @@ -296,6 +302,8 @@ public static void Contains(T expected, IEnumerable collection, string? me /// public static void Contains(object? expected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(collection, "Assert.Contains", "collection"); foreach (object? item in collection) @@ -328,6 +336,8 @@ public static void Contains(object? expected, IEnumerable collection, string? me /// public static void Contains(T expected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + if (!collection.Contains(expected, comparer)) { string userMessage = BuildUserMessageForExpectedExpressionAndCollectionExpression(message, expectedExpression, collectionExpression); @@ -352,6 +362,8 @@ public static void Contains(T expected, IEnumerable collection, IEqualityC /// public static void Contains(object? expected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(expected))] string expectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(collection, "Assert.Contains", "collection"); CheckParameterNotNull(comparer, "Assert.Contains", "comparer"); @@ -384,6 +396,8 @@ public static void Contains(object? expected, IEnumerable collection, IEqualityC /// public static void Contains(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + if (!collection.Any(predicate)) { string userMessage = BuildUserMessageForPredicateExpressionAndCollectionExpression(message, predicateExpression, collectionExpression); @@ -407,6 +421,8 @@ public static void Contains(Func predicate, IEnumerable collectio /// public static void Contains(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(collection, "Assert.Contains", "collection"); CheckParameterNotNull(predicate, "Assert.Contains", "predicate"); @@ -486,6 +502,8 @@ public static void Contains(string substring, string value, string? message = "" /// public static void Contains(string substring, string value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(substring))] string substringExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.Contains"); + CheckParameterNotNull(value, "Assert.Contains", "value"); CheckParameterNotNull(substring, "Assert.Contains", "substring"); @@ -518,6 +536,8 @@ public static void Contains(string substring, string value, StringComparison com /// public static void DoesNotContain(T notExpected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + if (collection.Contains(notExpected)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndCollectionExpression(message, notExpectedExpression, collectionExpression); @@ -541,6 +561,8 @@ public static void DoesNotContain(T notExpected, IEnumerable collection, s /// public static void DoesNotContain(object? notExpected, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(collection, "Assert.DoesNotContain", "collection"); foreach (object? item in collection) @@ -571,6 +593,8 @@ public static void DoesNotContain(object? notExpected, IEnumerable collection, s /// public static void DoesNotContain(T notExpected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + if (collection.Contains(notExpected, comparer)) { string userMessage = BuildUserMessageForNotExpectedExpressionAndCollectionExpression(message, notExpectedExpression, collectionExpression); @@ -595,6 +619,8 @@ public static void DoesNotContain(T notExpected, IEnumerable collection, I /// public static void DoesNotContain(object? notExpected, IEnumerable collection, IEqualityComparer comparer, string? message = "", [CallerArgumentExpression(nameof(notExpected))] string notExpectedExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(collection, "Assert.DoesNotContain", "collection"); CheckParameterNotNull(comparer, "Assert.DoesNotContain", "comparer"); @@ -625,6 +651,8 @@ public static void DoesNotContain(object? notExpected, IEnumerable collection, I /// public static void DoesNotContain(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + if (collection.Any(predicate)) { string userMessage = BuildUserMessageForPredicateExpressionAndCollectionExpression(message, predicateExpression, collectionExpression); @@ -648,6 +676,8 @@ public static void DoesNotContain(Func predicate, IEnumerable col /// public static void DoesNotContain(Func predicate, IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(predicate))] string predicateExpression = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(collection, "Assert.DoesNotContain", "collection"); CheckParameterNotNull(predicate, "Assert.DoesNotContain", "predicate"); @@ -725,6 +755,8 @@ public static void DoesNotContain(string substring, string value, string? messag /// public static void DoesNotContain(string substring, string value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(substring))] string substringExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotContain"); + CheckParameterNotNull(value, "Assert.DoesNotContain", "value"); CheckParameterNotNull(substring, "Assert.DoesNotContain", "substring"); @@ -764,6 +796,8 @@ public static void DoesNotContain(string substring, string value, StringComparis public static void IsInRange(T minValue, T maxValue, T value, string? message = "", [CallerArgumentExpression(nameof(minValue))] string minValueExpression = "", [CallerArgumentExpression(nameof(maxValue))] string maxValueExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : struct, IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsInRange"); + if (maxValue.CompareTo(minValue) < 0) { throw new ArgumentOutOfRangeException(nameof(maxValue), FrameworkMessages.IsInRangeMaxValueMustBeGreaterThanOrEqualMinValue); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Count.cs b/src/TestFramework/TestFramework/Assertions/Assert.Count.cs index 555d341686..c1f3a4e59c 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Count.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Count.cs @@ -197,6 +197,8 @@ public static void IsNotEmpty(IEnumerable collection, [InterpolatedStringH /// public static void IsNotEmpty(IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotEmpty"); + if (collection.Any()) { return; @@ -217,6 +219,8 @@ public static void IsNotEmpty(IEnumerable collection, string? message = "" /// public static void IsNotEmpty(IEnumerable collection, string? message = "", [CallerArgumentExpression(nameof(collection))] string collectionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotEmpty"); + if (collection.Cast().Any()) { return; @@ -320,6 +324,8 @@ public static void IsEmpty(IEnumerable collection, string? message = "", [Caller private static void HasCount(string assertionName, int expected, IEnumerable collection, string? message, string collectionExpression) { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertionName)); + int actualCount = collection.Count(); if (actualCount == expected) { diff --git a/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs b/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs index 772c1d1e48..48485bc71f 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.EndsWith.cs @@ -72,6 +72,8 @@ public static void EndsWith([NotNull] string? expectedSuffix, [NotNull] string? /// public static void EndsWith([NotNull] string? expectedSuffix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(expectedSuffix))] string expectedSuffixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.EndsWith"); + CheckParameterNotNull(value, "Assert.EndsWith", "value"); CheckParameterNotNull(expectedSuffix, "Assert.EndsWith", "expectedSuffix"); if (!value.EndsWith(expectedSuffix, comparisonType)) @@ -146,6 +148,8 @@ public static void DoesNotEndWith([NotNull] string? notExpectedSuffix, [NotNull] /// public static void DoesNotEndWith([NotNull] string? notExpectedSuffix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(notExpectedSuffix))] string notExpectedSuffixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotEndWith"); + CheckParameterNotNull(value, "Assert.DoesNotEndWith", "value"); CheckParameterNotNull(notExpectedSuffix, "Assert.DoesNotEndWith", "notExpectedSuffix"); if (value.EndsWith(notExpectedSuffix, comparisonType)) diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs b/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs index 3a3a4ebd24..826a93610b 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Fail.cs @@ -22,5 +22,8 @@ public sealed partial class Assert /// [DoesNotReturn] public static void Fail(string message = "") - => ThrowAssertFailed("Assert.Fail", BuildUserMessage(message)); + { + TelemetryCollector.TrackAssertionCall("Assert.Fail"); + ThrowAssertFailed("Assert.Fail", BuildUserMessage(message)); + } } diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs b/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs index a953f5cd33..fa75a25c2e 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IComparable.cs @@ -44,6 +44,8 @@ public sealed partial class Assert public static void IsGreaterThan(T lowerBound, T value, string? message = "", [CallerArgumentExpression(nameof(lowerBound))] string lowerBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsGreaterThan"); + if (value.CompareTo(lowerBound) > 0) { return; @@ -89,6 +91,8 @@ public static void IsGreaterThan(T lowerBound, T value, string? message = "", public static void IsGreaterThanOrEqualTo(T lowerBound, T value, string? message = "", [CallerArgumentExpression(nameof(lowerBound))] string lowerBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsGreaterThanOrEqualTo"); + if (value.CompareTo(lowerBound) >= 0) { return; @@ -134,6 +138,8 @@ public static void IsGreaterThanOrEqualTo(T lowerBound, T value, string? mess public static void IsLessThan(T upperBound, T value, string? message = "", [CallerArgumentExpression(nameof(upperBound))] string upperBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsLessThan"); + if (value.CompareTo(upperBound) < 0) { return; @@ -179,6 +185,8 @@ public static void IsLessThan(T upperBound, T value, string? message = "", [C public static void IsLessThanOrEqualTo(T upperBound, T value, string? message = "", [CallerArgumentExpression(nameof(upperBound))] string upperBoundExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsLessThanOrEqualTo"); + if (value.CompareTo(upperBound) <= 0) { return; @@ -216,6 +224,8 @@ public static void IsLessThanOrEqualTo(T upperBound, T value, string? message public static void IsPositive(T value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : struct, IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsPositive"); + var zero = default(T); // Handle special case for floating point NaN values @@ -270,6 +280,8 @@ public static void IsPositive(T value, string? message = "", [CallerArgumentE public static void IsNegative(T value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") where T : struct, IComparable { + TelemetryCollector.TrackAssertionCall("Assert.IsNegative"); + var zero = default(T); // Handle special case for floating point NaN values diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs b/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs index cca665dbb8..8948ecb71c 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Inconclusive.cs @@ -23,6 +23,8 @@ public sealed partial class Assert [DoesNotReturn] public static void Inconclusive(string message = "") { + TelemetryCollector.TrackAssertionCall("Assert.Inconclusive"); + string userMessage = BuildUserMessage(message); throw new AssertInconclusiveException( string.Format(CultureInfo.CurrentCulture, FrameworkMessages.AssertionFailed, "Assert.Inconclusive", userMessage)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs index 144f17cdbe..8ff9c51079 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsExactInstanceOfType.cs @@ -289,6 +289,8 @@ internal void ComputeAssertion(string valueExpression) /// public static void IsExactInstanceOfType([NotNull] object? value, [NotNull] Type? expectedType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsExactInstanceOfType"); + if (IsExactInstanceOfTypeFailing(value, expectedType)) { ThrowAssertIsExactInstanceOfTypeFailed(value, expectedType, BuildUserMessageForValueExpression(message, valueExpression)); @@ -371,6 +373,8 @@ private static void ThrowAssertIsExactInstanceOfTypeFailed(object? value, Type? /// public static void IsNotExactInstanceOfType(object? value, [NotNull] Type? wrongType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotExactInstanceOfType"); + if (IsNotExactInstanceOfTypeFailing(value, wrongType)) { ThrowAssertIsNotExactInstanceOfTypeFailed(value, wrongType, BuildUserMessageForValueExpression(message, valueExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs index bdd68f1607..7abdf6da8e 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsInstanceOfType.cs @@ -290,6 +290,8 @@ internal void ComputeAssertion(string valueExpression) /// public static void IsInstanceOfType([NotNull] object? value, [NotNull] Type? expectedType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsInstanceOfType"); + if (IsInstanceOfTypeFailing(value, expectedType)) { ThrowAssertIsInstanceOfTypeFailed(value, expectedType, BuildUserMessageForValueExpression(message, valueExpression)); @@ -374,6 +376,8 @@ private static void ThrowAssertIsInstanceOfTypeFailed(object? value, Type? expec /// public static void IsNotInstanceOfType(object? value, [NotNull] Type? wrongType, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotInstanceOfType"); + if (IsNotInstanceOfTypeFailing(value, wrongType)) { ThrowAssertIsNotInstanceOfTypeFailed(value, wrongType, BuildUserMessageForValueExpression(message, valueExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs index fd6db5b3d4..a2e5ba7ebf 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsNull.cs @@ -150,6 +150,8 @@ public static void IsNull(object? value, [InterpolatedStringHandlerArgument(name /// public static void IsNull(object? value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNull"); + if (IsNullFailing(value)) { ThrowAssertIsNullFailed(BuildUserMessageForValueExpression(message, valueExpression)); @@ -189,6 +191,8 @@ public static void IsNotNull([NotNull] object? value, [InterpolatedStringHandler /// public static void IsNotNull([NotNull] object? value, string? message = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsNotNull"); + if (IsNotNullFailing(value)) { ThrowAssertIsNotNullFailed(BuildUserMessageForValueExpression(message, valueExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs b/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs index 1c31f3add1..76f07b1a72 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.IsTrue.cs @@ -148,6 +148,8 @@ public static void IsTrue([DoesNotReturnIf(false)] bool? condition, [Interpolate /// public static void IsTrue([DoesNotReturnIf(false)] bool? condition, string? message = "", [CallerArgumentExpression(nameof(condition))] string conditionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsTrue"); + if (IsTrueFailing(condition)) { ThrowAssertIsTrueFailed(BuildUserMessageForConditionExpression(message, conditionExpression)); @@ -186,6 +188,8 @@ public static void IsFalse([DoesNotReturnIf(true)] bool? condition, [Interpolate /// public static void IsFalse([DoesNotReturnIf(true)] bool? condition, string? message = "", [CallerArgumentExpression(nameof(condition))] string conditionExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.IsFalse"); + if (IsFalseFailing(condition)) { ThrowAssertIsFalseFailed(BuildUserMessageForConditionExpression(message, conditionExpression)); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs b/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs index b204970b6d..28e5ba1ee0 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.Matches.cs @@ -39,6 +39,8 @@ public sealed partial class Assert /// public static void MatchesRegex([NotNull] Regex? pattern, [NotNull] string? value, string? message = "", [CallerArgumentExpression(nameof(pattern))] string patternExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.MatchesRegex"); + CheckParameterNotNull(value, "Assert.MatchesRegex", "value"); CheckParameterNotNull(pattern, "Assert.MatchesRegex", "pattern"); @@ -115,6 +117,8 @@ public static void MatchesRegex([NotNull] string? pattern, [NotNull] string? val /// public static void DoesNotMatchRegex([NotNull] Regex? pattern, [NotNull] string? value, string? message = "", [CallerArgumentExpression(nameof(pattern))] string patternExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotMatchRegex"); + CheckParameterNotNull(value, "Assert.DoesNotMatchRegex", "value"); CheckParameterNotNull(pattern, "Assert.DoesNotMatchRegex", "pattern"); diff --git a/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs b/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs index 6f5c728535..5e5e55837c 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.StartsWith.cs @@ -72,6 +72,8 @@ public static void StartsWith([NotNull] string? expectedPrefix, [NotNull] string /// public static void StartsWith([NotNull] string? expectedPrefix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(expectedPrefix))] string expectedPrefixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.StartsWith"); + CheckParameterNotNull(value, "Assert.StartsWith", "value"); CheckParameterNotNull(expectedPrefix, "Assert.StartsWith", "expectedPrefix"); if (!value.StartsWith(expectedPrefix, comparisonType)) @@ -144,6 +146,8 @@ public static void DoesNotStartWith([NotNull] string? notExpectedPrefix, [NotNul /// public static void DoesNotStartWith([NotNull] string? notExpectedPrefix, [NotNull] string? value, StringComparison comparisonType, string? message = "", [CallerArgumentExpression(nameof(notExpectedPrefix))] string notExpectedPrefixExpression = "", [CallerArgumentExpression(nameof(value))] string valueExpression = "") { + TelemetryCollector.TrackAssertionCall("Assert.DoesNotStartWith"); + CheckParameterNotNull(value, "Assert.DoesNotStartWith", "value"); CheckParameterNotNull(notExpectedPrefix, "Assert.DoesNotStartWith", "notExpectedPrefix"); if (value.StartsWith(notExpectedPrefix, comparisonType)) diff --git a/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs b/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs index b5478a28a6..83e9960958 100644 --- a/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs +++ b/src/TestFramework/TestFramework/Assertions/Assert.ThrowsException.cs @@ -321,6 +321,8 @@ public static TException ThrowsExactly(Func action, [Interp private static TException ThrowsException(Action action, bool isStrictType, string? message, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertMethodName)); + Ensure.NotNull(action); Ensure.NotNull(message); @@ -341,6 +343,8 @@ private static TException ThrowsException(Action action, bool isStri private static TException ThrowsException(Action action, bool isStrictType, Func messageBuilder, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertMethodName)); + Ensure.NotNull(action); Ensure.NotNull(messageBuilder); @@ -477,6 +481,8 @@ public static Task ThrowsExactlyAsync(Func action, private static async Task ThrowsExceptionAsync(Func action, bool isStrictType, string? message, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertMethodName)); + Ensure.NotNull(action); Ensure.NotNull(message); @@ -497,6 +503,8 @@ private static async Task ThrowsExceptionAsync(Func ThrowsExceptionAsync(Func action, bool isStrictType, Func messageBuilder, string actionExpression, [CallerMemberName] string assertMethodName = "") where TException : Exception { + TelemetryCollector.TrackAssertionCall(string.Concat("Assert.", assertMethodName)); + Ensure.NotNull(action); Ensure.NotNull(messageBuilder); diff --git a/src/TestFramework/TestFramework/Assertions/CollectionAssert.cs b/src/TestFramework/TestFramework/Assertions/CollectionAssert.cs index c8f37e098b..8fa5356cf7 100644 --- a/src/TestFramework/TestFramework/Assertions/CollectionAssert.cs +++ b/src/TestFramework/TestFramework/Assertions/CollectionAssert.cs @@ -69,6 +69,8 @@ public static void Contains([NotNull] ICollection? collection, object? element) /// public static void Contains([NotNull] ICollection? collection, object? element, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.Contains"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.Contains", "collection"); foreach (object? current in collection) @@ -120,6 +122,8 @@ public static void DoesNotContain([NotNull] ICollection? collection, object? ele /// public static void DoesNotContain([NotNull] ICollection? collection, object? element, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.DoesNotContain"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.DoesNotContain", "collection"); foreach (object? current in collection) @@ -160,6 +164,8 @@ public static void AllItemsAreNotNull([NotNull] ICollection? collection) /// public static void AllItemsAreNotNull([NotNull] ICollection? collection, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AllItemsAreNotNull"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.AllItemsAreNotNull", "collection"); foreach (object? current in collection) { @@ -202,6 +208,8 @@ public static void AllItemsAreUnique([NotNull] ICollection? collection) /// public static void AllItemsAreUnique([NotNull] ICollection? collection, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AllItemsAreUnique"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.AllItemsAreUnique", "collection"); message = Assert.ReplaceNulls(message); @@ -292,6 +300,8 @@ public static void IsSubsetOf([NotNull] ICollection? subset, [NotNull] ICollecti /// public static void IsSubsetOf([NotNull] ICollection? subset, [NotNull] ICollection? superset, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.IsSubsetOf"); + Assert.CheckParameterNotNull(subset, "CollectionAssert.IsSubsetOf", "subset"); Assert.CheckParameterNotNull(superset, "CollectionAssert.IsSubsetOf", "superset"); Tuple> isSubsetValue = IsSubsetOfHelper(subset, superset); @@ -352,6 +362,8 @@ public static void IsNotSubsetOf([NotNull] ICollection? subset, [NotNull] IColle /// public static void IsNotSubsetOf([NotNull] ICollection? subset, [NotNull] ICollection? superset, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.IsNotSubsetOf"); + Assert.CheckParameterNotNull(subset, "CollectionAssert.IsNotSubsetOf", "subset"); Assert.CheckParameterNotNull(superset, "CollectionAssert.IsNotSubsetOf", "superset"); Tuple> isSubsetValue = IsSubsetOfHelper(subset, superset); @@ -471,6 +483,8 @@ public static void AreEquivalent( [NotNullIfNotNull(nameof(actual))] IEnumerable? expected, [NotNullIfNotNull(nameof(expected))] IEnumerable? actual, [NotNull] IEqualityComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreEquivalent"); + Assert.CheckParameterNotNull(comparer, "Assert.AreCollectionsEqual", "comparer"); // Check whether one is null while the other is not. @@ -637,6 +651,8 @@ public static void AreNotEquivalent( [NotNullIfNotNull(nameof(actual))] IEnumerable? notExpected, [NotNullIfNotNull(nameof(notExpected))] IEnumerable? actual, [NotNull] IEqualityComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreNotEquivalent"); + Assert.CheckParameterNotNull(comparer, "Assert.AreCollectionsEqual", "comparer"); // Check whether one is null while the other is not. @@ -740,6 +756,8 @@ public static void AllItemsAreInstancesOfType([NotNull] ICollection? collection, public static void AllItemsAreInstancesOfType( [NotNull] ICollection? collection, [NotNull] Type? expectedType, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AllItemsAreInstancesOfType"); + Assert.CheckParameterNotNull(collection, "CollectionAssert.AllItemsAreInstancesOfType", "collection"); Assert.CheckParameterNotNull(expectedType, "CollectionAssert.AllItemsAreInstancesOfType", "expectedType"); int i = 0; @@ -813,6 +831,8 @@ public static void AreEqual(ICollection? expected, ICollection? actual) /// public static void AreEqual(ICollection? expected, ICollection? actual, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreEqual"); + string reason = string.Empty; if (!AreCollectionsEqual(expected, actual, new ObjectComparer(), ref reason)) { @@ -867,6 +887,8 @@ public static void AreNotEqual(ICollection? notExpected, ICollection? actual) /// public static void AreNotEqual(ICollection? notExpected, ICollection? actual, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreNotEqual"); + string reason = string.Empty; if (AreCollectionsEqual(notExpected, actual, new ObjectComparer(), ref reason)) { @@ -925,6 +947,8 @@ public static void AreEqual(ICollection? expected, ICollection? actual, [NotNull /// public static void AreEqual(ICollection? expected, ICollection? actual, [NotNull] IComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreEqual"); + string reason = string.Empty; if (!AreCollectionsEqual(expected, actual, comparer, ref reason)) { @@ -983,6 +1007,8 @@ public static void AreNotEqual(ICollection? notExpected, ICollection? actual, [N /// public static void AreNotEqual(ICollection? notExpected, ICollection? actual, [NotNull] IComparer? comparer, string? message) { + TelemetryCollector.TrackAssertionCall("CollectionAssert.AreNotEqual"); + string reason = string.Empty; if (AreCollectionsEqual(notExpected, actual, comparer, ref reason)) { diff --git a/src/TestFramework/TestFramework/Assertions/StringAssert.cs b/src/TestFramework/TestFramework/Assertions/StringAssert.cs index b2e36625ec..41007ada49 100644 --- a/src/TestFramework/TestFramework/Assertions/StringAssert.cs +++ b/src/TestFramework/TestFramework/Assertions/StringAssert.cs @@ -116,6 +116,8 @@ public static void Contains([NotNull] string? value, [NotNull] string? substring /// public static void Contains([NotNull] string? value, [NotNull] string? substring, StringComparison comparisonType, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.Contains"); + Assert.CheckParameterNotNull(value, "StringAssert.Contains", "value"); Assert.CheckParameterNotNull(substring, "StringAssert.Contains", "substring"); if (value.IndexOf(substring, comparisonType) < 0) @@ -213,6 +215,8 @@ public static void StartsWith([NotNull] string? value, [NotNull] string? substri /// public static void StartsWith([NotNull] string? value, [NotNull] string? substring, StringComparison comparisonType, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.StartsWith"); + Assert.CheckParameterNotNull(value, "StringAssert.StartsWith", "value"); Assert.CheckParameterNotNull(substring, "StringAssert.StartsWith", "substring"); if (!value.StartsWith(substring, comparisonType)) @@ -310,6 +314,8 @@ public static void EndsWith([NotNull] string? value, [NotNull] string? substring /// public static void EndsWith([NotNull] string? value, [NotNull] string? substring, StringComparison comparisonType, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.EndsWith"); + Assert.CheckParameterNotNull(value, "StringAssert.EndsWith", "value"); Assert.CheckParameterNotNull(substring, "StringAssert.EndsWith", "substring"); if (!value.EndsWith(substring, comparisonType)) @@ -364,6 +370,8 @@ public static void Matches([NotNull] string? value, [NotNull] Regex? pattern) /// public static void Matches([NotNull] string? value, [NotNull] Regex? pattern, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.Matches"); + Assert.CheckParameterNotNull(value, "StringAssert.Matches", "value"); Assert.CheckParameterNotNull(pattern, "StringAssert.Matches", "pattern"); @@ -415,6 +423,8 @@ public static void DoesNotMatch([NotNull] string? value, [NotNull] Regex? patter /// public static void DoesNotMatch([NotNull] string? value, [NotNull] Regex? pattern, string? message) { + TelemetryCollector.TrackAssertionCall("StringAssert.DoesNotMatch"); + Assert.CheckParameterNotNull(value, "StringAssert.DoesNotMatch", "value"); Assert.CheckParameterNotNull(pattern, "StringAssert.DoesNotMatch", "pattern"); diff --git a/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs b/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs new file mode 100644 index 0000000000..783815ae91 --- /dev/null +++ b/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.VisualStudio.TestTools.UnitTesting; + +/// +/// Collects aggregated telemetry data about MSTest API usage within a test session. +/// This data is used to understand which APIs are heavily used or unused to guide future investment. +/// +internal static class TelemetryCollector +{ + private static ConcurrentDictionary s_assertionCallCounts = new(); + + /// + /// Records that an assertion method was called. + /// + /// The full name of the assertion (e.g. "Assert.AreEqual", "CollectionAssert.Contains"). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void TrackAssertionCall(string assertionName) + => s_assertionCallCounts.AddOrUpdate(assertionName, 1, static (_, count) => count + 1); + + /// + /// Gets a snapshot of all assertion call counts and resets the counters. + /// This is thread-safe: it atomically swaps the dictionary and drains the old one. + /// + /// A dictionary mapping assertion names to call counts. + internal static Dictionary DrainAssertionCallCounts() + { + ConcurrentDictionary old = Interlocked.Exchange(ref s_assertionCallCounts, new ConcurrentDictionary()); +#pragma warning disable IDE0028 // Simplify collection initialization - ConcurrentDictionary snapshot copy + return new Dictionary(old); +#pragma warning restore IDE0028 // Simplify collection initialization + } +} diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs new file mode 100644 index 0000000000..8533f00c69 --- /dev/null +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs @@ -0,0 +1,294 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Acceptance.IntegrationTests; +using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; +using Microsoft.Testing.Platform.Helpers; + +namespace MSTest.Acceptance.IntegrationTests; + +[TestClass] +public sealed class TelemetryTests : AcceptanceTestBase +{ + private const string MTPAssetName = "TelemetryMTPProject"; + private const string VSTestAssetName = "TelemetryVSTestProject"; + private const string TestResultsFolderName = "TestResults"; + + #region MTP mode - Run + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task MTP_RunTests_SendsTelemetryWithSettingsAndAttributes(string tfm) + { + string diagPath = Path.Combine(AssetFixture.MTPProjectPath, "bin", "Release", tfm, TestResultsFolderName); + string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); + + var testHost = TestHost.LocateFrom(AssetFixture.MTPProjectPath, MTPAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--diagnostic", + disableTelemetry: false, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCodes.Success); + + string diagContentsPattern = +""" +.+ Send telemetry event: dotnet/testingplatform/mstest/sessionexit +.+mstest\.setting\.parallelization_enabled +"""; + string diagFilePath = await AssertDiagnosticReportAsync(testHostResult, diagPathPattern, diagContentsPattern); + + // Verify attribute usage and config source are also present + string content = await File.ReadAllTextAsync(diagFilePath, TestContext.CancellationToken); + Assert.IsTrue(Regex.IsMatch(content, "mstest\\.attribute_usage"), $"Expected attribute_usage in telemetry.\n{content}"); + Assert.IsTrue(Regex.IsMatch(content, "mstest\\.config_source"), $"Expected config_source in telemetry.\n{content}"); + } + + #endregion + + #region MTP mode - Discovery only + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task MTP_DiscoverTests_SendsTelemetryEvent(string tfm) + { + string diagPath = Path.Combine(AssetFixture.MTPProjectPath, "bin", "Release", tfm, TestResultsFolderName); + string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); + + var testHost = TestHost.LocateFrom(AssetFixture.MTPProjectPath, MTPAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--list-tests --diagnostic", + disableTelemetry: false, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCodes.Success); + + string diagContentsPattern = +""" +.+ Send telemetry event: dotnet/testingplatform/mstest/sessionexit +.+mstest\.attribute_usage +"""; + await AssertDiagnosticReportAsync(testHostResult, diagPathPattern, diagContentsPattern); + } + + #endregion + + #region MTP mode - Telemetry disabled + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task MTP_WhenTelemetryDisabled_DoesNotSendMSTestEvent(string tfm) + { + string diagPath = Path.Combine(AssetFixture.MTPProjectPath, "bin", "Release", tfm, TestResultsFolderName); + string diagPathPattern = Path.Combine(diagPath, @"log_.*.diag").Replace(@"\", @"\\"); + + var testHost = TestHost.LocateFrom(AssetFixture.MTPProjectPath, MTPAssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync( + "--diagnostic", + new Dictionary + { + { "TESTINGPLATFORM_TELEMETRY_OPTOUT", "1" }, + }, + disableTelemetry: false, + TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCodes.Success); + + string diagContentsPattern = +""" +.+ Microsoft.Testing.Platform.Telemetry.TelemetryManager DEBUG Telemetry is 'DISABLED' +"""; + string diagFilePath = await AssertDiagnosticReportAsync(testHostResult, diagPathPattern, diagContentsPattern); + + string content = await File.ReadAllTextAsync(diagFilePath, TestContext.CancellationToken); + Assert.IsFalse( + Regex.IsMatch(content, "Send telemetry event: dotnet/testingplatform/mstest/sessionexit"), + "MSTest telemetry event should not be sent when telemetry is disabled."); + } + + #endregion + + #region VSTest mode - Run + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task VSTest_RunTests_Succeeds(string tfm) + { + DotnetMuxerResult testResult = await DotnetCli.RunAsync( + $"test -c Release {AssetFixture.VSTestProjectPath} --no-build --framework {tfm}", + AcceptanceFixture.NuGetGlobalPackagesFolder.Path, + failIfReturnValueIsNotZero: false, + cancellationToken: TestContext.CancellationToken); + + Assert.AreEqual(0, testResult.ExitCode, $"dotnet test failed:\n{testResult.StandardOutput}\n{testResult.StandardError}"); + testResult.AssertOutputContains("Passed!"); + } + + #endregion + + #region VSTest mode - Discovery only + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task VSTest_DiscoverTests_Succeeds(string tfm) + { + DotnetMuxerResult testResult = await DotnetCli.RunAsync( + $"test -c Release {AssetFixture.VSTestProjectPath} --no-build --framework {tfm} --list-tests", + AcceptanceFixture.NuGetGlobalPackagesFolder.Path, + failIfReturnValueIsNotZero: false, + cancellationToken: TestContext.CancellationToken); + + Assert.AreEqual(0, testResult.ExitCode, $"dotnet test --list-tests failed:\n{testResult.StandardOutput}\n{testResult.StandardError}"); + testResult.AssertOutputContains("PassingTest"); + testResult.AssertOutputContains("DataDrivenTest"); + testResult.AssertOutputContains("TestWithTimeout"); + } + + #endregion + + #region Helpers + + private static async Task AssertDiagnosticReportAsync(TestHostResult testHostResult, string diagPathPattern, string diagContentsPattern, string level = "Trace", string flushType = "async") + { + string outputPattern = $""" +Diagnostic file \(level '{level}' with {flushType} flush\): {diagPathPattern} +"""; + testHostResult.AssertOutputMatchesRegex(outputPattern); + Match match = Regex.Match(testHostResult.StandardOutput, diagPathPattern); + Assert.IsTrue(match.Success, $"{testHostResult}\n{diagPathPattern}"); + + (bool success, string content) = await CheckDiagnosticContentsMatchAsync(match.Value, diagContentsPattern); + Assert.IsTrue(success, $"{content}\n{diagContentsPattern}"); + + return match.Value; + } + + private static async Task<(bool IsMatch, string Content)> CheckDiagnosticContentsMatchAsync(string path, string pattern) + { + using var reader = new StreamReader(path); + string content = await reader.ReadToEndAsync(); + + return (Regex.IsMatch(content, pattern), content); + } + + #endregion + + public sealed class TestAssetFixture() : TestAssetFixtureBase(AcceptanceFixture.NuGetGlobalPackagesFolder) + { + private const string MTPProjectId = nameof(TelemetryTests) + "_MTP"; + private const string VSTestProjectId = nameof(TelemetryTests) + "_VSTest"; + + public string MTPProjectPath => GetAssetPath(MTPProjectId); + + public string VSTestProjectPath => GetAssetPath(VSTestProjectId); + + public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate() + { + // MTP mode project (MSTest runner enabled, no global.json override) + yield return (MTPProjectId, MTPAssetName, + MTPSourceCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion)); + + // VSTest mode project (uses global.json to force VSTest runner) + yield return (VSTestProjectId, VSTestAssetName, + VSTestSourceCode + .PatchTargetFrameworks(TargetFrameworks.All) + .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion) + .PatchCodeWithReplace("$MicrosoftNETTestSdkVersion$", MicrosoftNETTestSdkVersion)); + } + + private const string MTPSourceCode = """ +#file TelemetryMTPProject.csproj + + + + Exe + true + $TargetFrameworks$ + Preview + + + + + + + + + +#file UnitTest1.cs +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void PassingTest() + { + } + + [TestMethod] + [DataRow(1)] + [DataRow(2)] + public void DataDrivenTest(int value) + { + Assert.IsTrue(value > 0); + } + + [TestMethod] + [Timeout(30000)] + public void TestWithTimeout() + { + } +} +"""; + + private const string VSTestSourceCode = """ +#file TelemetryVSTestProject.csproj + + + + $TargetFrameworks$ + Preview + false + true + + + + + + + + + + +#file UnitTest1.cs +using Microsoft.VisualStudio.TestTools.UnitTesting; + +[TestClass] +public class UnitTest1 +{ + [TestMethod] + public void PassingTest() + { + } + + [TestMethod] + [DataRow(1)] + [DataRow(2)] + public void DataDrivenTest(int value) + { + Assert.IsTrue(value > 0); + } + + [TestMethod] + [Timeout(30000)] + public void TestWithTimeout() + { + } +} +"""; + } + + public TestContext TestContext { get; set; } +} From b609215e68b6cd35ba436886360446e9973fd480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Tue, 17 Mar 2026 15:42:34 +0100 Subject: [PATCH 2/9] Fix Convert.ToHexString for .NET Framework targets Convert.ToHexString is .NET 5+ only. Use BitConverter.ToString with Replace for the .NET Framework code path, matching the existing #if NET split for SHA256. --- .../Telemetry/MSTestTelemetryDataCollector.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs index 8d6ad68e2e..1f8a4ad7c8 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs @@ -210,12 +210,12 @@ private static string AnonymizeString(string value) { #if NET byte[] hash = SHA256.HashData(Encoding.UTF8.GetBytes(value)); + return Convert.ToHexString(hash); #else using SHA256 sha256 = SHA256.Create(); byte[] hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(value)); + return BitConverter.ToString(hash).Replace("-", string.Empty); #endif - - return Convert.ToHexString(hash); } /// From 3e6f8b52e0b72315804ec555250b2a84d023f9e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Tue, 17 Mar 2026 16:47:18 +0100 Subject: [PATCH 3/9] Suppress IDE0032 for Volatile.Read/Write backing field --- .../Telemetry/MSTestTelemetryDataCollector.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs index 1f8a4ad7c8..1bfeb844bd 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs @@ -20,7 +20,9 @@ internal sealed class MSTestTelemetryDataCollector private readonly HashSet _customTestMethodTypes = []; private readonly HashSet _customTestClassTypes = []; +#pragma warning disable IDE0032 // Use auto property - Volatile.Read/Write requires a ref to a field private static MSTestTelemetryDataCollector? s_current; +#pragma warning restore IDE0032 // Use auto property /// /// Gets or sets the current telemetry data collector for the session. From 6265e4f22332c304a68be4237cf6e9cf1699bd50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Tue, 17 Mar 2026 16:49:48 +0100 Subject: [PATCH 4/9] Move GetAwaiter().GetResult() to MSTestDiscoverer call site Remove the synchronous SendTelemetryAndReset overload from MSTestTelemetryDataCollector. The blocking call now lives in MSTestDiscoverer (the only sync caller), wrapped in Task.Run to avoid SynchronizationContext deadlocks. --- .../VSTestAdapter/MSTestDiscoverer.cs | 3 +- .../Telemetry/MSTestTelemetryDataCollector.cs | 31 ------------------- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs index b34271447f..1362c48d13 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs @@ -63,7 +63,8 @@ internal void DiscoverTests(IEnumerable sources, IDiscoveryContext disco } finally { - MSTestTelemetryDataCollector.SendTelemetryAndReset(_telemetrySender); + // Use Task.Run to avoid capturing any SynchronizationContext that could cause deadlocks + Task.Run(() => MSTestTelemetryDataCollector.SendTelemetryAndResetAsync(_telemetrySender)).GetAwaiter().GetResult(); } } } diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs index 1bfeb844bd..d94e93263f 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs @@ -251,37 +251,6 @@ internal static async Task SendTelemetryAndResetAsync(Func - /// Synchronous version of for call sites that cannot use async. - /// Resets the current collector regardless of whether telemetry was sent. - /// - /// Optional delegate to send telemetry. If null, telemetry is silently discarded. - internal static void SendTelemetryAndReset(Func, Task>? telemetrySender) - { - try - { - MSTestTelemetryDataCollector? collector = Current; - if (collector is not { HasData: true } || telemetrySender is null) - { - return; - } - - Dictionary metrics = collector.BuildMetrics(); - if (metrics.Count > 0) - { - // Use Task.Run to avoid capturing any SynchronizationContext that could cause deadlocks - Task.Run(() => telemetrySender("dotnet/testingplatform/mstest/sessionexit", metrics)).GetAwaiter().GetResult(); - } - } - catch (Exception) - { - // Telemetry should never cause test failures - } - finally - { - Current = null; - } - } } [JsonSerializable(typeof(Dictionary))] From f1c1fa5346bd5b126e92eedad3bbcb73f96cd38d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Tue, 17 Mar 2026 16:56:34 +0100 Subject: [PATCH 5/9] Gate telemetry collection on TESTINGPLATFORM_TELEMETRY_OPTOUT env vars Add IsTelemetryOptedOut() to MSTestTelemetryDataCollector that checks the same environment variables as MTP's TelemetryManager: - TESTINGPLATFORM_TELEMETRY_OPTOUT - DOTNET_CLI_TELEMETRY_OPTOUT When either is '1' or 'true', skip initializing the collector entirely in VSTest mode (MSTestDiscoverer + MSTestExecutor). This avoids unnecessary data collection overhead when telemetry is disabled. In MTP mode, the opt-out is already handled by TelemetryManager which returns null from CreateTelemetrySender() when disabled. --- .../VSTestAdapter/MSTestDiscoverer.cs | 5 ++++- .../VSTestAdapter/MSTestExecutor.cs | 10 ++++++++-- .../Telemetry/MSTestTelemetryDataCollector.cs | 18 ++++++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs index 1362c48d13..9d5b0280e0 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs @@ -52,7 +52,10 @@ internal void DiscoverTests(IEnumerable sources, IDiscoveryContext disco Ensure.NotNull(discoverySink); // Initialize telemetry collection if not already set (e.g. first call in the session) - MSTestTelemetryDataCollector.Current ??= new MSTestTelemetryDataCollector(); + if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) + { + MSTestTelemetryDataCollector.Current ??= new MSTestTelemetryDataCollector(); + } try { diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs index 1b84d52365..7242f5915f 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs @@ -108,7 +108,10 @@ internal async Task RunTestsAsync(IEnumerable? tests, IRunContext? run Ensure.NotNullOrEmpty(tests); // Initialize telemetry collection if not already set - MSTestTelemetryDataCollector.Current ??= new MSTestTelemetryDataCollector(); + if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) + { + MSTestTelemetryDataCollector.Current ??= new MSTestTelemetryDataCollector(); + } if (!MSTestDiscovererHelpers.InitializeDiscovery(from test in tests select test.Source, runContext, frameworkHandle, configuration, new TestSourceHandler())) { @@ -136,7 +139,10 @@ internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? run Ensure.NotNullOrEmpty(sources); // Initialize telemetry collection if not already set - MSTestTelemetryDataCollector.Current ??= new MSTestTelemetryDataCollector(); + if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) + { + MSTestTelemetryDataCollector.Current ??= new MSTestTelemetryDataCollector(); + } TestSourceHandler testSourceHandler = new(); if (!MSTestDiscovererHelpers.InitializeDiscovery(sources, runContext, frameworkHandle, configuration, testSourceHandler)) diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs index d94e93263f..a8ad71a276 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs @@ -39,6 +39,24 @@ internal static MSTestTelemetryDataCollector? Current /// internal bool HasData { get; private set; } + /// + /// Checks whether telemetry collection is opted out via environment variables. + /// Mirrors the same checks as Microsoft.Testing.Platform's TelemetryManager. + /// + /// true if telemetry is opted out; false otherwise. + internal static bool IsTelemetryOptedOut() + { + string? telemetryOptOut = Environment.GetEnvironmentVariable("TESTINGPLATFORM_TELEMETRY_OPTOUT"); + if (telemetryOptOut is "1" or "true") + { + return true; + } + + string? cliTelemetryOptOut = Environment.GetEnvironmentVariable("DOTNET_CLI_TELEMETRY_OPTOUT"); + + return cliTelemetryOptOut is "1" or "true"; + } + /// /// Gets or sets the configuration source used for this session. /// From cbf914012f4eba3e738f5aaca777a03635e75cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Tue, 17 Mar 2026 19:24:44 +0100 Subject: [PATCH 6/9] Fix telemetry lifecycle and handler coverage gaps Address the most important correctness issues in the telemetry branch: - drain assertion counters even when telemetry is not sent, avoiding stale usage leaking into later sessions - make collector initialization atomic with Interlocked.CompareExchange - track Assert.That and interpolated-string-handler assertion paths so modern call sites are no longer systematically undercounted --- .../VSTestAdapter/MSTestDiscoverer.cs | 2 +- .../VSTestAdapter/MSTestExecutor.cs | 4 ++-- .../Telemetry/MSTestTelemetryDataCollector.cs | 15 +++++++++++++++ .../TestFramework/Assertions/Assert.AreEqual.cs | 4 ++++ .../TestFramework/Assertions/Assert.AreSame.cs | 2 ++ .../TestFramework/Assertions/Assert.Count.cs | 2 ++ .../Assertions/Assert.IsExactInstanceOfType.cs | 4 ++++ .../Assertions/Assert.IsInstanceOfType.cs | 4 ++++ .../TestFramework/Assertions/Assert.IsNull.cs | 2 ++ .../TestFramework/Assertions/Assert.IsTrue.cs | 2 ++ .../TestFramework/Assertions/Assert.That.cs | 2 ++ 11 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs index 9d5b0280e0..8be1e84ff9 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs @@ -54,7 +54,7 @@ internal void DiscoverTests(IEnumerable sources, IDiscoveryContext disco // Initialize telemetry collection if not already set (e.g. first call in the session) if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) { - MSTestTelemetryDataCollector.Current ??= new MSTestTelemetryDataCollector(); + _ = MSTestTelemetryDataCollector.EnsureInitialized(); } try diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs index 7242f5915f..9d451ff808 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs @@ -110,7 +110,7 @@ internal async Task RunTestsAsync(IEnumerable? tests, IRunContext? run // Initialize telemetry collection if not already set if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) { - MSTestTelemetryDataCollector.Current ??= new MSTestTelemetryDataCollector(); + _ = MSTestTelemetryDataCollector.EnsureInitialized(); } if (!MSTestDiscovererHelpers.InitializeDiscovery(from test in tests select test.Source, runContext, frameworkHandle, configuration, new TestSourceHandler())) @@ -141,7 +141,7 @@ internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? run // Initialize telemetry collection if not already set if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) { - MSTestTelemetryDataCollector.Current ??= new MSTestTelemetryDataCollector(); + _ = MSTestTelemetryDataCollector.EnsureInitialized(); } TestSourceHandler testSourceHandler = new(); diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs index a8ad71a276..a49a748fdf 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs @@ -34,6 +34,20 @@ internal static MSTestTelemetryDataCollector? Current set => Volatile.Write(ref s_current, value); } + internal static MSTestTelemetryDataCollector EnsureInitialized() + { + MSTestTelemetryDataCollector? collector = Current; + if (collector is not null) + { + return collector; + } + + collector = new MSTestTelemetryDataCollector(); + MSTestTelemetryDataCollector? existingCollector = Interlocked.CompareExchange(ref s_current, collector, null); + + return existingCollector ?? collector; + } + /// /// Gets a value indicating whether any data has been collected. /// @@ -250,6 +264,7 @@ internal static async Task SendTelemetryAndResetAsync(FuncThrown if the evaluated condition is . public static void That(Expression> condition, string? message = null, [CallerArgumentExpression(nameof(condition))] string? conditionExpression = null) { + TelemetryCollector.TrackAssertionCall("Assert.That"); + if (condition == null) { throw new ArgumentNullException(nameof(condition)); From 2a26a8166ceb8b31ddde9bdc4464f89ca33c43a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 18 Mar 2026 10:28:12 +0100 Subject: [PATCH 7/9] Fix TelemetryTests: regex pattern, VSTest global.json, and workingDirectory - Fix MTP_DiscoverTests_SendsTelemetryEvent regex to match across multiple lines between sessionexit event and attribute_usage - Add global.json with VSTest runner to VSTest test asset to opt out of MTP runner enforcement from root global.json - Add workingDirectory to VSTest test methods so dotnet test resolves the local global.json correctly --- .../TelemetryTests.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs index 8533f00c69..1e4dc4dfb4 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs @@ -65,8 +65,7 @@ public async Task MTP_DiscoverTests_SendsTelemetryEvent(string tfm) string diagContentsPattern = """ -.+ Send telemetry event: dotnet/testingplatform/mstest/sessionexit -.+mstest\.attribute_usage +.+ Send telemetry event: dotnet/testingplatform/mstest/sessionexit[\s\S]+?mstest\.attribute_usage """; await AssertDiagnosticReportAsync(testHostResult, diagPathPattern, diagContentsPattern); } @@ -117,6 +116,7 @@ public async Task VSTest_RunTests_Succeeds(string tfm) DotnetMuxerResult testResult = await DotnetCli.RunAsync( $"test -c Release {AssetFixture.VSTestProjectPath} --no-build --framework {tfm}", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, + workingDirectory: AssetFixture.VSTestProjectPath, failIfReturnValueIsNotZero: false, cancellationToken: TestContext.CancellationToken); @@ -135,6 +135,7 @@ public async Task VSTest_DiscoverTests_Succeeds(string tfm) DotnetMuxerResult testResult = await DotnetCli.RunAsync( $"test -c Release {AssetFixture.VSTestProjectPath} --no-build --framework {tfm} --list-tests", AcceptanceFixture.NuGetGlobalPackagesFolder.Path, + workingDirectory: AssetFixture.VSTestProjectPath, failIfReturnValueIsNotZero: false, cancellationToken: TestContext.CancellationToken); @@ -262,6 +263,13 @@ public void TestWithTimeout() +#file global.json +{ + "test": { + "runner": "VSTest" + } +} + #file UnitTest1.cs using Microsoft.VisualStudio.TestTools.UnitTesting; From 6259d0efb7da8b25c72f972ff95d9e98a16596e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 18 Mar 2026 14:33:39 +0100 Subject: [PATCH 8/9] Address PR review feedback - Remove unused classType parameter from TrackDiscoveredClass - Fix null! to string? with null in TrackDiscoveredClass switch - Update DrainAssertionCallCounts doc to note best-effort semantics - Add assertion_usage verification in MTP run telemetry test - Add WIN_UI guards alongside WINDOWS_UWP for telemetry code - Replace System.Text.Json with manual JSON serialization - Fix Encoding.UTF8 to System.Text.Encoding.UTF8 for netstandard2.0 --- .../MSTest.TestAdapter.csproj | 5 + .../MSTestBridgedTestFramework.cs | 1 + .../VSTestAdapter/MSTestDiscoverer.cs | 10 ++ .../VSTestAdapter/MSTestExecutor.cs | 19 ++- .../Discovery/AssemblyEnumerator.cs | 6 +- .../Discovery/TypeEnumerator.cs | 26 +++- .../MSTestSettings.cs | 2 +- .../Telemetry/MSTestTelemetryDataCollector.cs | 121 +++++++++++++++--- .../Internal/TelemetryCollector.cs | 6 +- .../TelemetryTests.cs | 1 + 10 files changed, 169 insertions(+), 28 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj b/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj index b6d063c7d2..d703a6f40c 100644 --- a/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj +++ b/src/Adapter/MSTest.TestAdapter/MSTest.TestAdapter.csproj @@ -51,6 +51,11 @@ $(DefineConstants);WINDOWS_UWP + + + $(DefineConstants);WIN_UI + + diff --git a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs index 5e9c52be45..2cb24b944d 100644 --- a/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs +++ b/src/Adapter/MSTest.TestAdapter/TestingPlatformAdapter/MSTestBridgedTestFramework.cs @@ -12,6 +12,7 @@ using Microsoft.Testing.Platform.Telemetry; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Helpers; using Microsoft.VisualStudio.TestPlatform.ObjectModel; diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs index 8be1e84ff9..d3961654b8 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs @@ -20,7 +20,9 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; internal sealed class MSTestDiscoverer : ITestDiscoverer { private readonly ITestSourceHandler _testSourceHandler; +#if !WINDOWS_UWP && !WIN_UI private readonly Func, Task>? _telemetrySender; +#endif public MSTestDiscoverer() : this(new TestSourceHandler()) @@ -30,7 +32,11 @@ public MSTestDiscoverer() internal /* for testing purposes */ MSTestDiscoverer(ITestSourceHandler testSourceHandler, Func, Task>? telemetrySender = null) { _testSourceHandler = testSourceHandler; +#if !WINDOWS_UWP && !WIN_UI _telemetrySender = telemetrySender; +#else + _ = telemetrySender; +#endif } /// @@ -52,10 +58,12 @@ internal void DiscoverTests(IEnumerable sources, IDiscoveryContext disco Ensure.NotNull(discoverySink); // Initialize telemetry collection if not already set (e.g. first call in the session) +#if !WINDOWS_UWP && !WIN_UI if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) { _ = MSTestTelemetryDataCollector.EnsureInitialized(); } +#endif try { @@ -67,7 +75,9 @@ internal void DiscoverTests(IEnumerable sources, IDiscoveryContext disco finally { // Use Task.Run to avoid capturing any SynchronizationContext that could cause deadlocks +#if !WINDOWS_UWP && !WIN_UI Task.Run(() => MSTestTelemetryDataCollector.SendTelemetryAndResetAsync(_telemetrySender)).GetAwaiter().GetResult(); +#endif } } } diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs index 9d451ff808..fc20eba8d4 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestExecutor.cs @@ -20,7 +20,9 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; internal sealed class MSTestExecutor : ITestExecutor { private readonly CancellationToken _cancellationToken; +#if !WINDOWS_UWP && !WIN_UI private readonly Func, Task>? _telemetrySender; +#endif /// /// Token for canceling the test run. @@ -40,7 +42,11 @@ internal MSTestExecutor(CancellationToken cancellationToken, Func @@ -108,10 +114,12 @@ internal async Task RunTestsAsync(IEnumerable? tests, IRunContext? run Ensure.NotNullOrEmpty(tests); // Initialize telemetry collection if not already set +#if !WINDOWS_UWP && !WIN_UI if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) { _ = MSTestTelemetryDataCollector.EnsureInitialized(); } +#endif if (!MSTestDiscovererHelpers.InitializeDiscovery(from test in tests select test.Source, runContext, frameworkHandle, configuration, new TestSourceHandler())) { @@ -139,10 +147,12 @@ internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? run Ensure.NotNullOrEmpty(sources); // Initialize telemetry collection if not already set +#if !WINDOWS_UWP && !WIN_UI if (!MSTestTelemetryDataCollector.IsTelemetryOptedOut()) { _ = MSTestTelemetryDataCollector.EnsureInitialized(); } +#endif TestSourceHandler testSourceHandler = new(); if (!MSTestDiscovererHelpers.InitializeDiscovery(sources, runContext, frameworkHandle, configuration, testSourceHandler)) @@ -167,8 +177,13 @@ internal async Task RunTestsAsync(IEnumerable? sources, IRunContext? run public void Cancel() => _testRunCancellationToken?.Cancel(); - private async Task SendTelemetryAsync() - => await MSTestTelemetryDataCollector.SendTelemetryAndResetAsync(_telemetrySender).ConfigureAwait(false); +#if !WINDOWS_UWP && !WIN_UI + private Task SendTelemetryAsync() + => MSTestTelemetryDataCollector.SendTelemetryAndResetAsync(_telemetrySender); +#else + private static Task SendTelemetryAsync() + => Task.CompletedTask; +#endif private async Task RunTestsFromRightContextAsync(IFrameworkHandle frameworkHandle, Func runTestsAction) { diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/AssemblyEnumerator.cs b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/AssemblyEnumerator.cs index 3920849371..8e610696d4 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/AssemblyEnumerator.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/AssemblyEnumerator.cs @@ -145,7 +145,11 @@ internal virtual TypeEnumerator GetTypeEnumerator(Type type, string assemblyFile var typeValidator = new TypeValidator(ReflectHelper, discoverInternals); var testMethodValidator = new TestMethodValidator(ReflectHelper, discoverInternals); - return new TypeEnumerator(type, assemblyFileName, ReflectHelper, typeValidator, testMethodValidator, MSTestTelemetryDataCollector.Current); +#if !WINDOWS_UWP && !WIN_UI + return new TypeEnumerator(type, assemblyFileName, ReflectHelper, typeValidator, testMethodValidator, Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.MSTestTelemetryDataCollector.Current); +#else + return new TypeEnumerator(type, assemblyFileName, ReflectHelper, typeValidator, testMethodValidator); +#endif } private List DiscoverTestsInType( diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs index 8e66b42b03..dd8acb3eb8 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Discovery/TypeEnumerator.cs @@ -19,8 +19,11 @@ internal class TypeEnumerator private readonly TypeValidator _typeValidator; private readonly TestMethodValidator _testMethodValidator; private readonly ReflectHelper _reflectHelper; - private readonly MSTestTelemetryDataCollector? _telemetryDataCollector; +#if !WINDOWS_UWP && !WIN_UI + private readonly Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.MSTestTelemetryDataCollector? _telemetryDataCollector; +#endif +#if !WINDOWS_UWP && !WIN_UI /// /// Initializes a new instance of the class. /// @@ -30,14 +33,27 @@ internal class TypeEnumerator /// The validator for test classes. /// The validator for test methods. /// Optional telemetry data collector for tracking API usage. - internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflectHelper, TypeValidator typeValidator, TestMethodValidator testMethodValidator, MSTestTelemetryDataCollector? telemetryDataCollector = null) + internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflectHelper, TypeValidator typeValidator, TestMethodValidator testMethodValidator, Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.MSTestTelemetryDataCollector? telemetryDataCollector = null) +#else + /// + /// Initializes a new instance of the class. + /// + /// The reflected type. + /// The name of the assembly being reflected. + /// An instance to reflection helper for type information. + /// The validator for test classes. + /// The validator for test methods. + internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflectHelper, TypeValidator typeValidator, TestMethodValidator testMethodValidator) +#endif { _type = type; _assemblyFilePath = assemblyFilePath; _reflectHelper = reflectHelper; _typeValidator = typeValidator; _testMethodValidator = testMethodValidator; +#if !WINDOWS_UWP && !WIN_UI _telemetryDataCollector = telemetryDataCollector; +#endif } /// @@ -53,11 +69,13 @@ internal TypeEnumerator(Type type, string assemblyFilePath, ReflectHelper reflec } // Track class-level attributes for telemetry +#if !WINDOWS_UWP && !WIN_UI if (_telemetryDataCollector is not null) { Attribute[] classAttributes = _reflectHelper.GetCustomAttributesCached(_type); - _telemetryDataCollector.TrackDiscoveredClass(_type, classAttributes); + _telemetryDataCollector.TrackDiscoveredClass(classAttributes); } +#endif // If test class is valid, then get the tests return GetTests(warnings); @@ -153,7 +171,9 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, bool classDisables }; Attribute[] attributes = _reflectHelper.GetCustomAttributesCached(method); +#if !WINDOWS_UWP && !WIN_UI _telemetryDataCollector?.TrackDiscoveredMethod(attributes); +#endif TestMethodAttribute? testMethodAttribute = null; // Backward looping for backcompat. This used to be calls to _reflectHelper.GetFirstAttributeOrDefault diff --git a/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.cs b/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.cs index 2605b54ee0..d1d642a70f 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/MSTestSettings.cs @@ -297,7 +297,7 @@ internal static void PopulateSettings(IDiscoveryContext? context, IMessageLogger RunConfigurationSettings = runConfigurationSettings; // Track configuration source for telemetry -#if !WINDOWS_UWP +#if !WINDOWS_UWP && !WIN_UI if (MSTestTelemetryDataCollector.Current is { } telemetry) { telemetry.ConfigurationSource = configuration?["mstest"] is not null diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs index a49a748fdf..8bab8353ff 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Telemetry/MSTestTelemetryDataCollector.cs @@ -1,10 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -#if !WINDOWS_UWP +#if !WINDOWS_UWP && !WIN_UI using System.Security.Cryptography; -using System.Text.Json; -using System.Text.Json.Serialization; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -114,7 +112,9 @@ internal void TrackDiscoveredMethod(Attribute[] attributes) RetryBaseAttribute => nameof(RetryBaseAttribute), ConditionBaseAttribute => nameof(ConditionBaseAttribute), TestCategoryAttribute => nameof(TestCategoryAttribute), +#if !WIN_UI DeploymentItemAttribute => nameof(DeploymentItemAttribute), +#endif _ => attributeName, }; @@ -127,9 +127,8 @@ internal void TrackDiscoveredMethod(Attribute[] attributes) /// /// Records the attributes found on a test class during discovery. /// - /// The type of the test class. /// The cached attributes from the class. - internal void TrackDiscoveredClass(Type classType, Attribute[] attributes) + internal void TrackDiscoveredClass(Attribute[] attributes) { HasData = true; @@ -143,12 +142,12 @@ internal void TrackDiscoveredClass(Type classType, Attribute[] attributes) _customTestClassTypes.Add(AnonymizeString(attributeType.FullName ?? attributeType.Name)); } - string trackingName = attribute switch + string? trackingName = attribute switch { TestClassAttribute => nameof(TestClassAttribute), ParallelizeAttribute => nameof(ParallelizeAttribute), DoNotParallelizeAttribute => nameof(DoNotParallelizeAttribute), - _ => null!, + _ => null, }; if (trackingName is not null) @@ -180,30 +179,119 @@ internal Dictionary BuildMetrics() // Attribute usage (aggregated counts as JSON) if (_attributeCounts.Count > 0) { - metrics["mstest.attribute_usage"] = JsonSerializer.Serialize(_attributeCounts, MSTestTelemetryJsonContext.Default.DictionaryStringInt64); + metrics["mstest.attribute_usage"] = SerializeDictionary(_attributeCounts); } // Custom/inherited types (anonymized names) if (_customTestMethodTypes.Count > 0) { - metrics["mstest.custom_test_method_types"] = JsonSerializer.Serialize(_customTestMethodTypes, MSTestTelemetryJsonContext.Default.HashSetString); + metrics["mstest.custom_test_method_types"] = SerializeCollection(_customTestMethodTypes); } if (_customTestClassTypes.Count > 0) { - metrics["mstest.custom_test_class_types"] = JsonSerializer.Serialize(_customTestClassTypes, MSTestTelemetryJsonContext.Default.HashSetString); + metrics["mstest.custom_test_class_types"] = SerializeCollection(_customTestClassTypes); } // Assertion usage (drain the static counters) Dictionary assertionCounts = TelemetryCollector.DrainAssertionCallCounts(); if (assertionCounts.Count > 0) { - metrics["mstest.assertion_usage"] = JsonSerializer.Serialize(assertionCounts, MSTestTelemetryJsonContext.Default.DictionaryStringInt64); + metrics["mstest.assertion_usage"] = SerializeDictionary(assertionCounts); } return metrics; } + private static string SerializeCollection(IEnumerable values) + { + System.Text.StringBuilder builder = new("["); + bool isFirst = true; + + foreach (string value in values) + { + if (!isFirst) + { + builder.Append(','); + } + + AppendJsonString(builder, value); + isFirst = false; + } + + builder.Append(']'); + return builder.ToString(); + } + + private static string SerializeDictionary(Dictionary values) + { + System.Text.StringBuilder builder = new("{"); + bool isFirst = true; + + foreach (KeyValuePair value in values) + { + if (!isFirst) + { + builder.Append(','); + } + + AppendJsonString(builder, value.Key); + builder.Append(':'); + builder.Append(value.Value.ToString(System.Globalization.CultureInfo.InvariantCulture)); + isFirst = false; + } + + builder.Append('}'); + return builder.ToString(); + } + + private static void AppendJsonString(System.Text.StringBuilder builder, string value) + { + builder.Append('"'); + + foreach (char character in value) + { + switch (character) + { + case '"': + builder.Append("\\\""); + break; + case '\\': + builder.Append("\\\\"); + break; + case '\b': + builder.Append("\\b"); + break; + case '\f': + builder.Append("\\f"); + break; + case '\n': + builder.Append("\\n"); + break; + case '\r': + builder.Append("\\r"); + break; + case '\t': + builder.Append("\\t"); + break; + default: + if (char.IsControl(character)) + { + builder.Append("\\u"); + builder.Append(((int)character).ToString("x4", System.Globalization.CultureInfo.InvariantCulture)); + } + else + { + builder.Append(character); + } + + break; + } + } + + builder.Append('"'); + } + private static void AddSettingsMetrics(Dictionary metrics) { MSTestSettings settings = MSTestSettings.CurrentSettings; @@ -243,11 +331,11 @@ private static void AddSettingsMetrics(Dictionary metrics) private static string AnonymizeString(string value) { #if NET - byte[] hash = SHA256.HashData(Encoding.UTF8.GetBytes(value)); + byte[] hash = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(value)); return Convert.ToHexString(hash); #else - using SHA256 sha256 = SHA256.Create(); - byte[] hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(value)); + using var sha256 = SHA256.Create(); + byte[] hash = sha256.ComputeHash(System.Text.Encoding.UTF8.GetBytes(value)); return BitConverter.ToString(hash).Replace("-", string.Empty); #endif } @@ -283,10 +371,5 @@ internal static async Task SendTelemetryAndResetAsync(Func))] -[JsonSerializable(typeof(HashSet))] -internal sealed partial class MSTestTelemetryJsonContext : JsonSerializerContext; #endif diff --git a/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs b/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs index 783815ae91..209a90e804 100644 --- a/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs +++ b/src/TestFramework/TestFramework/Internal/TelemetryCollector.cs @@ -21,9 +21,11 @@ internal static void TrackAssertionCall(string assertionName) /// /// Gets a snapshot of all assertion call counts and resets the counters. - /// This is thread-safe: it atomically swaps the dictionary and drains the old one. + /// This is thread-safe but best-effort: it atomically swaps the dictionary and copies the old one. + /// In-flight calls to that race with the swap may be lost. + /// This is acceptable for telemetry where approximate counts are sufficient. /// - /// A dictionary mapping assertion names to call counts. + /// A dictionary mapping assertion names to their (best-effort) call counts. internal static Dictionary DrainAssertionCallCounts() { ConcurrentDictionary old = Interlocked.Exchange(ref s_assertionCallCounts, new ConcurrentDictionary()); diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs index 1e4dc4dfb4..f4b4080368 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TelemetryTests.cs @@ -42,6 +42,7 @@ public async Task MTP_RunTests_SendsTelemetryWithSettingsAndAttributes(string tf string content = await File.ReadAllTextAsync(diagFilePath, TestContext.CancellationToken); Assert.IsTrue(Regex.IsMatch(content, "mstest\\.attribute_usage"), $"Expected attribute_usage in telemetry.\n{content}"); Assert.IsTrue(Regex.IsMatch(content, "mstest\\.config_source"), $"Expected config_source in telemetry.\n{content}"); + Assert.IsTrue(Regex.IsMatch(content, "mstest\\.assertion_usage"), $"Expected assertion_usage in telemetry.\n{content}"); } #endregion From 07a82f9ed69eade457a00d8ac20a00b62b1615ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 18 Mar 2026 14:48:42 +0100 Subject: [PATCH 9/9] Address human reviewer feedback - Remove misleading 'for testing purposes' comment on MSTestDiscoverer internal constructor (it's also used by MSTestBridgedTestFramework) - Remove unnecessary Task.Run wrapper around SendTelemetryAndResetAsync in MSTestDiscoverer (no SyncContext deadlock risk in VSTest/MTP hosts, and SendTelemetryAndResetAsync catches all exceptions internally) --- .../MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs index d3961654b8..a9b0500526 100644 --- a/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs +++ b/src/Adapter/MSTest.TestAdapter/VSTestAdapter/MSTestDiscoverer.cs @@ -29,7 +29,7 @@ public MSTestDiscoverer() { } - internal /* for testing purposes */ MSTestDiscoverer(ITestSourceHandler testSourceHandler, Func, Task>? telemetrySender = null) + internal MSTestDiscoverer(ITestSourceHandler testSourceHandler, Func, Task>? telemetrySender = null) { _testSourceHandler = testSourceHandler; #if !WINDOWS_UWP && !WIN_UI @@ -74,9 +74,8 @@ internal void DiscoverTests(IEnumerable sources, IDiscoveryContext disco } finally { - // Use Task.Run to avoid capturing any SynchronizationContext that could cause deadlocks #if !WINDOWS_UWP && !WIN_UI - Task.Run(() => MSTestTelemetryDataCollector.SendTelemetryAndResetAsync(_telemetrySender)).GetAwaiter().GetResult(); + MSTestTelemetryDataCollector.SendTelemetryAndResetAsync(_telemetrySender).GetAwaiter().GetResult(); #endif } }