diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestExecutionManager.Runner.cs b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestExecutionManager.Runner.cs index dd43d20b47..1814e96eb4 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestExecutionManager.Runner.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestExecutionManager.Runner.cs @@ -2,8 +2,9 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; -using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; using Microsoft.VisualStudio.TestPlatform.ObjectModel; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; @@ -17,11 +18,11 @@ internal void SendTestResults( TestTools.UnitTesting.TestResult[] unitTestResults, DateTimeOffset startTime, DateTimeOffset endTime, - ITestExecutionRecorder testExecutionRecorder) + ITestResultRecorder testResultRecorder) { if (unitTestResults.Length == 0) { - testExecutionRecorder.RecordEnd(test, TestOutcome.None); + testResultRecorder.RecordEmptyResult(test); return; } @@ -29,39 +30,14 @@ internal void SendTestResults( { _testRunCancellationToken?.ThrowIfCancellationRequested(); - var testResult = unitTestResult.ToTestResult( - test, - startTime, - endTime, - _environment.MachineName, - MSTestSettings.CurrentSettings); - - testExecutionRecorder.RecordEnd(test, testResult.Outcome); - - if (testResult.Outcome == TestOutcome.Failed) - { - if (PlatformServiceProvider.Instance.AdapterTraceLogger.IsInfoEnabled) - { - PlatformServiceProvider.Instance.AdapterTraceLogger.Info("MSTestExecutor:Test {0} failed. ErrorMessage:{1}, ErrorStackTrace:{2}.", testResult.TestCase.FullyQualifiedName, testResult.ErrorMessage, testResult.ErrorStackTrace); - } - #if !WINDOWS_UWP && !WIN_UI - _hasAnyTestFailed = true; -#endif - } - - try - { - if (testResult.Outcome != TestOutcome.NotFound - || !RuntimeContext.IsHotReloadEnabled) - { - testExecutionRecorder.RecordResult(testResult); - } - } - catch (TestCanceledException) + if (testResultRecorder.RecordResult(test, unitTestResult, startTime, endTime)) { - // Ignore this exception + _hasAnyTestFailed = true; } +#else + testResultRecorder.RecordResult(test, unitTestResult, startTime, endTime); +#endif } } @@ -95,6 +71,10 @@ private async Task ExecuteTestsWithTestRunnerAsync( ? new RemotingMessageLogger(testExecutionRecorder) : testExecutionRecorder; + // Translate the VSTest recorder into the platform-agnostic result recorder a single time for this + // test set. This is the boundary at which VSTest result construction is applied. + ITestResultRecorder testResultRecorder = testExecutionRecorder.ToTestResultRecorder(_environment.MachineName, MSTestSettings.CurrentSettings); + foreach (TestCase currentTest in orderedTests) { _testRunCancellationToken?.ThrowIfCancellationRequested(); @@ -105,7 +85,7 @@ private async Task ExecuteTestsWithTestRunnerAsync( UnitTestElement unitTestElement = currentTest.ToUnitTestElementWithUpdatedSource(source); - testExecutionRecorder.RecordStart(currentTest); + testResultRecorder.RecordStart(currentTest); DateTimeOffset startTime = DateTimeOffset.Now; @@ -143,7 +123,7 @@ private async Task ExecuteTestsWithTestRunnerAsync( DateTimeOffset endTime = DateTimeOffset.Now; - SendTestResults(currentTest, unitTestResult, startTime, endTime, testExecutionRecorder); + SendTestResults(currentTest, unitTestResult, startTime, endTime, testResultRecorder); } } } diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Interfaces/ITestResultRecorder.cs b/src/Adapter/MSTestAdapter.PlatformServices/Interfaces/ITestResultRecorder.cs new file mode 100644 index 0000000000..d944cd33ff --- /dev/null +++ b/src/Adapter/MSTestAdapter.PlatformServices/Interfaces/ITestResultRecorder.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.ObjectModel; + +using FrameworkTestResult = Microsoft.VisualStudio.TestTools.UnitTesting.TestResult; + +namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; + +/// +/// Platform-agnostic sink for reporting the lifecycle and results of test execution to whichever +/// test host is running the tests. +/// +/// +/// This abstraction lets the platform services layer report test start/end and results without taking a +/// dependency on a specific test platform's result object model (for example the VSTest TestResult, +/// TestOutcome and attachment types). The concrete recorder is provided at the platform boundary by a +/// wrapper over the host's result recorder (currently TestResultRecorderExtensions, which wraps the +/// VSTest ITestExecutionRecorder), and is expected to move fully out of the platform services layer in +/// a later phase. +/// +internal interface ITestResultRecorder +{ + /// + /// Signals that execution of the given test case has started. + /// + /// The test case whose execution is starting. + void RecordStart(TestCase testCase); + + /// + /// Signals that execution of the given test case ended without producing any result. + /// + /// The test case whose execution ended. + void RecordEmptyResult(TestCase testCase); + + /// + /// Reports a single framework for the given test case to the test host. + /// + /// The test case the result belongs to. + /// The framework result produced by executing the test. + /// The time at which the test started executing. + /// The time at which the test finished executing. + /// if the result was reported as a failure; otherwise, . + bool RecordResult(TestCase testCase, FrameworkTestResult unitTestResult, DateTimeOffset startTime, DateTimeOffset endTime); +} diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Services/TestResultRecorderExtensions.cs b/src/Adapter/MSTestAdapter.PlatformServices/Services/TestResultRecorderExtensions.cs new file mode 100644 index 0000000000..911f563609 --- /dev/null +++ b/src/Adapter/MSTestAdapter.PlatformServices/Services/TestResultRecorderExtensions.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter; +using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions; +using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers; +using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface; +using Microsoft.VisualStudio.TestPlatform.ObjectModel; +using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter; + +using FrameworkTestResult = Microsoft.VisualStudio.TestTools.UnitTesting.TestResult; + +namespace Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices; + +/// +/// Bridges a VSTest to the platform-agnostic . +/// +/// +/// This is the single translation point between the VSTest result object model (TestResult, +/// TestOutcome, attachments, ...) and the platform services result-reporting abstraction. It is +/// expected to move entirely into the adapter layer once the execution pipeline no longer flows VSTest +/// recorders through the platform services. +/// +internal static class TestResultRecorderExtensions +{ + /// + /// Wraps a VSTest as an . + /// + /// The host recorder to wrap. + /// The computer name stamped on reported results. + /// The current MSTest settings used to map framework outcomes to host outcomes. + /// A platform-agnostic recorder that forwards to . + public static ITestResultRecorder ToTestResultRecorder(this ITestExecutionRecorder testExecutionRecorder, string computerName, MSTestSettings settings) + => new HostTestResultRecorder(testExecutionRecorder, computerName, settings); + + private sealed class HostTestResultRecorder : ITestResultRecorder + { + private readonly ITestExecutionRecorder _testExecutionRecorder; + private readonly string _computerName; + private readonly MSTestSettings _settings; + + public HostTestResultRecorder(ITestExecutionRecorder testExecutionRecorder, string computerName, MSTestSettings settings) + { + _testExecutionRecorder = testExecutionRecorder; + _computerName = computerName; + _settings = settings; + } + + public void RecordStart(TestCase testCase) + => _testExecutionRecorder.RecordStart(testCase); + + public void RecordEmptyResult(TestCase testCase) + => _testExecutionRecorder.RecordEnd(testCase, TestOutcome.None); + + public bool RecordResult(TestCase testCase, FrameworkTestResult unitTestResult, DateTimeOffset startTime, DateTimeOffset endTime) + { + var testResult = unitTestResult.ToTestResult(testCase, startTime, endTime, _computerName, _settings); + + _testExecutionRecorder.RecordEnd(testCase, testResult.Outcome); + + bool isFailed = testResult.Outcome == TestOutcome.Failed; + if (isFailed && PlatformServiceProvider.Instance.AdapterTraceLogger.IsInfoEnabled) + { + PlatformServiceProvider.Instance.AdapterTraceLogger.Info("MSTestExecutor:Test {0} failed. ErrorMessage:{1}, ErrorStackTrace:{2}.", testResult.TestCase.FullyQualifiedName, testResult.ErrorMessage, testResult.ErrorStackTrace); + } + + try + { + if (testResult.Outcome != TestOutcome.NotFound + || !RuntimeContext.IsHotReloadEnabled) + { + _testExecutionRecorder.RecordResult(testResult); + } + } + catch (TestCanceledException) + { + // Ignore this exception + } + + // A failure is reported only once RecordResult has completed (a swallowed TestCanceledException + // still counts as completed). This mirrors the original inline flow where the failure was + // observed as part of reporting the result. + return isFailed; + } + } +} diff --git a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestExecutionManagerTests.cs b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestExecutionManagerTests.cs index 18cfc4734a..b9bb6963f4 100644 --- a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestExecutionManagerTests.cs +++ b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestExecutionManagerTests.cs @@ -125,7 +125,7 @@ public void SendTestResults_WhenUnitTestResultsIsEmpty_RecordsEndWithoutResult() TestCase testCase = GetTestCase(typeof(DummyTestClass), "PassingTest"); Microsoft.VisualStudio.TestTools.UnitTesting.TestResult[] unitTestResults = []; - _testExecutionManager.SendTestResults(testCase, unitTestResults, DateTimeOffset.Now, DateTimeOffset.Now, _frameworkHandle); + _testExecutionManager.SendTestResults(testCase, unitTestResults, DateTimeOffset.Now, DateTimeOffset.Now, _frameworkHandle.ToTestResultRecorder(EnvironmentWrapper.Instance.MachineName, MSTestSettings.CurrentSettings)); _frameworkHandle.TestCaseEndList.Should().Equal("PassingTest:None"); _frameworkHandle.ResultsList.Should().BeEmpty();