diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestMethodInfo.Execution.cs b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestMethodInfo.Execution.cs index 5514b9cd00..5c53d8cf10 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestMethodInfo.Execution.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestMethodInfo.Execution.cs @@ -78,7 +78,14 @@ private async Task ExecuteInternalAsync(object?[]? arguments, Cancel // After that, we invoke local cleanups (including Dispose) and finally global cleanups at last. foreach ((MethodInfo method, TimeoutInfo? timeoutInfo) in Parent.Parent.GlobalTestInitializations) { - await InvokeGlobalInitializeMethodAsync(method, timeoutInfo, timeoutTokenSource).ConfigureAwait(false); + TestFailedException? globalTestInitException = await InvokeGlobalInitializeMethodAsync(method, timeoutInfo, timeoutTokenSource).ConfigureAwait(false); + if (globalTestInitException is not null) + { + // The returned TestFailedException already carries the per-method timeout/cancellation + // message produced by FixtureMethodRunner; throwing it lets the outer catch record it + // as the test failure (HandleMethodException returns TestFailedException instances as-is). + throw globalTestInitException; + } } // TODO remove dry violation with TestMethodRunner @@ -195,7 +202,11 @@ private async Task ExecuteInternalAsync(object?[]? arguments, Cancel result.TestFailureException ??= HandleMethodException(ex, realException, TestClassName, TestMethodName); } - if (result.Outcome != UnitTestOutcome.Passed) + if (result.TestFailureException is TestFailedException testFailedException) + { + result.Outcome = testFailedException.Outcome; + } + else if (result.Outcome != UnitTestOutcome.Passed) { result.Outcome = ex is AssertInconclusiveException || ex.InnerException is AssertInconclusiveException ? UnitTestOutcome.Inconclusive diff --git a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestMethodInfo.Lifecycle.cs b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestMethodInfo.Lifecycle.cs index b2252f7f5d..2bdb6a0951 100644 --- a/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestMethodInfo.Lifecycle.cs +++ b/src/Adapter/MSTestAdapter.PlatformServices/Execution/TestMethodInfo.Lifecycle.cs @@ -38,6 +38,9 @@ private async SynchronizationContextPreservingTask RunTestCleanupMethodAsync(Tes _isTestCleanupInvoked = true; MethodInfo? testCleanupMethod = Parent.TestCleanupMethod; Exception? testCleanupException; + TestFailedException? globalTestCleanupException = null; + MethodInfo? globalTestCleanupFailingMethod = null; + MethodInfo? currentGlobalTestCleanupMethod = null; try { try @@ -79,13 +82,40 @@ private async SynchronizationContextPreservingTask RunTestCleanupMethodAsync(Tes foreach ((MethodInfo method, TimeoutInfo? timeoutInfo) in Parent.Parent.GlobalTestCleanups) { - await InvokeGlobalCleanupMethodAsync(method, timeoutInfo, timeoutTokenSource).ConfigureAwait(false); + currentGlobalTestCleanupMethod = method; + globalTestCleanupException = await InvokeGlobalCleanupMethodAsync(method, timeoutInfo, timeoutTokenSource).ConfigureAwait(false); + + if (globalTestCleanupException is not null) + { + // Stop on first global cleanup failure, mirroring the local test cleanup loop above. + globalTestCleanupFailingMethod = method; + break; + } + + currentGlobalTestCleanupMethod = null; } } } catch (Exception ex) { testCleanupException = ex; + if (globalTestCleanupFailingMethod is null && currentGlobalTestCleanupMethod is not null) + { + globalTestCleanupFailingMethod = currentGlobalTestCleanupMethod; + } + + if (globalTestCleanupFailingMethod is not null) + { + testCleanupMethod = globalTestCleanupFailingMethod; + } + } + + // If no earlier cleanup failure was recorded but a global test cleanup failed, surface it + // so the wrapping logic below produces a TestFailedException with the per-method message. + if (testCleanupException is null && globalTestCleanupException is not null) + { + testCleanupException = globalTestCleanupException; + testCleanupMethod = globalTestCleanupFailingMethod; } // If testCleanup was successful, then don't do anything @@ -95,7 +125,15 @@ private async SynchronizationContextPreservingTask RunTestCleanupMethodAsync(Tes } Exception realException = testCleanupException.GetRealException(); - UnitTestOutcome outcomeFromRealException = realException is AssertInconclusiveException ? UnitTestOutcome.Inconclusive : UnitTestOutcome.Failed; + + // Check `realException` (not `testCleanupException`) so that a `TestFailedException` wrapped in + // a `TargetInvocationException` / `TypeInitializationException` still preserves its Outcome + // (for example a Timeout outcome must not be silently downgraded to Failed). + UnitTestOutcome outcomeFromRealException = realException is TestFailedException testFailedException + ? testFailedException.Outcome + : realException is AssertInconclusiveException + ? UnitTestOutcome.Inconclusive + : UnitTestOutcome.Failed; result.Outcome = result.Outcome.GetMoreImportantOutcome(outcomeFromRealException); realException = testCleanupMethod != null diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutCooperativeGlobalTestCancellationTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutCooperativeGlobalTestCancellationTests.cs index 0f815127e2..f789faafc4 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutCooperativeGlobalTestCancellationTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutCooperativeGlobalTestCancellationTests.cs @@ -10,7 +10,6 @@ namespace MSTest.Acceptance.IntegrationTests; public sealed class TimeoutCooperativeGlobalTestCancellationTests : AcceptanceTestBase { [TestMethod] - [Ignore("Tracked by https://github.com/microsoft/testfx/issues/6198. The TestFailedException returned by InvokeGlobalInitializeMethodAsync is currently discarded in TestMethodInfo.Execution.cs, so timeouts on global test initialize methods do not fail the test.")] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task CooperativeCancellation_WhenGlobalTestInitTimeoutExpires_StepThrows(string tfm) { @@ -27,7 +26,6 @@ public async Task CooperativeCancellation_WhenGlobalTestInitTimeoutExpires_StepT } [TestMethod] - [Ignore("Tracked by https://github.com/microsoft/testfx/issues/6198. The TestFailedException returned by InvokeGlobalInitializeMethodAsync is currently discarded in TestMethodInfo.Execution.cs, so timeouts on global test initialize methods do not fail the test.")] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task CooperativeCancellation_WhenGlobalTestInitTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) { @@ -44,7 +42,6 @@ public async Task CooperativeCancellation_WhenGlobalTestInitTimeoutExpiresAndUse } [TestMethod] - [Ignore("Tracked by https://github.com/microsoft/testfx/issues/6198. The TestFailedException returned by InvokeGlobalCleanupMethodAsync is currently discarded in TestMethodInfo.Lifecycle.cs, so timeouts on global test cleanup methods do not fail the test.")] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task CooperativeCancellation_WhenGlobalTestCleanupTimeoutExpires_StepThrows(string tfm) { @@ -61,7 +58,6 @@ public async Task CooperativeCancellation_WhenGlobalTestCleanupTimeoutExpires_St } [TestMethod] - [Ignore("Tracked by https://github.com/microsoft/testfx/issues/6198. The TestFailedException returned by InvokeGlobalCleanupMethodAsync is currently discarded in TestMethodInfo.Lifecycle.cs, so timeouts on global test cleanup methods do not fail the test.")] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task CooperativeCancellation_WhenGlobalTestCleanupTimeoutExpiresAndUserChecksToken_StepThrows(string tfm) { diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutWhenCanceledTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutWhenCanceledTests.cs index 2a714a584c..4b6e8b45e5 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutWhenCanceledTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutWhenCanceledTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Platform.Acceptance.IntegrationTests; @@ -34,13 +34,11 @@ public async Task ClassInitBase_WhenTestContextCanceled_ClassInitializeTaskIsCan => await RunAndAssertTestWasCanceledAsync(tfm, "TESTCONTEXT_CANCEL_", "baseClassInit"); [TestMethod] - [Ignore("Tracked by https://github.com/microsoft/testfx/issues/6198. The TestFailedException returned by InvokeGlobalInitializeMethodAsync is currently discarded in TestMethodInfo.Execution.cs, so cancellations on global test initialize methods do not fail the test.")] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task GlobalTestInitialize_WhenTestContextCanceled_GlobalTestInitializeTaskIsCanceled(string tfm) => await RunAndAssertTestWasCanceledAsync(tfm, "TESTCONTEXT_CANCEL_", "globalTestInit"); [TestMethod] - [Ignore("Tracked by https://github.com/microsoft/testfx/issues/6198. The TestFailedException returned by InvokeGlobalCleanupMethodAsync is currently discarded in TestMethodInfo.Lifecycle.cs, so cancellations on global test cleanup methods do not fail the test.")] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task GlobalTestCleanup_WhenTestContextCanceled_GlobalTestCleanupTaskIsCanceled(string tfm) => await RunAndAssertTestWasCanceledAsync(tfm, "TESTCONTEXT_CANCEL_", "globalTestCleanup"); diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutWhenExpiresTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutWhenExpiresTests.cs index 6dc0c54c04..a5876b3c74 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutWhenExpiresTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TimeoutWhenExpiresTests.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Platform.Acceptance.IntegrationTests; @@ -119,31 +119,26 @@ public async Task TestCleanup_WhenTimeoutExpires_TestCleanupIsCanceled_Attribute => await RunAndAssertAttributeTakesPrecedenceAsync(tfm, "testCleanup"); [TestMethod] - [Ignore("Tracked by https://github.com/microsoft/testfx/issues/6198. The TestFailedException returned by InvokeGlobalInitializeMethodAsync is currently discarded in TestMethodInfo.Execution.cs, so timeouts on global test initialize methods do not fail the test.")] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task GlobalTestInitialize_WhenTimeoutExpires_GlobalTestInitializeTaskIsCanceled(string tfm) => await RunAndAssertTestTimedOutAsync(tfm, "LONG_WAIT_", "globalTestInit"); [TestMethod] - [Ignore("Tracked by https://github.com/microsoft/testfx/issues/6198. The TestFailedException returned by InvokeGlobalInitializeMethodAsync is currently discarded in TestMethodInfo.Execution.cs, so timeouts on global test initialize methods do not fail the test.")] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task GlobalTestInitialize_WhenTimeoutExpiresAndTestContextTokenIsUsed_GlobalTestInitializeExits(string tfm) => await RunAndAssertTestTimedOutAsync(tfm, "TIMEOUT_", "globalTestInit"); [TestMethod] - [Ignore("Tracked by https://github.com/microsoft/testfx/issues/6198. The TestFailedException returned by InvokeGlobalInitializeMethodAsync is currently discarded in TestMethodInfo.Execution.cs, so timeouts on global test initialize methods do not fail the test.")] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task GlobalTestInitialize_WhenTimeoutExpires_GlobalTestInitializeIsCanceled_AttributeTakesPrecedence(string tfm) => await RunAndAssertAttributeTakesPrecedenceAsync(tfm, "globalTestInit"); [TestMethod] - [Ignore("Tracked by https://github.com/microsoft/testfx/issues/6198. The TestFailedException returned by InvokeGlobalCleanupMethodAsync is currently discarded in TestMethodInfo.Lifecycle.cs, so timeouts on global test cleanup methods do not fail the test.")] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task GlobalTestCleanup_WhenTimeoutExpires_GlobalTestCleanupTaskIsCanceled(string tfm) => await RunAndAssertTestTimedOutAsync(tfm, "LONG_WAIT_", "globalTestCleanup"); [TestMethod] - [Ignore("Tracked by https://github.com/microsoft/testfx/issues/6198. The TestFailedException returned by InvokeGlobalCleanupMethodAsync is currently discarded in TestMethodInfo.Lifecycle.cs, so timeouts on global test cleanup methods do not fail the test.")] [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] public async Task GlobalTestCleanup_WhenTimeoutExpires_GlobalTestCleanupIsCanceled_AttributeTakesPrecedence(string tfm) => await RunAndAssertAttributeTakesPrecedenceAsync(tfm, "globalTestCleanup"); diff --git a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestMethodInfoTests.cs b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestMethodInfoTests.cs index e4ed2fa1a8..16f41d907b 100644 --- a/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestMethodInfoTests.cs +++ b/test/UnitTests/MSTestAdapter.PlatformServices.UnitTests/Execution/TestMethodInfoTests.cs @@ -156,6 +156,8 @@ public TestMethodInfoTests() DummyTestClass.TestInitializeMethodBody = value => { }; DummyTestClass.TestMethodBody = instance => { }; DummyTestClass.TestCleanupMethodBody = value => { }; + DummyTestClass.GlobalTestInitializeMethodBodyAsync = _ => Task.CompletedTask; + DummyTestClass.GlobalTestCleanupMethodBodyAsync = _ => Task.CompletedTask; } protected override void Dispose(bool disposing) @@ -921,6 +923,31 @@ public async Task TestMethodInfoInvokeWhenTestThrowsAssertInconclusiveReturnsExp "Microsoft.VisualStudio.TestPlatform.MSTestAdapter.UnitTests.Execution.TestMethodInfoTests.DummyTestClass.DummyTestInitializeMethod"); } + public async Task TestMethodInfoInvokeShouldPreserveTimeoutOutcomeForGlobalTestInitialize() + { + DummyTestClass.GlobalTestInitializeMethodBodyAsync = async testContext => + { + try + { + await Task.Delay(TimeSpan.FromSeconds(10), testContext.CancellationTokenSource.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Expected when the framework signals cancellation/timeout. + } + }; + _testAssemblyInfo.GlobalTestInitializations.Add((typeof(DummyTestClass).GetMethod(nameof(DummyTestClass.DummyGlobalTestInitializeMethod))!, TimeoutInfo.FromTimeout(1))); + + TestResult result = await _testMethodInfo.InvokeAsync(null); + + result.Outcome.Should().Be(UnitTestOutcome.Timeout); + var exception = result.TestFailureException as TestFailedException; + exception.Should().NotBeNull(); + exception!.Outcome.Should().Be(UnitTestOutcome.Timeout); + exception.Message.Should().Contain("DummyGlobalTestInitializeMethod"); + exception.Message.Should().Contain("timed out after 1ms"); + } + public async Task TestMethodInfoInvokeWhenConstructorThrowsAssertInconclusiveReturnsExpectedResult() { // Arrange. @@ -1218,6 +1245,46 @@ public async Task TestMethodInfoInvokeShouldSetMoreImportantOutcomeIfTestCleanup result.Outcome.Should().Be(UnitTestOutcome.Failed); } + public async Task TestMethodInfoInvokeShouldPreserveTimeoutOutcomeForGlobalTestCleanup() + { + DummyTestClass.GlobalTestCleanupMethodBodyAsync = async testContext => + { + try + { + await Task.Delay(TimeSpan.FromSeconds(10), testContext.CancellationTokenSource.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Expected when the framework signals cancellation/timeout. + } + }; + _testAssemblyInfo.GlobalTestCleanups.Add((typeof(DummyTestClass).GetMethod(nameof(DummyTestClass.DummyGlobalTestCleanupMethod))!, TimeoutInfo.FromTimeout(1))); + + TestResult result = await _testMethodInfo.InvokeAsync(null); + + result.Outcome.Should().Be(UnitTestOutcome.Timeout); + var exception = result.TestFailureException as TestFailedException; + exception.Should().NotBeNull(); + exception!.Outcome.Should().Be(UnitTestOutcome.Timeout); + exception.Message.Should().Contain("DummyGlobalTestCleanupMethod"); + exception.Message.Should().Contain("timed out after 1ms"); + } + + public async Task TestMethodInfoInvokeShouldAttributeGlobalTestCleanupMethodWhenItThrows() + { + DummyTestClass.TestCleanupMethodBody = _ => { }; + DummyTestClass.GlobalTestCleanupMethodBodyAsync = _ => throw new InvalidOperationException("global cleanup failed"); + _testClassInfo.TestCleanupMethod = typeof(DummyTestClass).GetMethod(nameof(DummyTestClass.DummyTestCleanupMethod))!; + _testAssemblyInfo.GlobalTestCleanups.Add((typeof(DummyTestClass).GetMethod(nameof(DummyTestClass.DummyGlobalTestCleanupMethod))!, TimeoutInfo: null)); + + TestResult result = await _testMethodInfo.InvokeAsync(null); + + result.Outcome.Should().Be(UnitTestOutcome.Failed); + var exception = result.TestFailureException as TestFailedException; + exception.Should().NotBeNull(); + exception!.Message.Should().Contain($"TestCleanup method {typeof(DummyTestClass).FullName}.{nameof(DummyTestClass.DummyGlobalTestCleanupMethod)} threw exception"); + } + public async Task TestMethodInfoInvokeShouldCallDisposeForDisposableTestClass() { bool disposeCalled = false; @@ -1848,6 +1915,10 @@ public class DummyTestClass : DummyTestClassBase public static Func DummyAsyncTestMethodBody { get; set; } = null!; + public static Func GlobalTestInitializeMethodBodyAsync { get; set; } = null!; + + public static Func GlobalTestCleanupMethodBodyAsync { get; set; } = null!; + public static TestContext GetTestContext() => s_tc; public TestContext TestContext @@ -1889,6 +1960,12 @@ public Task DummyAsyncTestMethod() => [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.NoOptimization)] public void DummyParamsArgumentMethod(int i, params string[] args) => TestMethodBody(this); + + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.NoOptimization)] + public static async Task DummyGlobalTestInitializeMethod(TestContext testContext) => await GlobalTestInitializeMethodBodyAsync(testContext); + + [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.NoOptimization)] + public static async Task DummyGlobalTestCleanupMethod(TestContext testContext) => await GlobalTestCleanupMethodBodyAsync(testContext); } public class DummyTestClassWithRetryAttributeMethods