diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalResources.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalResources.cs index 5fdd938848..7905c278c0 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalResources.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalResources.cs @@ -53,6 +53,8 @@ internal static partial class TerminalResources internal static string @Error => GetResourceString("Error"); + internal static string @ErroredAssembliesHeader => GetResourceString("ErroredAssembliesHeader"); + internal static string @ExitCode => GetResourceString("ExitCode"); internal static string @Expected => GetResourceString("Expected"); diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalResources.resx b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalResources.resx index 4899e7a77b..af81784291 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalResources.resx +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalResources.resx @@ -283,6 +283,9 @@ Valid values are 'All', 'Failed', 'None'. Default is 'All' (or 'Failed' when an Handshake failures: + + Errored assemblies: + Exit code diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.ErroredAssemblies.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.ErroredAssemblies.cs new file mode 100644 index 0000000000..9e805c0271 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.ErroredAssemblies.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.OutputDevice.Terminal; + +[UnsupportedOSPlatform("browser")] +internal sealed partial class TerminalTestReporter +{ +#if NET9_0_OR_GREATER + private readonly Lock _erroredAssembliesLock = new(); +#else + private readonly object _erroredAssembliesLock = new(); +#endif + + /// + /// Assemblies that handshaked but then exited with a non-zero code without reporting any failed test (crash, + /// Environment.FailFast, a hang-dump kill, an option rejected after the handshake, ...). Their process + /// output is printed inline when the assembly completes, but in a large multi-assembly run that inline output is + /// scrolled far above the summary, so we also record it here and re-print it in an end-of-run recap. + /// + private readonly List _erroredAssemblies = []; + + /// + /// Records an assembly that ended unsuccessfully with no failed test so its process output (exit code + stdout + + /// stderr) can be re-printed at the end of the run. Orchestrator-only: only the multi-process dotnet test + /// orchestrator reaches the exit-code overload of . + /// + private void RecordErroredAssembly(TestProgressState assemblyRun, int exitCode, string? outputData, string? errorData) + { + lock (_erroredAssembliesLock) + { + _erroredAssemblies.Add(new ErroredAssemblyRecord(assemblyRun.Assembly, assemblyRun.TargetFramework, assemblyRun.Architecture, exitCode, outputData, errorData)); + } + } + + /// + /// Re-print assemblies that errored during the run (non-zero exit with no failed test) so that — as with the + /// handshake-failure recap — the actionable process output is shown at the end of the run rather than buried in + /// the middle of a large multi-assembly run. + /// + private void AppendErroredAssemblyRecap(ITerminal terminal) + { + ErroredAssemblyRecord[] errored; + lock (_erroredAssembliesLock) + { + if (_erroredAssemblies.Count == 0) + { + return; + } + + errored = _erroredAssemblies.ToArray(); + } + + terminal.AppendLine(); + terminal.SetColor(TerminalColor.DarkRed); + terminal.AppendLine(TerminalResources.ErroredAssembliesHeader); + terminal.ResetColor(); + + foreach (ErroredAssemblyRecord failure in errored) + { + terminal.Append(SingleIndentation); + AppendAssemblyLinkTargetFrameworkAndArchitecture(terminal, failure.AssemblyPath, failure.TargetFramework, failure.Architecture); + terminal.AppendLine(); + AppendExecutableSummary(terminal, failure.ExitCode, failure.OutputData, failure.ErrorData); + } + } + + private readonly record struct ErroredAssemblyRecord( + string AssemblyPath, + string? TargetFramework, + string? Architecture, + int ExitCode, + string? OutputData, + string? ErrorData); +} diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs index dea0312569..9e356aaa82 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs @@ -123,6 +123,16 @@ internal void AssemblyRunCompleted(string executionId, int exitCode, string? out } _terminalWithProgress.WriteToTerminal(terminal => AppendExecutableSummary(terminal, exitCode, outputData, errorData)); + + // A non-zero exit with no failed test is a process-level error (crash, FailFast, hang-dump kill, an option + // rejected after the handshake, ...) whose inline output above is easily lost in a large run. Record it so + // it is re-printed in the end-of-run recap (see AppendErroredAssemblyRecap). Assemblies that exit non-zero + // *because* tests failed are intentionally excluded: those failures are already reported per-test, and this + // set mirrors the "error: N" count in the summary (failedAssembliesWithoutFailedTests). + if (assemblyRun.FailedTests == 0) + { + RecordErroredAssembly(assemblyRun, exitCode, outputData, errorData); + } } public void TestExecutionCompleted(DateTimeOffset endTime, int? exitCode) @@ -137,8 +147,8 @@ public void TestExecutionCompleted(DateTimeOffset endTime, int? exitCode) // This is relevant for HotReload scenarios. We want the next test sessions to start fresh, so we reset all // per-run state here (after the summary above has consumed it): the per-assembly runs, the collected - // artifacts, the handshake failures, and the cancellation flag. Otherwise a later session would re-print the - // previous session's artifacts/handshake failures or stay stuck in the aborted state. + // artifacts, the handshake failures, the errored assemblies, and the cancellation flag. Otherwise a later + // session would re-print the previous session's artifacts/failures or stay stuck in the aborted state. _assemblies.Clear(); _artifacts.Clear(); WasCancelled = false; @@ -147,6 +157,11 @@ public void TestExecutionCompleted(DateTimeOffset endTime, int? exitCode) _handshakeFailures.Clear(); } + lock (_erroredAssembliesLock) + { + _erroredAssemblies.Clear(); + } + _testExecutionStartTime = null; _testExecutionEndTime = null; } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Summary.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Summary.cs index 241e82f465..b03abc643e 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Summary.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Summary.cs @@ -203,6 +203,10 @@ private void AppendTestRunSummary(ITerminal terminal) // Re-print any handshake failures (orchestrator-only) at the very end so they aren't lost above the summary. // No-op for the in-process host, which never reports handshake failures. AppendHandshakeFailureRecap(terminal); + + // Re-print any assemblies that errored (non-zero exit with no failed test) for the same reason: the inline + // process output is otherwise buried in the middle of a large run. No-op for the in-process host. + AppendErroredAssemblyRecap(terminal); } /// diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.cs.xlf b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.cs.xlf index bbb8d3c8aa..60f5c865e3 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.cs.xlf @@ -57,6 +57,11 @@ error Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'. + + Errored assemblies: + Errored assemblies: + + Exit code Exit code diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.de.xlf b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.de.xlf index 4aeabedbe6..d664dd4942 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.de.xlf @@ -57,6 +57,11 @@ error Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'. + + Errored assemblies: + Errored assemblies: + + Exit code Exit code diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.es.xlf b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.es.xlf index 267165004e..ca1b15a5e8 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.es.xlf @@ -57,6 +57,11 @@ error Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'. + + Errored assemblies: + Errored assemblies: + + Exit code Exit code diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.fr.xlf b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.fr.xlf index 67e3f9b8d2..4c2d54fedf 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.fr.xlf @@ -57,6 +57,11 @@ error Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'. + + Errored assemblies: + Errored assemblies: + + Exit code Exit code diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.it.xlf b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.it.xlf index 6426bb3b12..c1b7b59448 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.it.xlf @@ -57,6 +57,11 @@ error Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'. + + Errored assemblies: + Errored assemblies: + + Exit code Exit code diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ja.xlf b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ja.xlf index 8e7661642f..37b12879c4 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ja.xlf @@ -57,6 +57,11 @@ error Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'. + + Errored assemblies: + Errored assemblies: + + Exit code Exit code diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ko.xlf b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ko.xlf index 6be5376134..236468c1f1 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ko.xlf @@ -57,6 +57,11 @@ error Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'. + + Errored assemblies: + Errored assemblies: + + Exit code Exit code diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.pl.xlf b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.pl.xlf index 5fb66e0b57..abf57e24fc 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.pl.xlf @@ -57,6 +57,11 @@ error Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'. + + Errored assemblies: + Errored assemblies: + + Exit code Exit code diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.pt-BR.xlf index 97a5c9148b..3fa4d6074f 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.pt-BR.xlf @@ -57,6 +57,11 @@ error Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'. + + Errored assemblies: + Errored assemblies: + + Exit code Exit code diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ru.xlf b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ru.xlf index bb7814ca73..762c39182c 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ru.xlf @@ -57,6 +57,11 @@ error Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'. + + Errored assemblies: + Errored assemblies: + + Exit code Exit code diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.tr.xlf b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.tr.xlf index 9591317535..e294d4f4f5 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.tr.xlf @@ -57,6 +57,11 @@ error Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'. + + Errored assemblies: + Errored assemblies: + + Exit code Exit code diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.zh-Hans.xlf index aeeb02fa59..3121dc82e5 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.zh-Hans.xlf @@ -57,6 +57,11 @@ error Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'. + + Errored assemblies: + Errored assemblies: + + Exit code Exit code diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.zh-Hant.xlf index 5bf030732f..bc52168460 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.zh-Hant.xlf @@ -57,6 +57,11 @@ error Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'. + + Errored assemblies: + Errored assemblies: + + Exit code Exit code diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs index 47e2f4af70..91169fa4e2 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs @@ -1755,6 +1755,61 @@ public void TestExecutionCompleted_WhenHandshakeFailures_PrintsRecapAndFailsRun( Assert.Contains($"{TerminalResources.TestRunSummary} {TerminalResources.Failed}!", output); } + // dotnet/sdk#51952: an assembly that DOES handshake but then exits non-zero without any failed test (a crash, + // Environment.FailFast, a hang-dump kill, an option rejected after the handshake, ...) has its process output + // printed inline when it completes, which is easily lost in the middle of a large multi-assembly run. The + // end-of-run summary must re-print an "Errored assemblies:" recap with the captured output so it is discoverable. + [TestMethod] + public void TestExecutionCompleted_WhenAssemblyErroredWithoutFailedTests_ReprintsErroredAssemblyRecap() + { + var stringBuilderConsole = new StringBuilderConsole(); + TerminalTestReporter terminalReporter = CreateOrchestratorReporter(stringBuilderConsole, showAssemblyStartAndComplete: false); + terminalReporter.TestExecutionStarted(DateTimeOffset.MinValue, workerCount: 1, isDiscovery: false, isHelp: false, isRetry: false); + + string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\repo\Crashy.dll" : "/repo/Crashy.dll"; + terminalReporter.AssemblyRunStarted(assembly, "net9.0", "x64", "exec-1", "inst-1"); + ReportOrchestratorTest(terminalReporter, assembly, "exec-1", "inst-1", "t-1", TestOutcome.Passed); + + // The single test passed but the process exits non-zero -> error category (FailedTests == 0). + terminalReporter.AssemblyRunCompleted("exec-1", exitCode: 42, outputData: "boom stdout", errorData: "boom stderr"); + + // It is NOT a handshake failure (the assembly was registered). + Assert.IsFalse(terminalReporter.HasHandshakeFailure); + + terminalReporter.TestExecutionCompleted(DateTimeOffset.MaxValue, exitCode: 42); + + string output = stringBuilderConsole.Output; + + // The end-of-run recap header is printed and identifies the errored assembly with its captured output. + Assert.Contains(TerminalResources.ErroredAssembliesHeader, output); + string recap = output[output.IndexOf(TerminalResources.ErroredAssembliesHeader, StringComparison.Ordinal)..]; + Assert.Contains("Crashy.dll", recap); + Assert.Contains($"{TerminalResources.ExitCode}: 42", recap); + Assert.Contains("boom stdout", recap); + Assert.Contains("boom stderr", recap); + } + + // Guard for the recap's scope: an assembly that exits non-zero BECAUSE a test failed must not be added to the + // "Errored assemblies:" recap - those failures are already reported per-test, and re-dumping every failing + // assembly's process output at the end would be noise. The recap is reserved for unexplained process errors. + [TestMethod] + public void TestExecutionCompleted_WhenAssemblyExitedNonZeroWithFailedTests_DoesNotReprintErroredAssemblyRecap() + { + var stringBuilderConsole = new StringBuilderConsole(); + TerminalTestReporter terminalReporter = CreateOrchestratorReporter(stringBuilderConsole, showAssemblyStartAndComplete: false); + terminalReporter.TestExecutionStarted(DateTimeOffset.MinValue, workerCount: 1, isDiscovery: false, isHelp: false, isRetry: false); + + string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\repo\Failing.dll" : "/repo/Failing.dll"; + terminalReporter.AssemblyRunStarted(assembly, "net9.0", "x64", "exec-1", "inst-1"); + ReportOrchestratorTest(terminalReporter, assembly, "exec-1", "inst-1", "t-1", TestOutcome.Fail); + + terminalReporter.AssemblyRunCompleted("exec-1", exitCode: 1, outputData: null, errorData: null); + terminalReporter.TestExecutionCompleted(DateTimeOffset.MaxValue, exitCode: 1); + + // No "Errored assemblies:" recap: the failure is already reported through the failed test. + Assert.DoesNotContain(TerminalResources.ErroredAssembliesHeader, stringBuilderConsole.Output); + } + [TestMethod] public void AssemblyRunStarted_AfterRetry_RendersLatestAttemptCounts() {