Re-print errored assemblies in dotnet test end-of-run recap

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalResources.cs

@@ -53,6 +53,8 @@ internal static partial class TerminalResources
 
     internal static string @Error => GetResourceString("Error");
 
+    internal static string @ErroredAssembliesHeader => GetResourceString("ErroredAssembliesHeader");
+
     internal static string @ExitCode => GetResourceString("ExitCode");
 
     internal static string @Expected => GetResourceString("Expected");

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalResources.resx

@@ -283,6 +283,9 @@ Valid values are 'All', 'Failed', 'None'. Default is 'All' (or 'Failed' when an
   <data name="HandshakeFailuresHeader" xml:space="preserve">
     <value>Handshake failures:</value>
   </data>
+  <data name="ErroredAssembliesHeader" xml:space="preserve">
+    <value>Errored assemblies:</value>
+  </data>
   <data name="ExitCode" xml:space="preserve">
     <value>Exit code</value>
   </data>

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.ErroredAssemblies.cs

@@ -0,0 +1,75 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.Testing.Platform.OutputDevice.Terminal;
+
+[UnsupportedOSPlatform("browser")]
+internal sealed partial class TerminalTestReporter
+{
+#if NET9_0_OR_GREATER
+    private readonly Lock _erroredAssembliesLock = new();
+#else
+    private readonly object _erroredAssembliesLock = new();
+#endif
+
+    /// <summary>
+    /// Assemblies that handshaked but then exited with a non-zero code without reporting any failed test (crash,
+    /// <c>Environment.FailFast</c>, a hang-dump kill, an option rejected after the handshake, ...). Their process
+    /// output is printed inline when the assembly completes, but in a large multi-assembly run that inline output is
+    /// scrolled far above the summary, so we also record it here and re-print it in an end-of-run recap.
+    /// </summary>
+    private readonly List<ErroredAssemblyRecord> _erroredAssemblies = [];
+
+    /// <summary>
+    /// Records an assembly that ended unsuccessfully with no failed test so its process output (exit code + stdout +
+    /// stderr) can be re-printed at the end of the run. Orchestrator-only: only the multi-process <c>dotnet test</c>
+    /// orchestrator reaches the exit-code overload of <see cref="AssemblyRunCompleted(string, int, string?, string?)"/>.
+    /// </summary>
+    private void RecordErroredAssembly(TestProgressState assemblyRun, int exitCode, string? outputData, string? errorData)
+    {
+        lock (_erroredAssembliesLock)
+        {
+            _erroredAssemblies.Add(new ErroredAssemblyRecord(assemblyRun.Assembly, assemblyRun.TargetFramework, assemblyRun.Architecture, exitCode, outputData, errorData));
+        }
+    }
+
+    /// <summary>
+    /// Re-print assemblies that errored during the run (non-zero exit with no failed test) so that — as with the
+    /// handshake-failure recap — the actionable process output is shown at the end of the run rather than buried in
+    /// the middle of a large multi-assembly run.
+    /// </summary>
+    private void AppendErroredAssemblyRecap(ITerminal terminal)
+    {
+        ErroredAssemblyRecord[] errored;
+        lock (_erroredAssembliesLock)
+        {
+            if (_erroredAssemblies.Count == 0)
+            {
+                return;
+            }
+
+            errored = _erroredAssemblies.ToArray();
+        }
+
+        terminal.AppendLine();
+        terminal.SetColor(TerminalColor.DarkRed);
+        terminal.AppendLine(TerminalResources.ErroredAssembliesHeader);
+        terminal.ResetColor();
+
+        foreach (ErroredAssemblyRecord failure in errored)
+        {
+            terminal.Append(SingleIndentation);
+            AppendAssemblyLinkTargetFrameworkAndArchitecture(terminal, failure.AssemblyPath, failure.TargetFramework, failure.Architecture);
+            terminal.AppendLine();
+            AppendExecutableSummary(terminal, failure.ExitCode, failure.OutputData, failure.ErrorData);
+        }
+    }
+
+    private readonly record struct ErroredAssemblyRecord(
+        string AssemblyPath,
+        string? TargetFramework,
+        string? Architecture,
+        int ExitCode,
+        string? OutputData,
+        string? ErrorData);
+}

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs

@@ -123,6 +123,16 @@ internal void AssemblyRunCompleted(string executionId, int exitCode, string? out
         }
 
         _terminalWithProgress.WriteToTerminal(terminal => AppendExecutableSummary(terminal, exitCode, outputData, errorData));
+
+        // A non-zero exit with no failed test is a process-level error (crash, FailFast, hang-dump kill, an option
+        // rejected after the handshake, ...) whose inline output above is easily lost in a large run. Record it so
+        // it is re-printed in the end-of-run recap (see AppendErroredAssemblyRecap). Assemblies that exit non-zero
+        // *because* tests failed are intentionally excluded: those failures are already reported per-test, and this
+        // set mirrors the "error: N" count in the summary (failedAssembliesWithoutFailedTests).
+        if (assemblyRun.FailedTests == 0)
+        {
+            RecordErroredAssembly(assemblyRun, exitCode, outputData, errorData);
+        }
     }
 
     public void TestExecutionCompleted(DateTimeOffset endTime, int? exitCode)
@@ -137,8 +147,8 @@ public void TestExecutionCompleted(DateTimeOffset endTime, int? exitCode)
 
         // This is relevant for HotReload scenarios. We want the next test sessions to start fresh, so we reset all
         // per-run state here (after the summary above has consumed it): the per-assembly runs, the collected
-        // artifacts, the handshake failures, and the cancellation flag. Otherwise a later session would re-print the
-        // previous session's artifacts/handshake failures or stay stuck in the aborted state.
+        // artifacts, the handshake failures, the errored assemblies, and the cancellation flag. Otherwise a later
+        // session would re-print the previous session's artifacts/failures or stay stuck in the aborted state.
         _assemblies.Clear();
         _artifacts.Clear();
         WasCancelled = false;
@@ -147,6 +157,11 @@ public void TestExecutionCompleted(DateTimeOffset endTime, int? exitCode)
             _handshakeFailures.Clear();
         }
 
+        lock (_erroredAssembliesLock)
+        {
+            _erroredAssemblies.Clear();
+        }
+
         _testExecutionStartTime = null;
         _testExecutionEndTime = null;
     }

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Summary.cs

@@ -203,6 +203,10 @@ private void AppendTestRunSummary(ITerminal terminal)
         // Re-print any handshake failures (orchestrator-only) at the very end so they aren't lost above the summary.
         // No-op for the in-process host, which never reports handshake failures.
         AppendHandshakeFailureRecap(terminal);
+
+        // Re-print any assemblies that errored (non-zero exit with no failed test) for the same reason: the inline
+        // process output is otherwise buried in the middle of a large run. No-op for the in-process host.
+        AppendErroredAssemblyRecap(terminal);
     }
 
     /// <summary>

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.cs.xlf

@@ -57,6 +57,11 @@
         <target state="new">error</target>
         <note>Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'.</note>
       </trans-unit>
+      <trans-unit id="ErroredAssembliesHeader">
+        <source>Errored assemblies:</source>
+        <target state="new">Errored assemblies:</target>
+        <note />
+      </trans-unit>
       <trans-unit id="ExitCode">
         <source>Exit code</source>
         <target state="new">Exit code</target>

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.de.xlf

@@ -57,6 +57,11 @@
         <target state="new">error</target>
         <note>Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'.</note>
       </trans-unit>
+      <trans-unit id="ErroredAssembliesHeader">
+        <source>Errored assemblies:</source>
+        <target state="new">Errored assemblies:</target>
+        <note />
+      </trans-unit>
       <trans-unit id="ExitCode">
         <source>Exit code</source>
         <target state="new">Exit code</target>

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.es.xlf

@@ -57,6 +57,11 @@
         <target state="new">error</target>
         <note>Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'.</note>
       </trans-unit>
+      <trans-unit id="ErroredAssembliesHeader">
+        <source>Errored assemblies:</source>
+        <target state="new">Errored assemblies:</target>
+        <note />
+      </trans-unit>
       <trans-unit id="ExitCode">
         <source>Exit code</source>
         <target state="new">Exit code</target>

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.fr.xlf

@@ -57,6 +57,11 @@
         <target state="new">error</target>
         <note>Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'.</note>
       </trans-unit>
+      <trans-unit id="ErroredAssembliesHeader">
+        <source>Errored assemblies:</source>
+        <target state="new">Errored assemblies:</target>
+        <note />
+      </trans-unit>
       <trans-unit id="ExitCode">
         <source>Exit code</source>
         <target state="new">Exit code</target>

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.it.xlf

@@ -57,6 +57,11 @@
         <target state="new">error</target>
         <note>Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'.</note>
       </trans-unit>
+      <trans-unit id="ErroredAssembliesHeader">
+        <source>Errored assemblies:</source>
+        <target state="new">Errored assemblies:</target>
+        <note />
+      </trans-unit>
       <trans-unit id="ExitCode">
         <source>Exit code</source>
         <target state="new">Exit code</target>

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ja.xlf

@@ -57,6 +57,11 @@
         <target state="new">error</target>
         <note>Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'.</note>
       </trans-unit>
+      <trans-unit id="ErroredAssembliesHeader">
+        <source>Errored assemblies:</source>
+        <target state="new">Errored assemblies:</target>
+        <note />
+      </trans-unit>
       <trans-unit id="ExitCode">
         <source>Exit code</source>
         <target state="new">Exit code</target>

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ko.xlf

@@ -57,6 +57,11 @@
         <target state="new">error</target>
         <note>Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'.</note>
       </trans-unit>
+      <trans-unit id="ErroredAssembliesHeader">
+        <source>Errored assemblies:</source>
+        <target state="new">Errored assemblies:</target>
+        <note />
+      </trans-unit>
       <trans-unit id="ExitCode">
         <source>Exit code</source>
         <target state="new">Exit code</target>

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.pl.xlf

@@ -57,6 +57,11 @@
         <target state="new">error</target>
         <note>Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'.</note>
       </trans-unit>
+      <trans-unit id="ErroredAssembliesHeader">
+        <source>Errored assemblies:</source>
+        <target state="new">Errored assemblies:</target>
+        <note />
+      </trans-unit>
       <trans-unit id="ExitCode">
         <source>Exit code</source>
         <target state="new">Exit code</target>

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.pt-BR.xlf

@@ -57,6 +57,11 @@
         <target state="new">error</target>
         <note>Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'.</note>
       </trans-unit>
+      <trans-unit id="ErroredAssembliesHeader">
+        <source>Errored assemblies:</source>
+        <target state="new">Errored assemblies:</target>
+        <note />
+      </trans-unit>
       <trans-unit id="ExitCode">
         <source>Exit code</source>
         <target state="new">Exit code</target>

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.ru.xlf

@@ -57,6 +57,11 @@
         <target state="new">error</target>
         <note>Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'.</note>
       </trans-unit>
+      <trans-unit id="ErroredAssembliesHeader">
+        <source>Errored assemblies:</source>
+        <target state="new">Errored assemblies:</target>
+        <note />
+      </trans-unit>
       <trans-unit id="ExitCode">
         <source>Exit code</source>
         <target state="new">Exit code</target>

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.tr.xlf

@@ -57,6 +57,11 @@
         <target state="new">error</target>
         <note>Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'.</note>
       </trans-unit>
+      <trans-unit id="ErroredAssembliesHeader">
+        <source>Errored assemblies:</source>
+        <target state="new">Errored assemblies:</target>
+        <note />
+      </trans-unit>
       <trans-unit id="ExitCode">
         <source>Exit code</source>
         <target state="new">Exit code</target>

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.zh-Hans.xlf

@@ -57,6 +57,11 @@
         <target state="new">error</target>
         <note>Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'.</note>
       </trans-unit>
+      <trans-unit id="ErroredAssembliesHeader">
+        <source>Errored assemblies:</source>
+        <target state="new">Errored assemblies:</target>
+        <note />
+      </trans-unit>
       <trans-unit id="ExitCode">
         <source>Exit code</source>
         <target state="new">Exit code</target>

src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/xlf/TerminalResources.zh-Hant.xlf

@@ -57,6 +57,11 @@
         <target state="new">error</target>
         <note>Label for the count of assemblies that failed without producing a failed test (crash / non-zero exit / handshake failure) in the run summary, e.g. 'error: 1'.</note>
       </trans-unit>
+      <trans-unit id="ErroredAssembliesHeader">
+        <source>Errored assemblies:</source>
+        <target state="new">Errored assemblies:</target>
+        <note />
+      </trans-unit>
       <trans-unit id="ExitCode">
         <source>Exit code</source>
         <target state="new">Exit code</target>

test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs

@@ -1755,6 +1755,61 @@ public void TestExecutionCompleted_WhenHandshakeFailures_PrintsRecapAndFailsRun(
         Assert.Contains($"{TerminalResources.TestRunSummary} {TerminalResources.Failed}!", output);
     }
 
+    // dotnet/sdk#51952: an assembly that DOES handshake but then exits non-zero without any failed test (a crash,
+    // Environment.FailFast, a hang-dump kill, an option rejected after the handshake, ...) has its process output
+    // printed inline when it completes, which is easily lost in the middle of a large multi-assembly run. The
+    // end-of-run summary must re-print an "Errored assemblies:" recap with the captured output so it is discoverable.
+    [TestMethod]
+    public void TestExecutionCompleted_WhenAssemblyErroredWithoutFailedTests_ReprintsErroredAssemblyRecap()
+    {
+        var stringBuilderConsole = new StringBuilderConsole();
+        TerminalTestReporter terminalReporter = CreateOrchestratorReporter(stringBuilderConsole, showAssemblyStartAndComplete: false);
+        terminalReporter.TestExecutionStarted(DateTimeOffset.MinValue, workerCount: 1, isDiscovery: false, isHelp: false, isRetry: false);
+
+        string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\repo\Crashy.dll" : "/repo/Crashy.dll";
+        terminalReporter.AssemblyRunStarted(assembly, "net9.0", "x64", "exec-1", "inst-1");
+        ReportOrchestratorTest(terminalReporter, assembly, "exec-1", "inst-1", "t-1", TestOutcome.Passed);
+
+        // The single test passed but the process exits non-zero -> error category (FailedTests == 0).
+        terminalReporter.AssemblyRunCompleted("exec-1", exitCode: 42, outputData: "boom stdout", errorData: "boom stderr");
+
+        // It is NOT a handshake failure (the assembly was registered).
+        Assert.IsFalse(terminalReporter.HasHandshakeFailure);
+
+        terminalReporter.TestExecutionCompleted(DateTimeOffset.MaxValue, exitCode: 42);
+
+        string output = stringBuilderConsole.Output;
+
+        // The end-of-run recap header is printed and identifies the errored assembly with its captured output.
+        Assert.Contains(TerminalResources.ErroredAssembliesHeader, output);
+        string recap = output[output.IndexOf(TerminalResources.ErroredAssembliesHeader, StringComparison.Ordinal)..];
+        Assert.Contains("Crashy.dll", recap);
+        Assert.Contains($"{TerminalResources.ExitCode}: 42", recap);
+        Assert.Contains("boom stdout", recap);
+        Assert.Contains("boom stderr", recap);
+    }
+
+    // Guard for the recap's scope: an assembly that exits non-zero BECAUSE a test failed must not be added to the
+    // "Errored assemblies:" recap - those failures are already reported per-test, and re-dumping every failing
+    // assembly's process output at the end would be noise. The recap is reserved for unexplained process errors.
+    [TestMethod]
+    public void TestExecutionCompleted_WhenAssemblyExitedNonZeroWithFailedTests_DoesNotReprintErroredAssemblyRecap()
+    {
+        var stringBuilderConsole = new StringBuilderConsole();
+        TerminalTestReporter terminalReporter = CreateOrchestratorReporter(stringBuilderConsole, showAssemblyStartAndComplete: false);
+        terminalReporter.TestExecutionStarted(DateTimeOffset.MinValue, workerCount: 1, isDiscovery: false, isHelp: false, isRetry: false);
+
+        string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\repo\Failing.dll" : "/repo/Failing.dll";
+        terminalReporter.AssemblyRunStarted(assembly, "net9.0", "x64", "exec-1", "inst-1");
+        ReportOrchestratorTest(terminalReporter, assembly, "exec-1", "inst-1", "t-1", TestOutcome.Fail);
+
+        terminalReporter.AssemblyRunCompleted("exec-1", exitCode: 1, outputData: null, errorData: null);
+        terminalReporter.TestExecutionCompleted(DateTimeOffset.MaxValue, exitCode: 1);
+
+        // No "Errored assemblies:" recap: the failure is already reported through the failed test.
+        Assert.DoesNotContain(TerminalResources.ErroredAssembliesHeader, stringBuilderConsole.Output);
+    }
+
     [TestMethod]
     public void AssemblyRunStarted_AfterRetry_RendersLatestAttemptCounts()
     {