diff --git a/src/Platform/Microsoft.Testing.Platform/CommandLine/PlatformCommandLineProvider.cs b/src/Platform/Microsoft.Testing.Platform/CommandLine/PlatformCommandLineProvider.cs index 812bd7b713..ddfaab74d2 100644 --- a/src/Platform/Microsoft.Testing.Platform/CommandLine/PlatformCommandLineProvider.cs +++ b/src/Platform/Microsoft.Testing.Platform/CommandLine/PlatformCommandLineProvider.cs @@ -22,6 +22,10 @@ internal sealed class PlatformCommandLineProvider : ICommandLineOptionsProvider public const string NoBannerOptionKey = "no-banner"; public const string SkipBuildersNumberCheckOptionKey = "internal-testingplatform-skipbuildercheck"; public const string DiscoverTestsOptionKey = "list-tests"; + public const string DiscoverTestsJsonArgument = "json"; + public const string DiscoverTestsTextArgument = "text"; + + private static readonly string SupportedDiscoverTestsValues = $"'{DiscoverTestsTextArgument}', '{DiscoverTestsJsonArgument}'"; public const string ResultDirectoryOptionKey = "results-directory"; public const string IgnoreExitCodeOptionKey = "ignore-exit-code"; public const string MinimumExpectedTestsOptionKey = "minimum-expected-tests"; @@ -55,7 +59,7 @@ internal sealed class PlatformCommandLineProvider : ICommandLineOptionsProvider new(DiagnosticVerbosityOptionKey, PlatformResources.PlatformCommandLineDiagnosticVerbosityOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), new(DiagnosticFileLoggerSynchronousWriteOptionKey, PlatformResources.PlatformCommandLineDiagnosticFileLoggerSynchronousWriteOptionDescription, ArgumentArity.Zero, false, isBuiltIn: true), MinimumExpectedTests, - new(DiscoverTestsOptionKey, PlatformResources.PlatformCommandLineDiscoverTestsOptionDescription, ArgumentArity.Zero, false, isBuiltIn: true), + new(DiscoverTestsOptionKey, PlatformResources.PlatformCommandLineDiscoverTestsOptionDescription, ArgumentArity.ZeroOrOne, false, isBuiltIn: true), new(IgnoreExitCodeOptionKey, PlatformResources.PlatformCommandLineIgnoreExitCodeOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), new(ExitOnProcessExitOptionKey, PlatformResources.PlatformCommandLineExitOnProcessExitOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), new(ConfigFileOptionKey, PlatformResources.PlatformCommandLineConfigFileOptionDescription, ArgumentArity.ExactlyOne, false, isBuiltIn: true), @@ -101,6 +105,14 @@ public Task ValidateOptionArgumentsAsync(CommandLineOption com } } + if (commandOption.Name == DiscoverTestsOptionKey + && arguments.Length == 1 + && !DiscoverTestsJsonArgument.Equals(arguments[0], StringComparison.OrdinalIgnoreCase) + && !DiscoverTestsTextArgument.Equals(arguments[0], StringComparison.OrdinalIgnoreCase)) + { + return ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, PlatformResources.PlatformCommandLineDiscoverTestsInvalidArgument, arguments[0], SupportedDiscoverTestsValues)); + } + if (commandOption.Name == ClientPortOptionKey && (!int.TryParse(arguments[0], out int _))) { return ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, PlatformResources.PlatformCommandLinePortOptionSingleArgument, ClientPortOptionKey)); @@ -156,6 +168,11 @@ public static int GetMinimumExpectedTests(ICommandLineOptions commandLineOptions return int.Parse(arguments[0], CultureInfo.InvariantCulture); } + public static bool IsListTestsJsonOutput(ICommandLineOptions commandLineOptions) + => commandLineOptions.TryGetOptionArgumentList(DiscoverTestsOptionKey, out string[]? arguments) + && arguments is { Length: 1 } + && DiscoverTestsJsonArgument.Equals(arguments[0], StringComparison.OrdinalIgnoreCase); + private static Task IsMinimumExpectedTestsOptionValidAsync(CommandLineOption option, string[] arguments) => option.Name == MinimumExpectedTestsOptionKey && (arguments.Length != 1 || !int.TryParse(arguments[0], out int value) || value == 0) diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/DiscoveredTestsJsonSerializer.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/DiscoveredTestsJsonSerializer.cs new file mode 100644 index 0000000000..490e303669 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/DiscoveredTestsJsonSerializer.cs @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Extensions.Messages; + +namespace Microsoft.Testing.Platform.OutputDevice; + +/// +/// Serializes the result of --list-tests json into a single JSON document with as much +/// information as is available for each . +/// +/// +/// Schema history (bump on any breaking change): +/// +/// +/// +/// Initial schema. Top-level: schemaVersion (int), tests (array of test +/// objects). Per test: uid, displayName, plus optional type +/// (assembly / namespace / type / method / arity / params / return — from +/// ), location (file path, start line, +/// end line — from ), traits (array of +/// { key, value } from ), properties +/// (array of { key, value } from +/// ; array — not object — so +/// duplicate keys survive serialization). Absent fields are omitted; +/// type.namespace is omitted for the global namespace. +/// +/// +/// +/// +internal static class DiscoveredTestsJsonSerializer +{ + /// + /// Schema version of the produced JSON document. Increment when introducing a breaking change. + /// + internal const int SchemaVersion = 1; + + /// + /// Serializes the discovered tests to a pretty-printed JSON document. + /// + /// The discovered tests. + /// The JSON document. + public static string Serialize(IEnumerable tests) + { + var writer = new JsonStringWriter(); + writer.WriteStartObject(); + writer.WriteNumber("schemaVersion", SchemaVersion); + writer.WriteStartArray("tests"); + foreach (TestNode test in tests) + { + WriteTestNode(writer, test); + } + + writer.WriteEndArray(); + writer.WriteEndObject(); + return writer.ToString(); + } + + private static void WriteTestNode(JsonStringWriter writer, TestNode test) + { + writer.WriteStartObject(); + writer.WriteString("uid", test.Uid.Value); + writer.WriteString("displayName", test.DisplayName); + + TestMethodIdentifierProperty? methodIdentifier = test.Properties.SingleOrDefault(); + if (methodIdentifier is not null) + { + writer.WriteStartObject("type"); + writer.WriteString("assemblyFullName", methodIdentifier.AssemblyFullName); + if (!RoslynString.IsNullOrEmpty(methodIdentifier.Namespace)) + { + writer.WriteString("namespace", methodIdentifier.Namespace); + } + + writer.WriteString("typeName", methodIdentifier.TypeName); + writer.WriteString("methodName", methodIdentifier.MethodName); + writer.WriteNumber("methodArity", methodIdentifier.MethodArity); + writer.WriteString("returnTypeFullName", methodIdentifier.ReturnTypeFullName); + writer.WriteStartArray("parameterTypeFullNames"); + foreach (string parameterFullName in methodIdentifier.ParameterTypeFullNames) + { + writer.WriteStringValue(parameterFullName); + } + + writer.WriteEndArray(); + writer.WriteEndObject(); + } + + TestFileLocationProperty? fileLocation = test.Properties.SingleOrDefault(); + if (fileLocation is not null) + { + writer.WriteStartObject("location"); + writer.WriteString("file", fileLocation.FilePath); + writer.WriteNumber("lineStart", fileLocation.LineSpan.Start.Line); + writer.WriteNumber("lineEnd", fileLocation.LineSpan.End.Line); + writer.WriteEndObject(); + } + + TestMetadataProperty[] traits = test.Properties.OfType(); + if (traits.Length > 0) + { + // PropertyBag stores properties as a linked list and prepends on Add, so OfType walks + // them in reverse insertion order. Reverse here so the JSON reflects the order in + // which the adapter recorded the traits, which is what consumers will reasonably + // expect and what makes diffs stable across runs. + Array.Reverse(traits); + writer.WriteStartArray("traits"); + foreach (TestMetadataProperty trait in traits) + { + writer.WriteStartObject(); + writer.WriteString("key", trait.Key); + writer.WriteString("value", trait.Value); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + + SerializableKeyValuePairStringProperty[] kvps = test.Properties.OfType(); + if (kvps.Length > 0) + { + // Same rationale as traits above: reverse so the JSON reflects insertion order. + // Emit as an array of {key, value} (mirroring traits) so duplicate keys — which + // PropertyBag allows — survive serialization. A JSON object would silently collapse them. + Array.Reverse(kvps); + writer.WriteStartArray("properties"); + foreach (SerializableKeyValuePairStringProperty kvp in kvps) + { + writer.WriteStartObject(); + writer.WriteString("key", kvp.Key); + writer.WriteString("value", kvp.Value); + writer.WriteEndObject(); + } + + writer.WriteEndArray(); + } + + writer.WriteEndObject(); + } +} diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/JsonStringWriter.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/JsonStringWriter.cs new file mode 100644 index 0000000000..45ff13c858 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/JsonStringWriter.cs @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.OutputDevice; + +/// +/// Minimal pretty-printing JSON writer used to serialize the --list-tests json document. +/// Avoids depending on either System.Text.Json (only available on .NET) or Jsonite +/// (only compiled on netstandard2.0) so the same code can run on every supported target. +/// +internal sealed class JsonStringWriter +{ + private readonly StringBuilder _builder = new(); + private readonly Stack _stack = new(); + + public void WriteStartObject(string? name = null) => WriteStartContainer(name, ContainerKind.Object); + + public void WriteEndObject() => WriteEndContainer(ContainerKind.Object); + + public void WriteStartArray(string name) => WriteStartContainer(name, ContainerKind.Array); + + public void WriteEndArray() => WriteEndContainer(ContainerKind.Array); + + public void WriteString(string name, string? value) + { + WritePropertyHead(name); + WriteEscapedString(value); + } + + public void WriteStringValue(string? value) + { + WriteValueHead(); + WriteEscapedString(value); + } + + public void WriteNumber(string name, int value) + { + WritePropertyHead(name); + _builder.Append(value.ToString(CultureInfo.InvariantCulture)); + } + + public override string ToString() => _builder.ToString(); + + private void WriteStartContainer(string? name, ContainerKind kind) + { + if (name is null) + { + WriteValueHead(); + } + else + { + WritePropertyHead(name); + } + + _builder.Append(kind == ContainerKind.Object ? '{' : '['); + _stack.Push(new ContainerState(kind)); + } + + private void WriteEndContainer(ContainerKind kind) + { + ContainerState state = _stack.Pop(); + if (state.Kind != kind) + { + throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "JSON writer mismatch: expected to close {0} but got {1}.", state.Kind, kind)); + } + + if (state.HasMembers) + { + _builder.Append('\n'); + AppendIndent(); + } + + _builder.Append(kind == ContainerKind.Object ? '}' : ']'); + } + + private void WritePropertyHead(string name) + { + WriteSeparatorAndIndent(); + WriteEscapedString(name); + _builder.Append(": "); + } + + private void WriteValueHead() => WriteSeparatorAndIndent(); + + private void WriteSeparatorAndIndent() + { + if (_stack.Count > 0) + { + ContainerState top = _stack.Peek(); + if (top.HasMembers) + { + _builder.Append(','); + } + + top.HasMembers = true; + } + + _builder.Append('\n'); + AppendIndent(); + } + + private void AppendIndent() => _builder.Append(' ', _stack.Count * 2); + + private void WriteEscapedString(string? value) + { + if (value is null) + { + _builder.Append("null"); + return; + } + + _builder.Append('"'); + for (int i = 0; i < value.Length; i++) + { + char c = value[i]; + switch (c) + { + case '\\': + _builder.Append("\\\\"); + break; + case '"': + _builder.Append("\\\""); + break; + case '\b': + _builder.Append("\\b"); + break; + case '\f': + _builder.Append("\\f"); + break; + case '\n': + _builder.Append("\\n"); + break; + case '\r': + _builder.Append("\\r"); + break; + case '\t': + _builder.Append("\\t"); + break; + case '\u2028': + case '\u2029': + // Valid JSON but invalid in JavaScript string literals (legacy eval() concern); + // System.Text.Json escapes these by default so we mirror that. + AppendUnicodeEscape(c); + break; + default: + if (c < 0x20) + { + AppendUnicodeEscape(c); + } + else if (char.IsHighSurrogate(c)) + { + if (i + 1 < value.Length && char.IsLowSurrogate(value[i + 1])) + { + _builder.Append(c); + _builder.Append(value[++i]); + } + else + { + _builder.Append('\uFFFD'); + } + } + else if (char.IsLowSurrogate(c)) + { + // Unpaired low surrogate. + _builder.Append('\uFFFD'); + } + else + { + _builder.Append(c); + } + + break; + } + } + + _builder.Append('"'); + } + + private void AppendUnicodeEscape(char c) + => _builder.Append("\\u").Append(((int)c).ToString("x4", CultureInfo.InvariantCulture)); + + private enum ContainerKind + { + Object, + Array, + } + + private sealed class ContainerState(ContainerKind kind) + { + public ContainerKind Kind { get; } = kind; + + public bool HasMembers { get; set; } + } +} diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs index fbc6fc7abf..a3563ec9a2 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.cs @@ -52,9 +52,16 @@ internal sealed partial class TerminalOutputDevice : IHotReloadPlatformOutputDev private readonly string? _targetFramework; private readonly string _assemblyName; + // Buffer for discovered test nodes when --list-tests json is active. Writes happen only from + // the message bus pump in ConsumeAsync (single-producer per consumer) and are fully drained + // by the platform before DisplayAfterSessionEndRunInternalAsync runs, so no extra locking is + // required. The list stays empty (and effectively unused) outside JSON mode. + private readonly List _discoveredTestsForJson = []; + private TerminalTestReporter? _terminalTestReporter; private bool _bannerDisplayed; private bool _isListTests; + private bool _isListTestsJson; private bool _isServerMode; private ILogger? _logger; private TestProcessRole? _processRole; @@ -105,19 +112,33 @@ public TerminalOutputDevice( public async Task InitializeAsync() { - await _policiesService.RegisterOnAbortCallbackAsync( - () => - { - _terminalTestReporter?.StartCancelling(); - return Task.CompletedTask; - }).ConfigureAwait(false); - if (_fileLoggerInformation is not null) { _logger = _loggerFactory.CreateLogger(GetType().ToString()); } _isListTests = _commandLineOptions.IsOptionSet(PlatformCommandLineProvider.DiscoverTestsOptionKey); + _isListTestsJson = PlatformCommandLineProvider.IsListTestsJsonOutput(_commandLineOptions); + + if (_isListTestsJson) + { + // In JSON discovery mode the standard abort callback would call + // TerminalTestReporter.StartCancelling which writes "Cancelling test session" to + // stdout, corrupting the JSON document. Route a single-line cancellation notice + // to stderr instead so the user still gets feedback on Ctrl+C. + await _policiesService.RegisterOnAbortCallbackAsync( + () => WriteToStandardErrorAsync(PlatformResources.CancellingTestSession)).ConfigureAwait(false); + } + else + { + await _policiesService.RegisterOnAbortCallbackAsync( + () => + { + _terminalTestReporter?.StartCancelling(); + return Task.CompletedTask; + }).ConfigureAwait(false); + } + _isServerMode = _commandLineOptions.IsOptionSet(PlatformCommandLineProvider.ServerOptionKey); bool noAnsi = _commandLineOptions.IsOptionSet(TerminalTestReporterCommandLineOptionsProvider.NoAnsiOption); @@ -230,10 +251,26 @@ private async Task LogDebugAsync(string message) } } + // Sole point that bypasses IConsole to reach stderr; used only by --list-tests json paths + // so stdout stays reserved for the JSON document while errors and the cancellation notice + // still surface somewhere. If IConsole ever grows a stderr abstraction, replace this helper. + private static async Task WriteToStandardErrorAsync(string message) + => await Console.Error.WriteLineAsync(message).ConfigureAwait(false); + public async Task DisplayBannerAsync(string? bannerMessage, CancellationToken cancellationToken) { RoslynDebug.Assert(_terminalTestReporter is not null); + if (_isListTestsJson) + { + // Machine-readable mode: keep stdout clean, suppress banner & file logger notice. + // The env var propagates to any child test host the controller spawns so they also + // stay quiet — important because the JSON document must be the sole stdout content. + _bannerDisplayed = true; + _environment.SetEnvironmentVariable(TESTINGPLATFORM_CONSOLEOUTPUTDEVICE_SKIP_BANNER, "1"); + return; + } + using (await _asyncMonitor.LockAsync(TimeoutHelper.DefaultHangTimeSpanTimeout).ConfigureAwait(false)) { if (!_bannerDisplayed && !_isServerMode) @@ -298,7 +335,7 @@ public async Task DisplayBeforeHotReloadSessionStartAsync(CancellationToken canc public async Task DisplayBeforeSessionStartAsync(CancellationToken cancellationToken) { - if (_isServerMode) + if (_isServerMode || _isListTestsJson) { return; } @@ -316,7 +353,18 @@ public async Task DisplayBeforeSessionStartAsync(CancellationToken cancellationT } public async Task DisplayAfterHotReloadSessionEndAsync(CancellationToken cancellationToken) - => await DisplayAfterSessionEndRunInternalAsync().ConfigureAwait(false); + { + if (_isListTestsJson) + { + // JSON discovery is a point-in-time snapshot. Re-emitting after every hot-reload + // cycle would produce multiple growing JSON documents on stdout, which would break + // any consumer that pipes the output (the accumulated _discoveredTestsForJson buffer + // would also re-include earlier tests every cycle). + return; + } + + await DisplayAfterSessionEndRunInternalAsync().ConfigureAwait(false); + } public async Task DisplayAfterSessionEndRunAsync(CancellationToken cancellationToken) { @@ -339,6 +387,19 @@ private async Task DisplayAfterSessionEndRunInternalAsync() { RoslynDebug.Assert(_terminalTestReporter is not null); + if (_isListTestsJson) + { + using (await _asyncMonitor.LockAsync(TimeoutHelper.DefaultHangTimeSpanTimeout).ConfigureAwait(false)) + { + if (_processRole == TestProcessRole.TestHost) + { + _console.WriteLine(DiscoveredTestsJsonSerializer.Serialize(_discoveredTestsForJson)); + } + } + + return; + } + using (await _asyncMonitor.LockAsync(TimeoutHelper.DefaultHangTimeSpanTimeout).ConfigureAwait(false)) { if (_processRole == TestProcessRole.TestHost) @@ -363,6 +424,29 @@ public async Task DisplayAsync(IOutputDeviceDataProducer producer, IOutputDevice { RoslynDebug.Assert(_terminalTestReporter is not null); + if (_isListTestsJson) + { + // Machine-readable mode: keep stdout reserved for the JSON document so consumers can + // pipe it directly. Errors and exceptions still need surfacing somewhere, so route + // them to stderr via WriteToStandardErrorAsync (the only place that bypasses IConsole, + // which does not abstract stderr today). Warnings and informational text are dropped + // to keep stdout strictly JSON. + switch (data) + { + case ErrorMessageOutputDeviceData errorData: + await LogDebugAsync(errorData.Message).ConfigureAwait(false); + await WriteToStandardErrorAsync(errorData.Message).ConfigureAwait(false); + break; + + case ExceptionOutputDeviceData exceptionData: + await LogDebugAsync(exceptionData.Exception.ToString()).ConfigureAwait(false); + await WriteToStandardErrorAsync(exceptionData.Exception.ToString()).ConfigureAwait(false); + break; + } + + return; + } + using (await _asyncMonitor.LockAsync(TimeoutHelper.DefaultHangTimeSpanTimeout).ConfigureAwait(false)) { switch (data) @@ -404,6 +488,18 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo return Task.CompletedTask; } + if (_isListTestsJson) + { + // Machine-readable mode: only buffer discovered tests, do not write anything to the terminal. + if (value is TestNodeUpdateMessage testNodeUpdate + && testNodeUpdate.TestNode.Properties.SingleOrDefault() is DiscoveredTestNodeStateProperty) + { + _discoveredTestsForJson.Add(testNodeUpdate.TestNode); + } + + return Task.CompletedTask; + } + switch (value) { case TestNodeUpdateMessage testNodeStateChanged: diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.cs b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.cs index ec217bf570..ad7386dab0 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.cs +++ b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.cs @@ -39,6 +39,8 @@ internal static partial class PlatformResources #if IS_MTP_UNIT_TESTS internal static string @PlatformCommandLineDiagnosticOptionExpectsSingleArgumentErrorMessage => GetResourceString("PlatformCommandLineDiagnosticOptionExpectsSingleArgumentErrorMessage"); + internal static string @PlatformCommandLineDiscoverTestsInvalidArgument => GetResourceString("PlatformCommandLineDiscoverTestsInvalidArgument"); + internal static string @PlatformCommandLineExitOnProcessExitSingleArgument => GetResourceString("PlatformCommandLineExitOnProcessExitSingleArgument"); internal static string @PlatformCommandLineDiagnosticOptionIsMissing => GetResourceString("PlatformCommandLineDiagnosticOptionIsMissing"); diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx index 2b1b96918f..988594b3f1 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx +++ b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx @@ -385,7 +385,11 @@ If not specified the file will be generated inside the default 'TestResults' dir The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', and 'Critical'. - List available tests. + List available tests. +Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. + + + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. Show the command line help. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf index eb50d2d177..e6a1a43bab 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf @@ -511,9 +511,15 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an Dostupné hodnoty jsou Trace, Debug, Information, Warning, Error a Critical. + + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + + - List available tests. - Umožňuje zobrazit seznam dostupných testů. + List available tests. +Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. + Umožňuje zobrazit seznam dostupných testů. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf index 95a59bc488..b27adab814 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf @@ -511,9 +511,15 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an Die verfügbaren Werte sind "Trace", "Debug", "Information", "Warning", "Error" und "Critical". + + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + + - List available tests. - Listen Sie verfügbare Tests auf. + List available tests. +Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. + Listen Sie verfügbare Tests auf. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf index 7498efda2d..54986963d0 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf @@ -511,9 +511,15 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an Los valores disponibles son 'Seguimiento', 'Depurar', 'Información', 'Advertencia', 'Error' y 'Crítico'. + + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + + - List available tests. - Enumere las pruebas disponibles. + List available tests. +Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. + Enumere las pruebas disponibles. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf index 84f1baa049..30d18cbebe 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf @@ -511,9 +511,15 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an Les valeurs disponibles sont « Trace », « Debug », « Information », « Warning », « Error » et « Critical ». + + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + + - List available tests. - Répertorier les tests disponibles. + List available tests. +Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. + Répertorier les tests disponibles. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf index 7dcc50f3cc..b196fedc8a 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf @@ -511,9 +511,15 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an I valori disponibili sono 'Trace', 'Debug', 'Information', 'Warning', 'Error' e 'Critical'. + + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + + - List available tests. - Elenca i test disponibili. + List available tests. +Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. + Elenca i test disponibili. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf index fd6c736980..dc765a409c 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf @@ -511,9 +511,15 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an 使用可能な値は、'Trace'、'Debug'、'Information'、'Warning'、'Error'、'Critical' です。 + + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + + - List available tests. - 使用可能なテストを一覧表示します。 + List available tests. +Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. + 使用可能なテストを一覧表示します。 diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf index ddbc83c020..c761f68e8e 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf @@ -511,9 +511,15 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an 사용 가능한 값은 'Trace', 'Debug', 'Information', 'Warning', 'Error' 및 'Critical'입니다. + + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + + - List available tests. - 사용 가능한 테스트를 나열합니다. + List available tests. +Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. + 사용 가능한 테스트를 나열합니다. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf index 3f4afe566f..02a8077af4 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf @@ -511,9 +511,15 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an Dostępne wartości to „Trace”, „Debug”, „Information”, „Warning”, „Error” i „Critical”. + + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + + - List available tests. - Wyświetl listę dostępnych testów. + List available tests. +Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. + Wyświetl listę dostępnych testów. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf index 6e7cd587d1..c652b82d12 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf @@ -511,9 +511,15 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an Os valores disponíveis são 'Rastreamento', 'Depuração', 'Informação', 'Aviso', 'Erro' e 'Crítico'. + + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + + - List available tests. - Listar testes disponíveis. + List available tests. +Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. + Listar testes disponíveis. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf index e6e8098163..6f5f433929 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf @@ -511,9 +511,15 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an Доступные значения: "Trace", "Debug", "Information", "Warning", "Error" и "Critical". + + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + + - List available tests. - Список доступных тестов. + List available tests. +Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. + Список доступных тестов. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf index 2a70594081..d8f0642004 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf @@ -511,9 +511,15 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an Kullanılabilir değerler: 'Trace', 'Debug', 'Information', 'Warning', 'Error' ve 'Critical'. + + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + + - List available tests. - Kullanılabilir testleri listeler. + List available tests. +Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. + Kullanılabilir testleri listeler. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf index 51121cef90..b64bd59f80 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf @@ -511,9 +511,15 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an 可用值为 "Trace"、"Debug"、"Information"、"Warning"、"Error" 和 "Critical"。 + + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + + - List available tests. - 列出可用的测试。 + List available tests. +Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. + 列出可用的测试。 diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf index 266c6d62bb..e38600716c 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf @@ -511,9 +511,15 @@ The available values are 'Trace', 'Debug', 'Information', 'Warning', 'Error', an 可用的值為 'Trace'、'Debug'、'Information'、'Warning'、'Error' 和 'Critical'。 + + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + '--list-tests' received unexpected value '{0}'. Supported values are: {1}. + + - List available tests. - 列出可用的測試。 + List available tests. +Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. + 列出可用的測試。 diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs index 58c3871342..ff334acdfc 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/HelpInfoTests.cs @@ -58,6 +58,7 @@ Show the command line help. Display .NET test application information. --list-tests List available tests. + Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. --minimum-expected-tests Specifies the minimum number of tests that are expected to run. --results-directory diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TestDiscoveryTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TestDiscoveryTests.cs index 8a75963330..372ca66f5a 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TestDiscoveryTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/TestDiscoveryTests.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Text.Json; + using Microsoft.Testing.Platform.Acceptance.IntegrationTests; using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers; using Microsoft.Testing.Platform.Helpers; @@ -40,6 +42,90 @@ public async Task DiscoverTests_WithFilter_FindsOnlyFilteredOnes(string currentT testHostResult.AssertOutputDoesNotContain("Test2"); } + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task DiscoverTests_WithJsonOutput_ProducesValidJsonDocumentWithExpectedFields(string currentTfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, currentTfm); + + TestHostResult testHostResult = await testHost.ExecuteAsync("--list-tests json", cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + + // Stdout must be the JSON document only — no banner, no progress, no summary. + // Trim because the test infrastructure may append a final newline. + string output = testHostResult.StandardOutput.Trim(); + Assert.IsTrue(output.StartsWith('{'), $"Expected stdout to start with '{{' but starts with: {output[..Math.Min(40, output.Length)]}"); + Assert.IsTrue(output.EndsWith('}'), $"Expected stdout to end with '}}' but ends with: {output[^Math.Min(40, output.Length)..]}"); + + // No error noise on stderr for a successful discovery. + Assert.AreEqual(string.Empty, testHostResult.StandardError.Trim()); + + // The JSON document must be valid and parseable. + using var document = JsonDocument.Parse(output); + JsonElement root = document.RootElement; + + Assert.AreEqual(1, root.GetProperty("schemaVersion").GetInt32()); + + JsonElement tests = root.GetProperty("tests"); + Assert.IsGreaterThanOrEqualTo(2, tests.GetArrayLength(), $"Expected at least 2 tests but got {tests.GetArrayLength()}."); + + // Collect all display names and assert the expected ones are present. + var displayNames = new HashSet(StringComparer.Ordinal); + for (int i = 0; i < tests.GetArrayLength(); i++) + { + displayNames.Add(tests[i].GetProperty("displayName").GetString()!); + } + + Assert.Contains("Test1", displayNames); + Assert.Contains("Test2", displayNames); + + // Each test should expose a unique uid. + var uids = new HashSet(StringComparer.Ordinal); + for (int i = 0; i < tests.GetArrayLength(); i++) + { + JsonElement test = tests[i]; + string uid = test.GetProperty("uid").GetString()!; + Assert.IsTrue(uids.Add(uid), $"Duplicated uid '{uid}'."); + } + + // Test1 should expose its TestMethodIdentifierProperty data, pinning every v1 schema field + // for that node from the outside. Note: MSTest's VSTestBridge currently leaves + // assemblyFullName and returnTypeFullName empty (TODO in MSTestBridgedTestFramework); + // assert presence only for those. + bool foundTest1WithType = false; + for (int i = 0; i < tests.GetArrayLength(); i++) + { + JsonElement test = tests[i]; + if (test.GetProperty("displayName").GetString() == "Test1" + && test.TryGetProperty("type", out JsonElement type)) + { + Assert.AreEqual(JsonValueKind.String, type.GetProperty("assemblyFullName").ValueKind); + Assert.AreEqual("TestClass", type.GetProperty("typeName").GetString()); + Assert.AreEqual("Test1", type.GetProperty("methodName").GetString()); + Assert.AreEqual(0, type.GetProperty("methodArity").GetInt32()); + Assert.AreEqual(JsonValueKind.String, type.GetProperty("returnTypeFullName").ValueKind); + Assert.AreEqual(JsonValueKind.Array, type.GetProperty("parameterTypeFullNames").ValueKind); + foundTest1WithType = true; + break; + } + } + + Assert.IsTrue(foundTest1WithType, "Expected at least one Test1 entry to expose 'type' metadata."); + } + + [TestMethod] + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + public async Task DiscoverTests_WithInvalidJsonArgument_FailsWithValidationError(string currentTfm) + { + var testHost = TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, currentTfm); + + TestHostResult testHostResult = await testHost.ExecuteAsync("--list-tests xml", cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.InvalidCommandLine); + testHostResult.AssertOutputContains("'--list-tests' received unexpected value 'xml'"); + } + public sealed class TestAssetFixture() : TestAssetFixtureBase() { public string TargetAssetPath => GetAssetPath(AssetName); diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs index 7449a51387..107cdc8392 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs @@ -52,6 +52,7 @@ Show the command line help. Display .NET test application information. --list-tests List available tests. + Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. --minimum-expected-tests Specifies the minimum number of tests that are expected to run. --results-directory @@ -238,9 +239,10 @@ Note that this is slowing down the test execution. Hidden: True Description: For testing purposes --list-tests - Arity: 0 + Arity: 0..1 Hidden: False Description: List available tests. + Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. --minimum-expected-tests Arity: 0..1 Hidden: False diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs index 202fbc81b6..ca047d3106 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoTests.cs @@ -52,6 +52,7 @@ Show the command line help. Display .NET test application information. --list-tests List available tests. + Optionally accepts 'text' (the default human-readable output) or 'json' to print the discovered tests as a JSON document on standard output. --minimum-expected-tests Specifies the minimum number of tests that are expected to run. --results-directory @@ -223,9 +224,10 @@ Note that this is slowing down the test execution\. Hidden: True Description: For testing purposes --list-tests - Arity: 0 + Arity: 0..1 Hidden: False Description: List available tests\. + Optionally accepts 'text' \(the default human-readable output\) or 'json' to print the discovered tests as a JSON document on standard output\. --minimum-expected-tests Arity: 0\.\.1 Hidden: False diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/CommandLine/PlatformCommandLineProviderTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/CommandLine/PlatformCommandLineProviderTests.cs index f838ea4e3c..e43463a544 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/CommandLine/PlatformCommandLineProviderTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/CommandLine/PlatformCommandLineProviderTests.cs @@ -168,6 +168,70 @@ public async Task IsInvalid_When_Both_DiscoverTests_MinimumExpectedTests_Provide Assert.AreEqual(PlatformResources.PlatformCommandLineMinimumExpectedTestsIncompatibleDiscoverTests, validateOptionsResult.ErrorMessage); } + [TestMethod] + public async Task IsValid_When_DiscoverTests_HasNoArgument() + { + var provider = new PlatformCommandLineProvider(); + CommandLineOption option = provider.GetCommandLineOptions().First(x => x.Name == PlatformCommandLineProvider.DiscoverTestsOptionKey); + + ValidationResult validateOptionsResult = await provider.ValidateOptionArgumentsAsync(option, []).ConfigureAwait(false); + Assert.IsTrue(validateOptionsResult.IsValid); + } + + [TestMethod] + [DataRow("json")] + [DataRow("JSON")] + [DataRow("Json")] + [DataRow("text")] + [DataRow("TEXT")] + public async Task IsValid_When_DiscoverTests_HasAcceptedArgument_AnyCasing(string argument) + { + var provider = new PlatformCommandLineProvider(); + CommandLineOption option = provider.GetCommandLineOptions().First(x => x.Name == PlatformCommandLineProvider.DiscoverTestsOptionKey); + + ValidationResult validateOptionsResult = await provider.ValidateOptionArgumentsAsync(option, [argument]).ConfigureAwait(false); + Assert.IsTrue(validateOptionsResult.IsValid); + } + + [TestMethod] + public async Task IsInvalid_When_DiscoverTests_HasUnknownArgument() + { + var provider = new PlatformCommandLineProvider(); + CommandLineOption option = provider.GetCommandLineOptions().First(x => x.Name == PlatformCommandLineProvider.DiscoverTestsOptionKey); + + ValidationResult validateOptionsResult = await provider.ValidateOptionArgumentsAsync(option, ["xml"]).ConfigureAwait(false); + Assert.IsFalse(validateOptionsResult.IsValid); + Assert.AreEqual( + string.Format( + CultureInfo.InvariantCulture, + PlatformResources.PlatformCommandLineDiscoverTestsInvalidArgument, + "xml", + $"'{PlatformCommandLineProvider.DiscoverTestsTextArgument}', '{PlatformCommandLineProvider.DiscoverTestsJsonArgument}'"), + validateOptionsResult.ErrorMessage); + } + + [TestMethod] + public void IsListTestsJsonOutput_ReturnsTrueOnlyWhenJsonArgumentSet() + { + Assert.IsFalse(PlatformCommandLineProvider.IsListTestsJsonOutput(new TestCommandLineOptions([]))); + Assert.IsFalse(PlatformCommandLineProvider.IsListTestsJsonOutput(new TestCommandLineOptions(new Dictionary + { + { PlatformCommandLineProvider.DiscoverTestsOptionKey, [] }, + }))); + Assert.IsFalse(PlatformCommandLineProvider.IsListTestsJsonOutput(new TestCommandLineOptions(new Dictionary + { + { PlatformCommandLineProvider.DiscoverTestsOptionKey, ["text"] }, + }))); + Assert.IsTrue(PlatformCommandLineProvider.IsListTestsJsonOutput(new TestCommandLineOptions(new Dictionary + { + { PlatformCommandLineProvider.DiscoverTestsOptionKey, ["json"] }, + }))); + Assert.IsTrue(PlatformCommandLineProvider.IsListTestsJsonOutput(new TestCommandLineOptions(new Dictionary + { + { PlatformCommandLineProvider.DiscoverTestsOptionKey, ["JSON"] }, + }))); + } + [TestMethod] public async Task IsNotValid_If_ExitOnProcess_Not_Running() { diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/DiscoveredTestsJsonSerializerTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/DiscoveredTestsJsonSerializerTests.cs new file mode 100644 index 0000000000..ad82529e52 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/DiscoveredTestsJsonSerializerTests.cs @@ -0,0 +1,262 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Text.Json; + +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.OutputDevice; + +namespace Microsoft.Testing.Platform.UnitTests.OutputDevice; + +[TestClass] +public sealed class DiscoveredTestsJsonSerializerTests +{ + [TestMethod] + public void Serialize_Empty_ProducesEnvelopeWithNoTests() + { + string json = DiscoveredTestsJsonSerializer.Serialize([]); + + using var document = JsonDocument.Parse(json); + JsonElement root = document.RootElement; + + Assert.AreEqual(DiscoveredTestsJsonSerializer.SchemaVersion, root.GetProperty("schemaVersion").GetInt32()); + Assert.AreEqual(JsonValueKind.Array, root.GetProperty("tests").ValueKind); + Assert.AreEqual(0, root.GetProperty("tests").GetArrayLength()); + } + + [TestMethod] + public void Serialize_TestNodeWithOnlyUidAndDisplayName_IncludesNoOptionalFields() + { + var node = new TestNode + { + Uid = new TestNodeUid("abc"), + DisplayName = "MyDisplay", + }; + + using var document = JsonDocument.Parse(DiscoveredTestsJsonSerializer.Serialize([node])); + JsonElement test = document.RootElement.GetProperty("tests")[0]; + + Assert.AreEqual("abc", test.GetProperty("uid").GetString()); + Assert.AreEqual("MyDisplay", test.GetProperty("displayName").GetString()); + Assert.IsFalse(test.TryGetProperty("type", out _)); + Assert.IsFalse(test.TryGetProperty("location", out _)); + Assert.IsFalse(test.TryGetProperty("traits", out _)); + Assert.IsFalse(test.TryGetProperty("properties", out _)); + } + + [TestMethod] + public void Serialize_TestNodeWithMethodIdentifier_EmitsTypeBlock() + { + var node = new TestNode + { + Uid = new TestNodeUid("uid1"), + DisplayName = "Test", + Properties = new PropertyBag( + new TestMethodIdentifierProperty( + assemblyFullName: "MyAssembly, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null", + @namespace: "My.Tests", + typeName: "MyClass+Nested`1", + methodName: "MyMethod", + methodArity: 2, + parameterTypeFullNames: ["System.Int32", "System.String"], + returnTypeFullName: "System.Threading.Tasks.Task")), + }; + + using var document = JsonDocument.Parse(DiscoveredTestsJsonSerializer.Serialize([node])); + JsonElement type = document.RootElement.GetProperty("tests")[0].GetProperty("type"); + + Assert.AreEqual("MyAssembly, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null", type.GetProperty("assemblyFullName").GetString()); + Assert.AreEqual("My.Tests", type.GetProperty("namespace").GetString()); + Assert.AreEqual("MyClass+Nested`1", type.GetProperty("typeName").GetString()); + Assert.AreEqual("MyMethod", type.GetProperty("methodName").GetString()); + Assert.AreEqual(2, type.GetProperty("methodArity").GetInt32()); + Assert.AreEqual("System.Threading.Tasks.Task", type.GetProperty("returnTypeFullName").GetString()); + + JsonElement parameters = type.GetProperty("parameterTypeFullNames"); + Assert.AreEqual(2, parameters.GetArrayLength()); + Assert.AreEqual("System.Int32", parameters[0].GetString()); + Assert.AreEqual("System.String", parameters[1].GetString()); + } + + [TestMethod] + public void Serialize_TestNodeWithFileLocation_EmitsLocationBlock() + { + var node = new TestNode + { + Uid = new TestNodeUid("uid"), + DisplayName = "Test", + Properties = new PropertyBag( + new TestFileLocationProperty("/repo/src/Test.cs", new LinePositionSpan(new LinePosition(10, 0), new LinePosition(20, 0)))), + }; + + using var document = JsonDocument.Parse(DiscoveredTestsJsonSerializer.Serialize([node])); + JsonElement location = document.RootElement.GetProperty("tests")[0].GetProperty("location"); + + Assert.AreEqual("/repo/src/Test.cs", location.GetProperty("file").GetString()); + Assert.AreEqual(10, location.GetProperty("lineStart").GetInt32()); + Assert.AreEqual(20, location.GetProperty("lineEnd").GetInt32()); + } + + [TestMethod] + public void Serialize_TestNodeWithTraits_EmitsTraitsArray() + { + var node = new TestNode + { + Uid = new TestNodeUid("uid"), + DisplayName = "Test", + Properties = new PropertyBag( + new TestMetadataProperty("Category", "Integration"), + new TestMetadataProperty("Priority", "1")), + }; + + using var document = JsonDocument.Parse(DiscoveredTestsJsonSerializer.Serialize([node])); + JsonElement traits = document.RootElement.GetProperty("tests")[0].GetProperty("traits"); + + Assert.AreEqual(2, traits.GetArrayLength()); + + // The serializer reverses PropertyBag.OfType so the JSON reflects insertion order. + Assert.AreEqual("Category", traits[0].GetProperty("key").GetString()); + Assert.AreEqual("Integration", traits[0].GetProperty("value").GetString()); + Assert.AreEqual("Priority", traits[1].GetProperty("key").GetString()); + Assert.AreEqual("1", traits[1].GetProperty("value").GetString()); + } + + [TestMethod] + public void Serialize_TestNodeWithKeyValueProperties_EmitsPropertiesArray() + { + var node = new TestNode + { + Uid = new TestNodeUid("uid"), + DisplayName = "Test", + Properties = new PropertyBag( + new SerializableKeyValuePairStringProperty("ExecutionId", "exec-123"), + new SerializableKeyValuePairStringProperty("Other", "value")), + }; + + using var document = JsonDocument.Parse(DiscoveredTestsJsonSerializer.Serialize([node])); + JsonElement properties = document.RootElement.GetProperty("tests")[0].GetProperty("properties"); + + // Emitted as array of {key, value} to preserve insertion order and allow duplicates. + Assert.AreEqual(JsonValueKind.Array, properties.ValueKind); + Assert.AreEqual(2, properties.GetArrayLength()); + Assert.AreEqual("ExecutionId", properties[0].GetProperty("key").GetString()); + Assert.AreEqual("exec-123", properties[0].GetProperty("value").GetString()); + Assert.AreEqual("Other", properties[1].GetProperty("key").GetString()); + Assert.AreEqual("value", properties[1].GetProperty("value").GetString()); + } + + [TestMethod] + public void Serialize_TestNodeWithDuplicatePropertyKeys_PreservesBothEntries() + { + var node = new TestNode + { + Uid = new TestNodeUid("uid"), + DisplayName = "Test", + Properties = new PropertyBag( + new SerializableKeyValuePairStringProperty("Tag", "first"), + new SerializableKeyValuePairStringProperty("Tag", "second")), + }; + + using var document = JsonDocument.Parse(DiscoveredTestsJsonSerializer.Serialize([node])); + JsonElement properties = document.RootElement.GetProperty("tests")[0].GetProperty("properties"); + + Assert.AreEqual(2, properties.GetArrayLength()); + Assert.AreEqual("Tag", properties[0].GetProperty("key").GetString()); + Assert.AreEqual("first", properties[0].GetProperty("value").GetString()); + Assert.AreEqual("Tag", properties[1].GetProperty("key").GetString()); + Assert.AreEqual("second", properties[1].GetProperty("value").GetString()); + } + + [TestMethod] + public void Serialize_TestNodeWithGlobalNamespace_OmitsNamespaceField() + { + var node = new TestNode + { + Uid = new TestNodeUid("uid"), + DisplayName = "Test", + Properties = new PropertyBag( + new TestMethodIdentifierProperty( + assemblyFullName: "MyAssembly", + @namespace: string.Empty, + typeName: "GlobalType", + methodName: "GlobalMethod", + methodArity: 0, + parameterTypeFullNames: [], + returnTypeFullName: "System.Void")), + }; + + using var document = JsonDocument.Parse(DiscoveredTestsJsonSerializer.Serialize([node])); + JsonElement type = document.RootElement.GetProperty("tests")[0].GetProperty("type"); + + Assert.IsFalse(type.TryGetProperty("namespace", out _)); + Assert.AreEqual("GlobalType", type.GetProperty("typeName").GetString()); + } + + [TestMethod] + public void Serialize_DisplayNameWithSpecialCharacters_EscapesProperly() + { + var node = new TestNode + { + Uid = new TestNodeUid("uid"), + DisplayName = "Quotes\" Backslash\\ Newline\n Tab\t", + }; + + // Should round-trip through JsonDocument without throwing. + using var document = JsonDocument.Parse(DiscoveredTestsJsonSerializer.Serialize([node])); + Assert.AreEqual( + "Quotes\" Backslash\\ Newline\n Tab\t", + document.RootElement.GetProperty("tests")[0].GetProperty("displayName").GetString()); + } + + [TestMethod] + public void Serialize_DisplayNameWithLoneSurrogate_ReplacesWithReplacementCharacter() + { + // Lone high surrogate followed by a non-surrogate. + var node = new TestNode + { + Uid = new TestNodeUid("uid"), + DisplayName = "\uD83DZ", + }; + + // Must still parse as valid JSON; the lone surrogate is replaced by U+FFFD. + using var document = JsonDocument.Parse(DiscoveredTestsJsonSerializer.Serialize([node])); + Assert.AreEqual( + "\uFFFDZ", + document.RootElement.GetProperty("tests")[0].GetProperty("displayName").GetString()); + } + + [TestMethod] + public void Serialize_DisplayNameWithValidSurrogatePair_PreservesPair() + { + // 😀 = U+1F600 encoded as surrogate pair U+D83D U+DE00. + var node = new TestNode + { + Uid = new TestNodeUid("uid"), + DisplayName = "Smile \uD83D\uDE00", + }; + + using var document = JsonDocument.Parse(DiscoveredTestsJsonSerializer.Serialize([node])); + Assert.AreEqual( + "Smile \uD83D\uDE00", + document.RootElement.GetProperty("tests")[0].GetProperty("displayName").GetString()); + } + + [TestMethod] + public void Serialize_DisplayNameWithLineAndParagraphSeparators_EscapesThem() + { + var node = new TestNode + { + Uid = new TestNodeUid("uid"), + DisplayName = "line\u2028para\u2029end", + }; + + string json = DiscoveredTestsJsonSerializer.Serialize([node]); + Assert.Contains("\\u2028", json); + Assert.Contains("\\u2029", json); + + using var document = JsonDocument.Parse(json); + Assert.AreEqual( + "line\u2028para\u2029end", + document.RootElement.GetProperty("tests")[0].GetProperty("displayName").GetString()); + } +}