diff --git a/Directory.Build.props b/Directory.Build.props index d043d95385..baeca8286a 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -80,6 +80,13 @@ --> 1.0.0 + alpha + + + 1.0.0 + alpha + true + true $(MicrosoftTestingExtensionsCommonVersion) @@ -152,6 +156,13 @@ + + $(MicrosoftTestingExtensionsGitHubActionsReportVersion) + + + $(MicrosoftTestingExtensionsCtrfReportVersion) diff --git a/src/Package/MSTest.Sdk/Sdk/Runner/NativeAOT.targets b/src/Package/MSTest.Sdk/Sdk/Runner/NativeAOT.targets index f61b50710c..58f85b8816 100644 --- a/src/Package/MSTest.Sdk/Sdk/Runner/NativeAOT.targets +++ b/src/Package/MSTest.Sdk/Sdk/Runner/NativeAOT.targets @@ -25,6 +25,7 @@ + diff --git a/src/Package/MSTest.Sdk/Sdk/Sdk.props.template b/src/Package/MSTest.Sdk/Sdk/Sdk.props.template index 004df17238..1bc238c1a9 100644 --- a/src/Package/MSTest.Sdk/Sdk/Sdk.props.template +++ b/src/Package/MSTest.Sdk/Sdk/Sdk.props.template @@ -22,6 +22,7 @@ ${MicrosoftTestingExtensionsCodeCoverageVersion} ${MicrosoftTestingExtensionsCtrfReportVersion} ${MicrosoftTestingExtensionsFakesVersion} + ${MicrosoftTestingExtensionsGitHubActionsReportVersion} ${MicrosoftTestingExtensionsJUnitReportVersion} ${MicrosoftTestingExtensionsOpenTelemetryVersion} ${MicrosoftTestingPlatformVersion} diff --git a/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets b/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets index dcc6680431..3a7a3bd8b6 100644 --- a/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets +++ b/src/Package/MSTest.Sdk/Sdk/VSTest/VSTest.targets @@ -6,6 +6,7 @@ + diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsArtifactUploader.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsArtifactUploader.cs index 78aa165da1..5e0947bd6f 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsArtifactUploader.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsArtifactUploader.cs @@ -69,7 +69,7 @@ public AzureDevOpsArtifactUploader( && artifactNameArguments is [string artifactName] ? artifactName : null; - _targetFrameworkMoniker = new(TargetFrameworkMonikerHelper.GetTargetFrameworkMoniker); + _targetFrameworkMoniker = new(TargetFrameworkMonikerHelper.GetTargetFrameworkMonikerIncludingPlatform); } public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage), typeof(FileArtifact)]; diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsLogGroupReporter.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsLogGroupReporter.cs index 4ce3ff7d02..3b7e633789 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsLogGroupReporter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsLogGroupReporter.cs @@ -53,7 +53,7 @@ public AzureDevOpsLogGroupReporter( _outputDevice = outputDevice; _testApplicationModuleInfo = testApplicationModuleInfo; _logger = loggerFactory.CreateLogger(); - _targetFrameworkMoniker = new(TargetFrameworkMonikerHelper.GetTargetFrameworkMoniker); + _targetFrameworkMoniker = new(TargetFrameworkMonikerHelper.GetTargetFrameworkMonikerIncludingPlatform); } public string Uid => nameof(AzureDevOpsLogGroupReporter); diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsReporter.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsReporter.cs index c6a790b653..84817cc67b 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsReporter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsReporter.cs @@ -3,7 +3,6 @@ using Microsoft.Testing.Extensions.AzureDevOpsReport.Resources; using Microsoft.Testing.Extensions.Reporting; -using Microsoft.Testing.Platform; using Microsoft.Testing.Platform.CommandLine; using Microsoft.Testing.Platform.Extensions; using Microsoft.Testing.Platform.Extensions.Messages; @@ -20,26 +19,9 @@ internal sealed class AzureDevOpsReporter : IOutputDeviceDataProducer { internal const double KnownFlakyFailureRateThreshold = 0.25; - private const string DeterministicBuildRoot = "/_/"; private const int MinSamplesForRegressionAnnotation = 5; private const string QuarantineBuildTagLine = "##vso[build.addbuildtag]has-quarantined-test-failure"; private const string WarningSeverity = "warning"; - private static readonly char[] NewlineCharacters = ['\r', '\n']; - - // Fully-qualified type prefixes for MSTest assertion implementations. A stack frame whose - // 'code' (i.e., the "Namespace.Type.Method(args)" portion) starts with any of these is treated - // as framework internals and skipped when looking for the user's call site to annotate. - // Matching on the type name (rather than the source file name) is robust to partial-class - // splits (e.g. Assert.AreEqual.cs, Assert.IComparable.cs) and extension-based assertion - // implementations such as Assert.That in Assert.That.cs, and it avoids false positives on user - // files innocently named *Assert.cs. See https://github.com/microsoft/testfx/issues/6925. - private static readonly string[] AssertionImplementationCodePrefixes = - [ - "Microsoft.VisualStudio.TestTools.UnitTesting.Assert.", - "Microsoft.VisualStudio.TestTools.UnitTesting.AssertExtensions.", - "Microsoft.VisualStudio.TestTools.UnitTesting.CollectionAssert.", - "Microsoft.VisualStudio.TestTools.UnitTesting.StringAssert.", - ]; private readonly IOutputDevice _outputDisplay; private readonly ILogger _logger; @@ -69,7 +51,7 @@ public AzureDevOpsReporter( _outputDisplay = outputDisplay; _historyService = historyService; _logger = loggerFactory.CreateLogger(); - _targetFrameworkMoniker = TargetFrameworkMonikerHelper.GetTargetFrameworkMoniker(); + _targetFrameworkMoniker = TargetFrameworkMonikerHelper.GetTargetFrameworkMonikerIncludingPlatform(); } public Type[] DataTypesConsumed { get; } = @@ -170,7 +152,7 @@ private async Task WriteExceptionAsync(string testDisplayName, string testName, string severity = GetSeverity(testName, isQuarantined); string annotationSuffix = BuildAnnotationSuffix(testName, isQuarantined); - string? line = GetErrorText(testDisplayName, explanation, exception, severity, _fileSystem, _logger, _targetFrameworkMoniker, annotationSuffix, _userStackFrameFilters); + string? line = GetErrorText(testDisplayName, explanation, exception, severity, _fileSystem, _logger, _targetFrameworkMoniker, annotationSuffix, _userStackFrameFilters, StackTraceSourceLocationResolver.SkipAssertionFramesForCurrentRuntime); if (line is null) { if (_logger.IsEnabled(LogLevel.Trace)) @@ -190,12 +172,12 @@ private async Task WriteExceptionAsync(string testDisplayName, string testName, } internal static /* for testing */ string? GetErrorText(string testDisplayName, string? explanation, Exception? exception, string severity, IFileSystem fileSystem, ILogger logger, string targetFrameworkMoniker) - => GetErrorText(testDisplayName, explanation, exception, severity, fileSystem, logger, targetFrameworkMoniker, additionalMessageSuffix: null, userStackFrameFilters: null); + => GetErrorText(testDisplayName, explanation, exception, severity, fileSystem, logger, targetFrameworkMoniker, additionalMessageSuffix: null, userStackFrameFilters: null, skipAssertionFrames: true); internal static /* for testing */ string? GetErrorText(string testDisplayName, string? explanation, Exception? exception, string severity, IFileSystem fileSystem, ILogger logger, string targetFrameworkMoniker, string? additionalMessageSuffix) - => GetErrorText(testDisplayName, explanation, exception, severity, fileSystem, logger, targetFrameworkMoniker, additionalMessageSuffix, userStackFrameFilters: null); + => GetErrorText(testDisplayName, explanation, exception, severity, fileSystem, logger, targetFrameworkMoniker, additionalMessageSuffix, userStackFrameFilters: null, skipAssertionFrames: true); - internal static /* for testing */ string? GetErrorText(string testDisplayName, string? explanation, Exception? exception, string severity, IFileSystem fileSystem, ILogger logger, string targetFrameworkMoniker, string? additionalMessageSuffix, Regex[]? userStackFrameFilters) + internal static /* for testing */ string? GetErrorText(string testDisplayName, string? explanation, Exception? exception, string severity, IFileSystem fileSystem, ILogger logger, string targetFrameworkMoniker, string? additionalMessageSuffix, Regex[]? userStackFrameFilters, bool skipAssertionFrames) { string message = explanation ?? exception?.Message ?? AzureDevOpsResources.NoFailureMessageFallback; string formattedMessage = $"{FormatErrorMessage(testDisplayName, targetFrameworkMoniker, message)}{additionalMessageSuffix}"; @@ -208,87 +190,17 @@ private async Task WriteExceptionAsync(string testDisplayName, string testName, logger.LogTrace($"Found repo root '{repoRoot}'"); } - foreach (string? stackFrame in stackTrace.Split(NewlineCharacters, StringSplitOptions.RemoveEmptyEntries)) - { - (string Code, string File, int LineNumber)? location = GetStackFrameLocation(stackFrame); - if (location is null) - { - if (logger.IsEnabled(LogLevel.Trace)) - { - logger.LogTrace("StackFrame location was null, continuing to next."); - } - - continue; - } - - string file = location.Value.File; - string code = location.Value.Code; - - if (IsAssertionImplementationFrame(code) || IsUserStackFrameFilterMatch(code, userStackFrameFilters, logger)) - { - if (logger.IsEnabled(LogLevel.Trace)) - { - logger.LogTrace($"StackFrame code '{code}' is an MSTest assertion implementation, continuing to next."); - } - - continue; - } - - string relativePath; - if (file.StartsWith(DeterministicBuildRoot, StringComparison.OrdinalIgnoreCase)) - { - if (logger.IsEnabled(LogLevel.Trace)) - { - logger.LogTrace($"Path '{file}' is coming from deterministic build."); - } - - relativePath = file.Substring(DeterministicBuildRoot.Length); - if (logger.IsEnabled(LogLevel.Trace)) - { - logger.LogTrace($"Using relative path '{relativePath}'."); - } - } - else if (file.StartsWith(repoRoot, StringComparison.OrdinalIgnoreCase)) - { - if (logger.IsEnabled(LogLevel.Trace)) - { - logger.LogTrace($"Path '{file}' is in current repo '{repoRoot}'."); - } - - relativePath = file.Substring(repoRoot.Length); - if (logger.IsEnabled(LogLevel.Trace)) - { - logger.LogTrace($"Using relative path '{relativePath}'."); - } - } - else - { - if (logger.IsEnabled(LogLevel.Trace)) - { - logger.LogTrace($"Path '{file}' does not belong to current repo '{repoRoot}'. Continue to next."); - } - - continue; - } - - string fullPath = Path.Combine(repoRoot, relativePath); - if (!fileSystem.ExistFile(fullPath)) - { - if (logger.IsEnabled(LogLevel.Trace)) - { - logger.LogTrace($"Path '{fullPath}' does not exist on disk. Continue to next."); - } - - continue; - } - - string relativeNormalizedPath = relativePath.Replace('\\', '/'); - if (logger.IsEnabled(LogLevel.Trace)) - { - logger.LogTrace($"Normalized path for GitHub '{relativeNormalizedPath}'."); - } + (string RelativeNormalizedPath, int LineNumber)? location = StackTraceSourceLocationResolver.TryResolve( + stackTrace, + repoRoot, + fileSystem, + logger, + skipAssertionFrames, + code => IsUserStackFrameFilterMatch(code, userStackFrameFilters, logger)); - string line = $"##vso[task.logissue type={severity};sourcepath={relativeNormalizedPath};linenumber={location.Value.LineNumber};columnnumber=1]{AzDoEscaper.Escape(formattedMessage)}"; + if (location is not null) + { + string line = $"##vso[task.logissue type={severity};sourcepath={location.Value.RelativeNormalizedPath};linenumber={location.Value.LineNumber};columnnumber=1]{AzDoEscaper.Escape(formattedMessage)}"; if (logger.IsEnabled(LogLevel.Trace)) { logger.LogTrace($"Reported full message '{line}'."); @@ -463,19 +375,6 @@ private static bool DisplayNameContainsTfm(string displayName, string tfm) => displayName.EndsWith($"({tfm})", StringComparison.Ordinal) || displayName.EndsWith($"(\"{tfm}\")", StringComparison.Ordinal); - private static bool IsAssertionImplementationFrame(string code) - { - foreach (string prefix in AssertionImplementationCodePrefixes) - { - if (code.StartsWith(prefix, StringComparison.Ordinal)) - { - return true; - } - } - - return false; - } - private static bool IsUserStackFrameFilterMatch(string code, Regex[]? userStackFrameFilters, ILogger logger) { if (userStackFrameFilters is null || userStackFrameFilters.Length == 0) @@ -503,28 +402,4 @@ private static bool IsUserStackFrameFilterMatch(string code, Regex[]? userStackF return false; } - - private static (string Code, string File, int LineNumber)? GetStackFrameLocation(string stackTraceLine) - { - Match match = StackTraceHelper.GetFrameRegex().Match(stackTraceLine); - if (!match.Success) - { - return null; - } - - string code = match.Groups["code"].Value; - if (RoslynString.IsNullOrWhiteSpace(code)) - { - return null; - } - - string file = match.Groups["file"].Value; - if (RoslynString.IsNullOrWhiteSpace(file)) - { - return null; - } - - int line = int.TryParse(match.Groups["line"].Value, out int value) ? value : 0; - return (code, file, line); - } } diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsSlowTestReporter.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsSlowTestReporter.cs index d3de325f48..c745c8b886 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsSlowTestReporter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsSlowTestReporter.cs @@ -4,14 +4,10 @@ using Microsoft.Testing.Extensions.AzureDevOpsReport.Resources; using Microsoft.Testing.Extensions.Reporting; using Microsoft.Testing.Platform.CommandLine; -using Microsoft.Testing.Platform.Extensions; using Microsoft.Testing.Platform.Extensions.Messages; -using Microsoft.Testing.Platform.Extensions.OutputDevice; -using Microsoft.Testing.Platform.Extensions.TestHost; using Microsoft.Testing.Platform.Helpers; using Microsoft.Testing.Platform.Logging; using Microsoft.Testing.Platform.OutputDevice; -using Microsoft.Testing.Platform.Services; namespace Microsoft.Testing.Extensions.AzureDevOpsReport; @@ -20,32 +16,18 @@ namespace Microsoft.Testing.Extensions.AzureDevOpsReport; /// threshold for tests with a known-short historical runtime and decorating each emission with the historical /// p95/p99 fetched from Azure DevOps test history. /// -/// -/// This is a self-contained emitter for the Azure DevOps host. Once the platform-level IProgressEnricher -/// hook (issue #9139) ships, the surfacing/backoff logic should migrate onto it and this type should only -/// supply the history-driven threshold and decoration. -/// -internal sealed class AzureDevOpsSlowTestReporter : IDataConsumer, ITestSessionLifetimeHandler, IOutputDeviceDataProducer +internal sealed class AzureDevOpsSlowTestReporter : SlowTestReporterBase { private const string AzureDevOpsTfBuildVariableName = "TF_BUILD"; - private static readonly TimeSpan ScanInterval = TimeSpan.FromSeconds(1); private readonly ICommandLineOptions _commandLineOptions; private readonly IEnvironment _environment; - private readonly IOutputDevice _outputDevice; - private readonly ITask _task; - private readonly IClock _clock; - private readonly ILogger _logger; private readonly IAzureDevOpsHistoryService _historyService; - private readonly ConcurrentDictionary _inProgress = new(StringComparer.Ordinal); private readonly bool _isEnabled; private readonly TimeSpan _staticThreshold; private double _multiplier; private volatile int _minimumSampleCount; - private volatile bool _active; - private CancellationTokenSource? _loopCancellationTokenSource; - private Task? _loopTask; public AzureDevOpsSlowTestReporter( ICommandLineOptions commandLineOptions, @@ -55,203 +37,61 @@ public AzureDevOpsSlowTestReporter( IClock clock, ILoggerFactory loggerFactory, IAzureDevOpsHistoryService historyService) + : base(outputDevice, task, clock, loggerFactory.CreateLogger()) { _commandLineOptions = commandLineOptions; _environment = environment; - _outputDevice = outputDevice; - _task = task; - _clock = clock; - _logger = loggerFactory.CreateLogger(); _historyService = historyService; _staticThreshold = TimeSpan.FromSeconds(AzureDevOpsCommandLineOptions.SlowTestStaticThresholdSeconds); _isEnabled = commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName) && commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory); } - public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)]; + public override string Uid => nameof(AzureDevOpsSlowTestReporter); - public string Uid => nameof(AzureDevOpsSlowTestReporter); + public override string DisplayName => AzureDevOpsResources.DisplayName; - public string Version => ExtensionVersion.DefaultSemVer; + public override string Description => AzureDevOpsResources.Description; - public string DisplayName => AzureDevOpsResources.DisplayName; + protected override bool IsEnabled => _isEnabled; - public string Description => AzureDevOpsResources.Description; - - public Task IsEnabledAsync() => Task.FromResult(_isEnabled); - - public async Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) + protected override bool OnActivating() { - try + if (!string.Equals(_environment.GetEnvironmentVariable(AzureDevOpsTfBuildVariableName), "true", StringComparison.OrdinalIgnoreCase)) { - testSessionContext.CancellationToken.ThrowIfCancellationRequested(); - - _active = false; - _inProgress.Clear(); - - if (!_isEnabled) + // Outside Azure DevOps the feature truly no-ops: we only leave a low-noise trace and never + // surface an output-device line, so local/dev runs that happen to pass the option stay quiet. + if (Logger.IsEnabled(LogLevel.Trace)) { - return; + Logger.LogTrace(AzureDevOpsResources.SlowTestHistoryRequiresTfBuildWarning); } - if (!string.Equals(_environment.GetEnvironmentVariable(AzureDevOpsTfBuildVariableName), "true", StringComparison.OrdinalIgnoreCase)) - { - // Outside Azure DevOps the feature truly no-ops: we only leave a low-noise trace and never - // surface an output-device line, so local/dev runs that happen to pass the option stay quiet. - if (_logger.IsEnabled(LogLevel.Trace)) - { - _logger.LogTrace(AzureDevOpsResources.SlowTestHistoryRequiresTfBuildWarning); - } - - return; - } - - // 'double' cannot be marked 'volatile', so publish the multiplier through Volatile.Write; the - // remaining fields use the 'volatile' modifier. Writing _active = true last (below) acts as the - // release fence that publishes all three to the test-data-producer threads in ConsumeAsync. - Volatile.Write(ref _multiplier, GetMultiplier()); - _minimumSampleCount = GetMinimumSampleCount(); - - _loopCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(testSessionContext.CancellationToken); - _active = true; - _loopTask = _task.RunLongRunning(() => ScanLoopAsync(_loopCancellationTokenSource.Token), nameof(AzureDevOpsSlowTestReporter), _loopCancellationTokenSource.Token); - } - catch (OperationCanceledException) - { - throw; + return false; } - catch (Exception ex) - { - LogUnexpectedException(nameof(OnTestSessionStartingAsync), ex); - } - } - - public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) - { - try - { - cancellationToken.ThrowIfCancellationRequested(); - if (!_active || value is not TestNodeUpdateMessage update) - { - return Task.CompletedTask; - } - - string uid = update.TestNode.Uid; - TestNodeStateProperty? state = update.TestNode.Properties.SingleOrDefault(); - if (state is InProgressTestNodeStateProperty) - { - string testName = TestNodeIdentity.GetTestName(update.TestNode); - TimeSpan threshold = ResolveThreshold(testName); - _inProgress[uid] = new InProgressTest(testName, _clock.UtcNow, threshold); - } - else if (state is not null) - { - // Any non-in-progress state (passed/failed/skipped/error/timeout/cancelled) is terminal for surfacing. - _inProgress.TryRemove(uid, out _); - } - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception ex) - { - LogUnexpectedException(nameof(ConsumeAsync), ex); - } + // 'double' cannot be marked 'volatile', so publish the multiplier through Volatile.Write; the + // remaining fields use the 'volatile' modifier. The base class writes _active = true last, which + // acts as the release fence that publishes all three to the test-data-producer threads in ConsumeAsync. + Volatile.Write(ref _multiplier, GetMultiplier()); + _minimumSampleCount = GetMinimumSampleCount(); - return Task.CompletedTask; + return true; } - public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionContext) - { - _active = false; - - CancellationTokenSource? loopCancellationTokenSource = _loopCancellationTokenSource; - if (loopCancellationTokenSource is not null) - { -#pragma warning disable VSTHRD103 // CancelAsync is unavailable on all target frameworks. - loopCancellationTokenSource.Cancel(); -#pragma warning restore VSTHRD103 - } - - if (_loopTask is not null) - { - try - { - await _loopTask.ConfigureAwait(false); - } - catch (OperationCanceledException) - { - // Expected during normal shutdown: cancelling _loopCancellationTokenSource above unblocks the - // scan loop, which surfaces as a cancellation here. Nothing to do — swallow and finish teardown. - } - catch (Exception ex) - { - LogUnexpectedException(nameof(OnTestSessionFinishingAsync), ex); - } - } + protected override string GetTestName(TestNode testNode) => TestNodeIdentity.GetTestName(testNode); - loopCancellationTokenSource?.Dispose(); - _loopCancellationTokenSource = null; - _loopTask = null; - _inProgress.Clear(); - } - - private async Task ScanLoopAsync(CancellationToken cancellationToken) - { - while (!cancellationToken.IsCancellationRequested) - { - try - { - await _task.Delay(ScanInterval, cancellationToken).ConfigureAwait(false); - await ScanOnceAsync(_clock.UtcNow, cancellationToken).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - return; - } - } - } - - // Internal for unit testing: performs a single surfacing pass at the given 'now' so tests can drive - // the emission/backoff logic deterministically without relying on the timer-driven loop. - internal async Task ScanOnceAsync(DateTimeOffset now, CancellationToken cancellationToken) + protected override TimeSpan ResolveThreshold(string testName) { - foreach (KeyValuePair entry in _inProgress) - { - InProgressTest test = entry.Value; - TimeSpan elapsed = now - test.StartTime; - if (elapsed < test.NextEmitThreshold) - { - continue; - } - - // Exponential backoff so a genuinely stuck test does not spam the log: T, 2T, 4T, ... - // Clamp at TimeSpan.MaxValue so a very long-running test cannot overflow Ticks * 2 into a - // negative value (which would make the backoff fire on every scan). - long currentTicks = test.NextEmitThreshold.Ticks; - test.NextEmitThreshold = currentTicks > TimeSpan.MaxValue.Ticks / 2 - ? TimeSpan.MaxValue - : TimeSpan.FromTicks(currentTicks * 2); - - try - { - await EmitSlowTestAsync(test, elapsed, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - LogUnexpectedException(nameof(ScanOnceAsync), ex); - } - } + bool hasStats = _historyService.TryGetDurationStats(testName, out DurationHistoryStats stats); + return AzureDevOpsSlowTestThresholds.ComputeThreshold(_staticThreshold, stats, hasStats, Volatile.Read(ref _multiplier), _minimumSampleCount); } - private async Task EmitSlowTestAsync(InProgressTest test, TimeSpan elapsed, CancellationToken cancellationToken) + protected override Task EmitSlowTestAsync(string testName, TimeSpan elapsed, CancellationToken cancellationToken) { string elapsedText = AzureDevOpsSlowTestThresholds.FormatDuration(elapsed.TotalMilliseconds); - string line = string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.SlowTestStillRunning, elapsedText, test.TestName); + string line = string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.SlowTestStillRunning, elapsedText, testName); - if (_historyService.TryGetDurationStats(test.TestName, out DurationHistoryStats stats) + if (_historyService.TryGetDurationStats(testName, out DurationHistoryStats stats) && AzureDevOpsSlowTestThresholds.HasUsableHistory(stats, hasStats: true, _minimumSampleCount)) { string decoration = string.Format( @@ -263,13 +103,7 @@ private async Task EmitSlowTestAsync(InProgressTest test, TimeSpan elapsed, Canc line = $"{line} {decoration}"; } - await _outputDevice.DisplayAsync(this, new AzureDevOpsCommandOutputDeviceData(line), cancellationToken).ConfigureAwait(false); - } - - private TimeSpan ResolveThreshold(string testName) - { - bool hasStats = _historyService.TryGetDurationStats(testName, out DurationHistoryStats stats); - return AzureDevOpsSlowTestThresholds.ComputeThreshold(_staticThreshold, stats, hasStats, Volatile.Read(ref _multiplier), _minimumSampleCount); + return OutputDevice.DisplayAsync(this, new AzureDevOpsCommandOutputDeviceData(line), cancellationToken); } private double GetMultiplier() @@ -287,28 +121,4 @@ private int GetMinimumSampleCount() && minimum >= 1 ? minimum : AzureDevOpsCommandLineOptions.SlowTestHistoryDefaultMinSample; - - private void LogUnexpectedException(string callbackName, Exception ex) - { - if (_logger.IsEnabled(LogLevel.Warning)) - { - _logger.LogWarning($"Unexpected exception in {callbackName}: {ex}"); - } - } - - private sealed class InProgressTest - { - public InProgressTest(string testName, DateTimeOffset startTime, TimeSpan threshold) - { - TestName = testName; - StartTime = startTime; - NextEmitThreshold = threshold; - } - - public string TestName { get; } - - public DateTimeOffset StartTime { get; } - - public TimeSpan NextEmitThreshold { get; set; } - } } diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsSummaryReporter.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsSummaryReporter.cs index 169fc9a8ec..bf525b603a 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsSummaryReporter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsSummaryReporter.cs @@ -63,7 +63,7 @@ public AzureDevOpsSummaryReporter( _testApplicationModuleInfo = testApplicationModuleInfo; _logger = loggerFactory.CreateLogger(); _isEnabled = commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsSummary); - _targetFrameworkMoniker = new(TargetFrameworkMonikerHelper.GetTargetFrameworkMoniker); + _targetFrameworkMoniker = new(TargetFrameworkMonikerHelper.GetTargetFrameworkMonikerIncludingPlatform); } public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)]; diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsTestResultsPublisher.Configuration.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsTestResultsPublisher.Configuration.cs index f90604e41b..a3a8fa7da8 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsTestResultsPublisher.Configuration.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsTestResultsPublisher.Configuration.cs @@ -43,7 +43,7 @@ private bool TryCreatePublishConfiguration(out AzureDevOpsPublishConfiguration? string currentTestApplicationPath = _testApplicationModuleInfo.GetCurrentTestApplicationFullPath(); string assemblyName = _testApplicationModuleInfo.TryGetAssemblyName() ?? Path.GetFileNameWithoutExtension(currentTestApplicationPath); string automatedTestStorage = Path.GetFileNameWithoutExtension(currentTestApplicationPath); - string targetFrameworkMoniker = TargetFrameworkMonikerHelper.GetTargetFrameworkMoniker(); + string targetFrameworkMoniker = TargetFrameworkMonikerHelper.GetTargetFrameworkMonikerIncludingPlatform(); string agentName = _environment.GetEnvironmentVariable("AGENT_NAME") ?? _environment.MachineName; string? stageName = _environment.GetEnvironmentVariable("SYSTEM_STAGENAME"); string? jobName = _environment.GetEnvironmentVariable("SYSTEM_JOBNAME"); diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Microsoft.Testing.Extensions.AzureDevOpsReport.csproj b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Microsoft.Testing.Extensions.AzureDevOpsReport.csproj index e9a3e00d87..0b5d6b5a40 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Microsoft.Testing.Extensions.AzureDevOpsReport.csproj +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Microsoft.Testing.Extensions.AzureDevOpsReport.csproj @@ -12,7 +12,10 @@ + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.FileNaming.cs b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.FileNaming.cs index 4e13db9e73..2427386651 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.FileNaming.cs +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/CtrfReportEngine.FileNaming.cs @@ -11,7 +11,7 @@ private string BuildDefaultFileName(DateTimeOffset finishTime) ?? _environment.GetEnvironmentVariable("USER") ?? "user"; string moduleName = Path.GetFileNameWithoutExtension(_testApplicationModuleInfo.GetCurrentTestApplicationFullPath()); - string targetFrameworkMoniker = TargetFrameworkMonikerHelper.GetTargetFrameworkMoniker(); + string targetFrameworkMoniker = TargetFrameworkMonikerHelper.GetTargetFrameworkMonikerIncludingPlatform(); string raw = $"{user}_{_environment.MachineName}_{moduleName}_{targetFrameworkMoniker}_{finishTime:yyyy-MM-dd_HH_mm_ss}.ctrf.json"; return ReplaceInvalidFileNameChars(raw); } diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/BannedSymbols.txt b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/BannedSymbols.txt new file mode 100644 index 0000000000..64ef236c50 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/BannedSymbols.txt @@ -0,0 +1,9 @@ +P:System.DateTime.Now; Use 'IClock' instead +P:System.DateTime.UtcNow; Use 'IClock' instead +M:System.Threading.Tasks.Task.Run(System.Action); Use 'ITask' instead +M:System.Threading.Tasks.Task.WhenAll(System.Threading.Tasks.Task[]); Use 'ITask' instead +M:System.Threading.Tasks.Task.WhenAll(System.Collections.Generic.IEnumerable{System.Threading.Tasks.Task}); Use 'ITask' instead +M:System.String.IsNullOrEmpty(System.String); Use 'RoslynString.IsNullOrEmpty' instead +M:System.String.IsNullOrWhiteSpace(System.String); Use 'RoslynString.IsNullOrWhiteSpace' instead +M:System.Diagnostics.Debug.Assert(System.Boolean); Use 'RoslynDebug.Assert' instead +M:System.Diagnostics.Debug.Assert(System.Boolean,System.String); Use 'RoslynDebug.Assert' instead diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs new file mode 100644 index 0000000000..5a003fcd65 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsAnnotationReporter.cs @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.GitHubActionsReport.Resources; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.OutputDevice; + +namespace Microsoft.Testing.Extensions.GitHubActionsReport; + +/// +/// Emits a GitHub Actions ::error workflow command for each failing test so the failure surfaces +/// both in the workflow run's Annotations tab and, when the source location can be resolved, on the +/// pull request's "Files changed" diff gutter. +/// +internal sealed class GitHubActionsAnnotationReporter : + IDataConsumer, + IOutputDeviceDataProducer +{ + private readonly IEnvironment _environment; + private readonly IFileSystem _fileSystem; + private readonly IOutputDevice _outputDisplay; + private readonly ILogger _logger; + private readonly bool _isEnabled; + + public GitHubActionsAnnotationReporter( + ICommandLineOptions commandLine, + IEnvironment environment, + IFileSystem fileSystem, + IOutputDevice outputDisplay, + ILoggerFactory loggerFactory) + { + _environment = environment; + _fileSystem = fileSystem; + _outputDisplay = outputDisplay; + _logger = loggerFactory.CreateLogger(); + _isEnabled = GitHubActionsFeature.IsEnabled(commandLine, environment, GitHubActionsCommandLineOptions.GitHubActionsAnnotations); + } + + public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)]; + + /// + public string Uid => nameof(GitHubActionsAnnotationReporter); + + /// + public string Version => ExtensionVersion.DefaultSemVer; + + /// + public string DisplayName { get; } = GitHubActionsResources.DisplayName; + + /// + public string Description { get; } = GitHubActionsResources.Description; + + /// + public Task IsEnabledAsync() => Task.FromResult(_isEnabled); + + public async Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) + { + try + { + cancellationToken.ThrowIfCancellationRequested(); + + if (value is not TestNodeUpdateMessage nodeUpdateMessage) + { + return; + } + + // FirstOrDefault (not SingleOrDefault): a malformed node that somehow carries more than one state + // property must degrade to "no annotation for this test" rather than throwing into the platform's + // data-consumer dispatch. + TestNodeStateProperty? nodeState = nodeUpdateMessage.TestNode.Properties.FirstOrDefault(); + + (string? Explanation, Exception? Exception)? failure = nodeState switch + { + FailedTestNodeStateProperty failed => (failed.Explanation, failed.Exception), + ErrorTestNodeStateProperty error => (error.Explanation, error.Exception), + TimeoutTestNodeStateProperty timeout => (timeout.Explanation, timeout.Exception), +#pragma warning disable CS0618, MTP0001 // Type or member is obsolete + CancelledTestNodeStateProperty cancelled => (cancelled.Explanation, cancelled.Exception), +#pragma warning restore CS0618, MTP0001 // Type or member is obsolete + _ => null, + }; + + if (failure is null) + { + return; + } + + await WriteAnnotationAsync(GetTestName(nodeUpdateMessage.TestNode), failure.Value.Explanation, failure.Value.Exception, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + // Mirror the sibling reporters: a failure while building/emitting an annotation (e.g. a malformed + // stack-trace path making IFileSystem.ExistFile throw) degrades to "no annotation" instead of + // propagating into the platform's consumer dispatch. + LogUnexpectedException(nameof(ConsumeAsync), ex); + } + } + + private async Task WriteAnnotationAsync(string testName, string? explanation, Exception? exception, CancellationToken cancellationToken) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + _logger.LogTrace("Failure received."); + } + + string repoRoot = GitHubActionsRepositoryRoot.Resolve(_environment) ?? string.Empty; + string line = GetErrorAnnotation(testName, explanation, exception, repoRoot, _fileSystem, _logger, StackTraceSourceLocationResolver.SkipAssertionFramesForCurrentRuntime); + + if (_logger.IsEnabled(LogLevel.Trace)) + { + _logger.LogTrace($"Showing failure annotation '{line}'."); + } + + // Prepend a newline so the '::error' workflow command always starts at column 0 on its own line. + // In CI the terminal output device runs in SimpleAnsi mode and emits a color reset ('\e[m') WITHOUT a + // trailing newline after the preceding colored "failed" test block. Emitting the annotation directly + // would yield "\e[m::error ..." and GitHub only recognizes a workflow command when the line begins + // with '::', so the dangling reset would silently drop the annotation. The leading newline pushes the + // reset onto its own (ignored) line and keeps the annotation parseable. + await _outputDisplay.DisplayAsync(this, new FormattedTextOutputDeviceData($"\n{line}"), cancellationToken).ConfigureAwait(false); + } + + internal static /* for testing */ string GetErrorAnnotation(string testName, string? explanation, Exception? exception, string? repoRoot, IFileSystem fileSystem, ILogger logger, bool skipAssertionFrames) + { + string message = explanation ?? exception?.Message ?? GitHubActionsResources.NoFailureMessageFallback; + string title = string.Format(CultureInfo.InvariantCulture, GitHubActionsResources.AnnotationTitle, testName); + + (string RelativeNormalizedPath, int LineNumber)? location = StackTraceSourceLocationResolver.TryResolve(exception?.StackTrace, repoRoot, fileSystem, logger, skipAssertionFrames); + if (location is not null) + { + return string.Format( + CultureInfo.InvariantCulture, + "::error file={0},line={1},col=1,title={2}::{3}", + GitHubActionsEscaper.EscapeProperty(location.Value.RelativeNormalizedPath), + location.Value.LineNumber.ToString(CultureInfo.InvariantCulture), + GitHubActionsEscaper.EscapeProperty(title), + GitHubActionsEscaper.EscapeData(message)); + } + + // Fallback: source location could not be resolved. The file/line/col properties are optional; + // a title-only annotation still surfaces in the workflow Annotations tab. + return string.Format( + CultureInfo.InvariantCulture, + "::error title={0}::{1}", + GitHubActionsEscaper.EscapeProperty(title), + GitHubActionsEscaper.EscapeData(message)); + } + + private static string GetTestName(TestNode testNode) + => TestNodeIdentity.GetTestName(testNode); + + private void LogUnexpectedException(string callbackName, Exception ex) + { + if (_logger.IsEnabled(LogLevel.Warning)) + { + _logger.LogWarning($"Unexpected exception in {callbackName}: {ex}"); + } + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsCommandLineOptions.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsCommandLineOptions.cs new file mode 100644 index 0000000000..3bab57e7f2 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsCommandLineOptions.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Extensions.GitHubActionsReport; + +internal static class GitHubActionsCommandLineOptions +{ + public const string GitHubActionsOptionName = "report-gh"; + public const string GitHubActionsGroups = "report-gh-groups"; + public const string GitHubActionsAnnotations = "report-gh-annotations"; + public const string GitHubActionsStepSummary = "report-gh-step-summary"; + public const string GitHubActionsSlowTestNotices = "report-gh-slow-test-notices"; + public const string GitHubActionsSlowTestThreshold = "report-gh-slow-test-threshold"; + + public const int SlowTestThresholdDefaultSeconds = 60; +} diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsCommandLineProvider.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsCommandLineProvider.cs new file mode 100644 index 0000000000..e9bcd78ebf --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsCommandLineProvider.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.GitHubActionsReport.Resources; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.CommandLine; +using Microsoft.Testing.Platform.Helpers; + +namespace Microsoft.Testing.Extensions.GitHubActionsReport; + +internal sealed class GitHubActionsCommandLineProvider : CommandLineOptionsProviderBase +{ + public GitHubActionsCommandLineProvider() + : base( + nameof(GitHubActionsCommandLineProvider), + ExtensionVersion.DefaultSemVer, + GitHubActionsResources.DisplayName, + GitHubActionsResources.Description, + [ + new CommandLineOption(GitHubActionsCommandLineOptions.GitHubActionsGroups, GitHubActionsResources.GroupsOptionDescription, ArgumentArity.ExactlyOne, false), + new CommandLineOption(GitHubActionsCommandLineOptions.GitHubActionsAnnotations, GitHubActionsResources.AnnotationsOptionDescription, ArgumentArity.ExactlyOne, false), + new CommandLineOption(GitHubActionsCommandLineOptions.GitHubActionsStepSummary, GitHubActionsResources.StepSummaryOptionDescription, ArgumentArity.ExactlyOne, false), + new CommandLineOption(GitHubActionsCommandLineOptions.GitHubActionsSlowTestNotices, GitHubActionsResources.SlowTestNoticesOptionDescription, ArgumentArity.ExactlyOne, false), + new CommandLineOption(GitHubActionsCommandLineOptions.GitHubActionsSlowTestThreshold, GitHubActionsResources.SlowTestThresholdOptionDescription, ArgumentArity.ExactlyOne, false), + new CommandLineOption(GitHubActionsCommandLineOptions.GitHubActionsOptionName, GitHubActionsResources.OptionDescription, ArgumentArity.Zero, false), + ]) + { + } + + public override Task ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments) + => commandOption.Name switch + { + GitHubActionsCommandLineOptions.GitHubActionsGroups or GitHubActionsCommandLineOptions.GitHubActionsAnnotations or GitHubActionsCommandLineOptions.GitHubActionsStepSummary or GitHubActionsCommandLineOptions.GitHubActionsSlowTestNotices + when !CommandLineOptionArgumentValidator.IsValidBooleanArgument(arguments[0]) + => ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, GitHubActionsResources.InvalidOnOffValue, arguments[0])), + GitHubActionsCommandLineOptions.GitHubActionsSlowTestThreshold + when !(TimeSpanParser.TryParse(arguments[0], TimeSpanDefaultUnit.Seconds, out TimeSpan threshold) && threshold > TimeSpan.Zero) + => ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, GitHubActionsResources.InvalidSlowTestThreshold, arguments[0])), + _ => ValidationResult.ValidTask, + }; +} diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsEscaper.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsEscaper.cs new file mode 100644 index 0000000000..c37531ca2c --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsEscaper.cs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform; + +namespace Microsoft.Testing.Extensions.GitHubActionsReport; + +internal static class GitHubActionsEscaper +{ + /// + /// Escapes a value used as the data portion of a GitHub Actions workflow command (the text after the + /// double-colon). See . + /// + public static string EscapeData(string value) + { + if (RoslynString.IsNullOrEmpty(value)) + { + return value; + } + + var result = new StringBuilder(value.Length); + foreach (char c in value) + { + switch (c) + { + case '%': + result.Append("%25"); + break; + case '\r': + result.Append("%0D"); + break; + case '\n': + result.Append("%0A"); + break; + default: + result.Append(c); + break; + } + } + + return result.ToString(); + } + + /// + /// Escapes a value used as a property of a GitHub Actions workflow command (e.g. the file, + /// line, col or title of an ::error command). In addition to the data + /// escapes, properties must also escape : and , because those characters delimit the + /// command syntax. See . + /// + public static string EscapeProperty(string value) + { + if (RoslynString.IsNullOrEmpty(value)) + { + return value; + } + + var result = new StringBuilder(value.Length); + foreach (char c in value) + { + switch (c) + { + case '%': + result.Append("%25"); + break; + case '\r': + result.Append("%0D"); + break; + case '\n': + result.Append("%0A"); + break; + case ':': + result.Append("%3A"); + break; + case ',': + result.Append("%2C"); + break; + default: + result.Append(c); + break; + } + } + + return result.ToString(); + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExtensions.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExtensions.cs new file mode 100644 index 0000000000..f339915f01 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExtensions.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.GitHubActionsReport; +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Extensions; + +/// +/// Provides extension methods for adding GitHub Actions reporting support to the test application builder. +/// +[Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")] +public static class GitHubActionsExtensions +{ + /// + /// Adds support to the test application builder. + /// + /// The test application builder. + public static void AddGitHubActionsProvider(this ITestApplicationBuilder builder) + { + var compositeSummaryReporter = new CompositeExtensionFactory(serviceProvider => + new GitHubActionsSummaryReporter( + serviceProvider.GetCommandLineOptions(), + serviceProvider.GetEnvironment(), + serviceProvider.GetFileSystem(), + serviceProvider.GetOutputDevice(), + serviceProvider.GetTestApplicationModuleInfo(), + serviceProvider.GetLoggerFactory())); + + var compositeSlowTestReporter = new CompositeExtensionFactory(serviceProvider => + new GitHubActionsSlowTestReporter( + serviceProvider.GetCommandLineOptions(), + serviceProvider.GetEnvironment(), + serviceProvider.GetOutputDevice(), + serviceProvider.GetTask(), + serviceProvider.GetClock(), + serviceProvider.GetLoggerFactory())); + + var compositeReporter = new CompositeExtensionFactory(serviceProvider => + new GitHubActionsReporter( + serviceProvider.GetCommandLineOptions(), + serviceProvider.GetEnvironment(), + serviceProvider.GetOutputDevice(), + serviceProvider.GetTestApplicationModuleInfo(), + serviceProvider.GetLoggerFactory())); + + builder.TestHost.AddDataConsumer(serviceProvider => + new GitHubActionsAnnotationReporter( + serviceProvider.GetCommandLineOptions(), + serviceProvider.GetEnvironment(), + serviceProvider.GetFileSystem(), + serviceProvider.GetOutputDevice(), + serviceProvider.GetLoggerFactory())); + + builder.TestHost.AddDataConsumer(compositeSummaryReporter); + builder.TestHost.AddTestSessionLifetimeHandler(compositeSummaryReporter); + builder.TestHost.AddDataConsumer(compositeSlowTestReporter); + builder.TestHost.AddTestSessionLifetimeHandler(compositeSlowTestReporter); + + // Register the group reporter last, as both a data consumer (no-op) and a session-lifetime handler, so its + // closing '::endgroup::' is ordered into the consumer phase after every other reporter's final output. + builder.TestHost.AddDataConsumer(compositeReporter); + builder.TestHost.AddTestSessionLifetimeHandler(compositeReporter); + builder.CommandLine.AddProvider(() => new GitHubActionsCommandLineProvider()); + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsFeature.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsFeature.cs new file mode 100644 index 0000000000..74d74373ae --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsFeature.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Helpers; + +namespace Microsoft.Testing.Extensions.GitHubActionsReport; + +/// +/// Shared activation logic for the GitHub Actions reporters. The extension activates only when running +/// on GitHub Actions (GITHUB_ACTIONS=true) and the --report-gh master switch is set; each +/// individual feature is then on by default but can be turned off with its --report-gh-* knob set +/// to off. +/// +internal static class GitHubActionsFeature +{ + public static bool IsRunningOnGitHubActions(IEnvironment environment) + => string.Equals(environment.GetEnvironmentVariable("GITHUB_ACTIONS"), "true", StringComparison.OrdinalIgnoreCase); + + public static bool IsMasterEnabled(ICommandLineOptions commandLine, IEnvironment environment) + => IsRunningOnGitHubActions(environment) + && commandLine.IsOptionSet(GitHubActionsCommandLineOptions.GitHubActionsOptionName); + + public static bool IsKnobEnabled(ICommandLineOptions commandLine, string knobOptionName) + => !(commandLine.TryGetOptionArgumentList(knobOptionName, out string[]? arguments) + && arguments is [string value] + && CommandLineOptionArgumentValidator.IsOffValue(value)); + + public static bool IsEnabled(ICommandLineOptions commandLine, IEnvironment environment, string knobOptionName) + => IsMasterEnabled(commandLine, environment) && IsKnobEnabled(commandLine, knobOptionName); +} diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsReporter.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsReporter.cs new file mode 100644 index 0000000000..ea8f06ebe2 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsReporter.cs @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.GitHubActionsReport.Resources; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Extensions.TestHost; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Extensions.GitHubActionsReport; + +/// +/// Wraps each test assembly's output in a GitHub Actions log group (::group:: / ::endgroup::) +/// so the runner UI collapses the assembly's test output by default. +/// +/// +/// Like the Azure DevOps sibling (AzureDevOpsLogGroupReporter), this handler also implements +/// (with a no-op ) +/// purely so that, at session end, its runs in the +/// consumer phase — i.e. after the producer-only handlers. Combined with registering it last, this ensures the +/// closing ::endgroup:: is emitted after the other reporters' final output, so the group truly wraps the +/// whole assembly's output. The flag guarantees ::endgroup:: is only emitted +/// when a matching ::group:: was actually opened. +/// +internal sealed class GitHubActionsReporter : + IDataConsumer, + ITestSessionLifetimeHandler, + IOutputDeviceDataProducer +{ + private readonly ICommandLineOptions _commandLine; + private readonly IEnvironment _environment; + private readonly IOutputDevice _outputDisplay; + private readonly ITestApplicationModuleInfo _testApplicationModuleInfo; + private readonly ILogger _logger; + private readonly string _targetFrameworkMoniker; + + private bool _groupOpened; + + public GitHubActionsReporter( + ICommandLineOptions commandLine, + IEnvironment environment, + IOutputDevice outputDisplay, + ITestApplicationModuleInfo testApplicationModuleInfo, + ILoggerFactory loggerFactory) + { + _commandLine = commandLine; + _environment = environment; + _outputDisplay = outputDisplay; + _testApplicationModuleInfo = testApplicationModuleInfo; + _logger = loggerFactory.CreateLogger(); + _targetFrameworkMoniker = TargetFrameworkMonikerHelper.GetTargetFrameworkMonikerIncludingPlatform(); + } + + /// + public string Uid => nameof(GitHubActionsReporter); + + /// + public string Version => ExtensionVersion.DefaultSemVer; + + /// + public string DisplayName { get; } = GitHubActionsResources.DisplayName; + + /// + public string Description { get; } = GitHubActionsResources.Description; + + /// + public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)]; + + /// + public Task IsEnabledAsync() + { + bool isEnabled = GitHubActionsFeature.IsEnabled(_commandLine, _environment, GitHubActionsCommandLineOptions.GitHubActionsGroups); + + if (_logger.IsEnabled(LogLevel.Trace)) + { + _logger.LogTrace($"{nameof(GitHubActionsReport)} groups is {(isEnabled ? "enabled" : "disabled")}."); + } + + return Task.FromResult(isEnabled); + } + + // No-op: this consumer subscribes to data only to be ordered in the consumer phase at session end + // (see the type-level remarks). It does not act on individual messages. + public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) + => Task.CompletedTask; + + /// + public async Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) + { + try + { + testSessionContext.CancellationToken.ThrowIfCancellationRequested(); + + string name = _testApplicationModuleInfo.TryGetAssemblyName() ?? _testApplicationModuleInfo.GetDisplayName(); + string title = string.Format(CultureInfo.InvariantCulture, GitHubActionsResources.GroupTitle, name, _targetFrameworkMoniker); + await _outputDisplay.DisplayAsync(this, new FormattedTextOutputDeviceData($"::group::{GitHubActionsEscaper.EscapeData(title)}"), testSessionContext.CancellationToken).ConfigureAwait(false); + _groupOpened = true; + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + LogUnexpectedException(nameof(OnTestSessionStartingAsync), ex); + } + } + + /// + public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionContext) + { + try + { + testSessionContext.CancellationToken.ThrowIfCancellationRequested(); + + if (!_groupOpened) + { + return; + } + + await _outputDisplay.DisplayAsync(this, new FormattedTextOutputDeviceData("::endgroup::"), testSessionContext.CancellationToken).ConfigureAwait(false); + _groupOpened = false; + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + LogUnexpectedException(nameof(OnTestSessionFinishingAsync), ex); + } + } + + private void LogUnexpectedException(string callbackName, Exception ex) + { + if (_logger.IsEnabled(LogLevel.Warning)) + { + _logger.LogWarning($"Unexpected exception in {callbackName}: {ex}"); + } + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsRepositoryRoot.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsRepositoryRoot.cs new file mode 100644 index 0000000000..be62d468a8 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsRepositoryRoot.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.TestInfrastructure; + +namespace Microsoft.Testing.Extensions.GitHubActionsReport; + +/// +/// Resolves the repository root used to turn an absolute source-file path from a stack trace into a +/// workspace-relative path for a GitHub Actions annotation. On a runner this is the +/// GITHUB_WORKSPACE directory; off a runner (e.g. local runs and unit tests) it walks up from the +/// application base directory looking for a .git directory or file. +/// +internal static class GitHubActionsRepositoryRoot +{ + public static string? Resolve(IEnvironment environment) + { + string? workspace = environment.GetEnvironmentVariable("GITHUB_WORKSPACE"); + return !RoslynString.IsNullOrWhiteSpace(workspace) + ? EnsureTrailingSeparator(workspace!) + : FindGitRoot(); + } + + internal static /* for testing */ string? FindGitRoot() + { + // Reuse the shared RootFinder walk (from AppContext.BaseDirectory up to the drive root, looking for a + // '.git' directory or worktree file, with a process-lifetime cache) rather than duplicating it here. + // RootFinder.Find() throws when no repository is found; a reporter running outside a git checkout must + // instead degrade to "no source location", so translate that into null. + try + { + return RootFinder.Find(); + } + catch (InvalidOperationException) + { + return null; + } + } + + private static string EnsureTrailingSeparator(string path) + => path.EndsWith(Path.DirectorySeparatorChar.ToString(), StringComparison.Ordinal) + || path.EndsWith(Path.AltDirectorySeparatorChar.ToString(), StringComparison.Ordinal) + ? path + : path + Path.DirectorySeparatorChar; +} diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSlowTestReporter.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSlowTestReporter.cs new file mode 100644 index 0000000000..1c6ac0bd3e --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSlowTestReporter.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.GitHubActionsReport.Resources; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.OutputDevice; + +namespace Microsoft.Testing.Extensions.GitHubActionsReport; + +/// +/// Surfaces tests that are still running past a per-test threshold as GitHub Actions ::notice +/// workflow commands, mirroring AzureDevOpsSlowTestReporter. +/// +internal sealed class GitHubActionsSlowTestReporter : SlowTestReporterBase +{ + private readonly bool _isEnabled; + private readonly TimeSpan _threshold; + + public GitHubActionsSlowTestReporter( + ICommandLineOptions commandLineOptions, + IEnvironment environment, + IOutputDevice outputDevice, + ITask task, + IClock clock, + ILoggerFactory loggerFactory) + : base(outputDevice, task, clock, loggerFactory.CreateLogger()) + { + _isEnabled = GitHubActionsFeature.IsEnabled(commandLineOptions, environment, GitHubActionsCommandLineOptions.GitHubActionsSlowTestNotices); + _threshold = GetThreshold(commandLineOptions); + } + + public override string Uid => nameof(GitHubActionsSlowTestReporter); + + public override string DisplayName => GitHubActionsResources.DisplayName; + + public override string Description => GitHubActionsResources.Description; + + protected override bool IsEnabled => _isEnabled; + + protected override string GetTestName(TestNode testNode) => TestNodeIdentity.GetTestName(testNode); + + protected override TimeSpan ResolveThreshold(string testName) => _threshold; + + protected override Task EmitSlowTestAsync(string testName, TimeSpan elapsed, CancellationToken cancellationToken) + { + string line = BuildNoticeLine(testName, elapsed); + return OutputDevice.DisplayAsync(this, new FormattedTextOutputDeviceData(line), cancellationToken); + } + + internal static /* for testing */ string BuildNoticeLine(string testName, TimeSpan elapsed) + { + string message = string.Format( + CultureInfo.InvariantCulture, + GitHubActionsResources.SlowTestStillRunning, + testName, + ((long)elapsed.TotalSeconds).ToString(CultureInfo.InvariantCulture)); + string title = string.Format(CultureInfo.InvariantCulture, GitHubActionsResources.SlowTestNoticeTitle, testName); + + return string.Format( + CultureInfo.InvariantCulture, + "::notice title={0}::{1}", + GitHubActionsEscaper.EscapeProperty(title), + GitHubActionsEscaper.EscapeData(message)); + } + + // Re-reads the threshold option, which the CLI provider has already validated (it guarantees a + // parseable positive duration). Accepts a bare number of seconds or a value with a unit suffix + // (e.g. '90s', '2m'). This mirrors the sibling AzureDevOpsSlowTestReporter, which likewise reads its + // history options straight from ICommandLineOptions rather than threading a parsed value through. + private static TimeSpan GetThreshold(ICommandLineOptions commandLineOptions) + => commandLineOptions.TryGetOptionArgumentList(GitHubActionsCommandLineOptions.GitHubActionsSlowTestThreshold, out string[]? arguments) + && arguments is [string value] + && TimeSpanParser.TryParse(value, TimeSpanDefaultUnit.Seconds, out TimeSpan threshold) + && threshold > TimeSpan.Zero + ? threshold + : TimeSpan.FromSeconds(GitHubActionsCommandLineOptions.SlowTestThresholdDefaultSeconds); +} diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs new file mode 100644 index 0000000000..e50aa4df41 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsSummaryReporter.cs @@ -0,0 +1,354 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.GitHubActionsReport.Resources; +using Microsoft.Testing.Platform; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Extensions.TestHost; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Extensions.GitHubActionsReport; + +/// +/// Writes a markdown roll-up of the test run (totals, failures, slowest tests) to the file pointed to by +/// the GITHUB_STEP_SUMMARY environment variable. GitHub renders that file on the workflow run's +/// summary page. See +/// . +/// +internal sealed class GitHubActionsSummaryReporter : + IDataConsumer, + ITestSessionLifetimeHandler, + IOutputDeviceDataProducer +{ + private const string StepSummaryEnvironmentVariable = "GITHUB_STEP_SUMMARY"; + private const int MaxFailures = 20; + private const int MaxSlowestTests = 10; + + private readonly IEnvironment _environment; + private readonly IFileSystem _fileSystem; + private readonly IOutputDevice _outputDevice; + private readonly ITestApplicationModuleInfo _testApplicationModuleInfo; + private readonly ILogger _logger; + private readonly Lazy _targetFrameworkMoniker; + private readonly bool _isEnabled; + +#if NET9_0_OR_GREATER + private readonly System.Threading.Lock _stateLock = new(); +#else + private readonly object _stateLock = new(); +#endif +#pragma warning disable IDE0028 // Collection initialization can be simplified - target-typed `new` cannot pass the comparer in the same syntactic form expected. + private readonly Dictionary _records = new Dictionary(StringComparer.Ordinal); +#pragma warning restore IDE0028 + + public GitHubActionsSummaryReporter( + ICommandLineOptions commandLineOptions, + IEnvironment environment, + IFileSystem fileSystem, + IOutputDevice outputDevice, + ITestApplicationModuleInfo testApplicationModuleInfo, + ILoggerFactory loggerFactory) + { + _environment = environment; + _fileSystem = fileSystem; + _outputDevice = outputDevice; + _testApplicationModuleInfo = testApplicationModuleInfo; + _logger = loggerFactory.CreateLogger(); + _targetFrameworkMoniker = new(TargetFrameworkMonikerHelper.GetTargetFrameworkMonikerIncludingPlatform); + _isEnabled = GitHubActionsFeature.IsEnabled(commandLineOptions, environment, GitHubActionsCommandLineOptions.GitHubActionsStepSummary); + } + + public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)]; + + public string Uid => nameof(GitHubActionsSummaryReporter); + + public string Version => ExtensionVersion.DefaultSemVer; + + public string DisplayName => GitHubActionsResources.DisplayName; + + public string Description => GitHubActionsResources.Description; + + public Task IsEnabledAsync() => Task.FromResult(_isEnabled); + + public Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) + { + lock (_stateLock) + { + _records.Clear(); + } + + return Task.CompletedTask; + } + + public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) + { + try + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!_isEnabled || value is not TestNodeUpdateMessage update) + { + return Task.CompletedTask; + } + + TestNodeStateProperty? state = update.TestNode.Properties.FirstOrDefault(); + TerminalKind kind = GetTerminalKind(state); + if (kind == TerminalKind.NotTerminal) + { + return Task.CompletedTask; + } + + string uid = update.TestNode.Uid; + string displayName = update.TestNode.DisplayName; + + // Resolve the stable, fully-qualified name the same way the annotation and slow-test reporters do + // (preferring TestMethodIdentifierProperty) so a given test renders identically across all three surfaces. + string fullyQualifiedName = TestNodeIdentity.GetTestName(update.TestNode); + + TimingProperty? timing = null; + PropertyBag.PropertyBagEnumerator enumerator = update.TestNode.Properties.GetStructEnumerator(); + while (enumerator.MoveNext()) + { + if (enumerator.Current is TimingProperty t) + { + timing = t; + break; + } + } + + TimeSpan duration = timing?.GlobalTiming.Duration ?? TimeSpan.Zero; + + lock (_stateLock) + { + _records[uid] = new TestRecord(displayName, fullyQualifiedName, kind, duration); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + LogUnexpectedException(nameof(ConsumeAsync), ex); + } + + return Task.CompletedTask; + } + + public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionContext) + { + try + { + testSessionContext.CancellationToken.ThrowIfCancellationRequested(); + + if (!_isEnabled) + { + return; + } + + string? path = _environment.GetEnvironmentVariable(StepSummaryEnvironmentVariable); + if (RoslynString.IsNullOrWhiteSpace(path)) + { + // Outside a GitHub Actions step (or when summaries are unsupported) there is nowhere to + // write. Stay quiet apart from a low-noise trace so local/dev runs don't get a warning. + if (_logger.IsEnabled(LogLevel.Trace)) + { + _logger.LogTrace($"'{StepSummaryEnvironmentVariable}' is not set; skipping job summary."); + } + + return; + } + + List snapshot; + lock (_stateLock) + { + snapshot = [.. _records.Values]; + } + + string markdown = BuildMarkdown(snapshot, _testApplicationModuleInfo.TryGetAssemblyName() ?? "unknown assembly name", _targetFrameworkMoniker.Value); + + try + { + using IFileStream stream = _fileSystem.NewFileStream(path!, FileMode.Append, FileAccess.Write, FileShare.ReadWrite); + using var writer = new StreamWriter(stream.Stream, new UTF8Encoding(false)); + await writer.WriteAsync(markdown).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + string warning = string.Format(CultureInfo.InvariantCulture, GitHubActionsResources.StepSummaryWriteFailedWarning, path, ex.Message); + if (_logger.IsEnabled(LogLevel.Warning)) + { + _logger.LogWarning(warning); + } + + await _outputDevice.DisplayAsync(this, new WarningMessageOutputDeviceData(warning), testSessionContext.CancellationToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + LogUnexpectedException(nameof(OnTestSessionFinishingAsync), ex); + } + } + + internal static /* for testing */ string BuildMarkdown(IReadOnlyList records, string assemblyName, string targetFrameworkMoniker) + { + int total = records.Count; + int passed = 0; + int failed = 0; + int skipped = 0; + TimeSpan totalDuration = TimeSpan.Zero; + var failures = new List(); + + foreach (TestRecord record in records) + { + totalDuration += record.Duration; + switch (record.Kind) + { + case TerminalKind.Passed: + passed++; + break; + case TerminalKind.Failed: + failed++; + if (failures.Count < MaxFailures) + { + failures.Add(record); + } + + break; + case TerminalKind.Skipped: + skipped++; + break; + } + } + + string statusIcon = failed > 0 ? "❌" : "✅"; + + var builder = new StringBuilder(); + builder.Append("## ").Append(statusIcon).Append(" Test Run Summary — ").Append(assemblyName).Append(" (").Append(targetFrameworkMoniker).Append(")\n\n"); + builder.Append("| Total | Passed | Failed | Skipped | Duration |\n"); + builder.Append("|---:|---:|---:|---:|---:|\n"); + builder.Append("| ").Append(total.ToString(CultureInfo.InvariantCulture)) + .Append(" | ").Append(passed.ToString(CultureInfo.InvariantCulture)) + .Append(" | ").Append(failed.ToString(CultureInfo.InvariantCulture)) + .Append(" | ").Append(skipped.ToString(CultureInfo.InvariantCulture)) + .Append(" | ").Append(FormatDuration(totalDuration)).Append(" |\n\n"); + + if (failures.Count > 0) + { + builder.Append("### ❌ Failures (").Append(failed.ToString(CultureInfo.InvariantCulture)).Append(")\n\n"); + foreach (TestRecord failure in failures) + { + builder.Append("- `").Append(EscapeInlineCode(failure.FullyQualifiedName)).Append("`\n"); + } + + builder.Append('\n'); + } + + IEnumerable slowest = records + .Where(static r => r.Duration > TimeSpan.Zero) + .OrderByDescending(static r => r.Duration) + .Take(MaxSlowestTests); + + bool slowestEmitted = false; + foreach (TestRecord record in slowest) + { + if (!slowestEmitted) + { + builder.Append("### ⏱ Slowest tests\n\n"); + slowestEmitted = true; + } + + builder.Append("- `").Append(EscapeInlineCode(record.FullyQualifiedName)).Append("` — ").Append(FormatDuration(record.Duration)).Append('\n'); + } + + if (slowestEmitted) + { + builder.Append('\n'); + } + + return builder.ToString(); + } + + private static string FormatDuration(TimeSpan duration) + { + if (duration < TimeSpan.FromSeconds(1)) + { + return string.Format(CultureInfo.InvariantCulture, "{0}ms", (int)duration.TotalMilliseconds); + } + + if (duration < TimeSpan.FromMinutes(1)) + { + return string.Format(CultureInfo.InvariantCulture, "{0:0.00}s", duration.TotalSeconds); + } + + if (duration < TimeSpan.FromHours(1)) + { + return string.Format(CultureInfo.InvariantCulture, "{0}m {1:00}s", (int)duration.TotalMinutes, duration.Seconds); + } + + long totalHours = (long)Math.Floor(duration.TotalHours); + return string.Format(CultureInfo.InvariantCulture, "{0}h {1:00}m {2:00}s", totalHours, duration.Minutes, duration.Seconds); + } + + private static string EscapeInlineCode(string value) + => RoslynString.IsNullOrEmpty(value) ? value : value.Replace("`", "'").Replace("\r", string.Empty).Replace("\n", " "); + + private static TerminalKind GetTerminalKind(TestNodeStateProperty? state) + => state switch + { + PassedTestNodeStateProperty => TerminalKind.Passed, + FailedTestNodeStateProperty => TerminalKind.Failed, + ErrorTestNodeStateProperty => TerminalKind.Failed, + TimeoutTestNodeStateProperty => TerminalKind.Failed, + SkippedTestNodeStateProperty => TerminalKind.Skipped, +#pragma warning disable CS0618, MTP0001 + CancelledTestNodeStateProperty => TerminalKind.Failed, +#pragma warning restore CS0618, MTP0001 + _ => TerminalKind.NotTerminal, + }; + + private void LogUnexpectedException(string callbackName, Exception ex) + { + if (_logger.IsEnabled(LogLevel.Warning)) + { + _logger.LogWarning($"Unexpected exception in {callbackName}: {ex}"); + } + } + + internal readonly struct TestRecord + { + public TestRecord(string displayName, string fullyQualifiedName, TerminalKind kind, TimeSpan duration) + { + DisplayName = displayName; + FullyQualifiedName = fullyQualifiedName; + Kind = kind; + Duration = duration; + } + + public string DisplayName { get; } + + public string FullyQualifiedName { get; } + + public TerminalKind Kind { get; } + + public TimeSpan Duration { get; } + } + + internal enum TerminalKind + { + NotTerminal, + Passed, + Failed, + Skipped, + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Microsoft.Testing.Extensions.GitHubActionsReport.csproj b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Microsoft.Testing.Extensions.GitHubActionsReport.csproj new file mode 100644 index 0000000000..ed32e39de2 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Microsoft.Testing.Extensions.GitHubActionsReport.csproj @@ -0,0 +1,68 @@ + + + + netstandard2.0;$(SupportedNetFrameworks) + $(MicrosoftTestingExtensionsGitHubActionsReportVersionPrefix) + $(MicrosoftTestingExtensionsGitHubActionsReportPreReleaseVersionLabel) + true + true + $(RepoRoot)src\Platform\SharedExtensionHelpers\BuildInfo.cs.template + + + + + + + + + + + + + + + + + + + + + + + + + + + true + buildMultiTargeting + + + buildTransitive/$(TargetFramework) + + + build/$(TargetFramework) + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md new file mode 100644 index 0000000000..0d674a4040 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md @@ -0,0 +1,43 @@ +# Microsoft.Testing.Extensions.GitHubActionsReport + +Microsoft.Testing.Extensions.GitHubActionsReport is an extension for [Microsoft.Testing.Platform](https://www.nuget.org/packages/Microsoft.Testing.Platform) that emits GitHub Actions-native workflow commands so test runs on GitHub Actions produce a first-class experience. + +Microsoft.Testing.Platform is open source. You can find `Microsoft.Testing.Extensions.GitHubActionsReport` code in the [microsoft/testfx](https://github.com/microsoft/testfx) GitHub repository. + +## Install the package + +```dotnetcli +dotnet add package Microsoft.Testing.Extensions.GitHubActionsReport +``` + +## About + +This package extends Microsoft.Testing.Platform with: + +- **Per-assembly log groups**: emits `::group::` / `::endgroup::` workflow commands so each test assembly's output is collapsed by default in the runner UI +- **Failure annotations**: emits an `::error` workflow command for each failing test so failures appear in the workflow Annotations tab and, when the source location can be resolved, on the pull request's "Files changed" diff gutter +- **Job summary**: appends a markdown roll-up (totals, failures, slowest tests) to the file pointed to by `GITHUB_STEP_SUMMARY`, which GitHub renders on the workflow run summary page +- **Slow-test notices**: emits a `::notice` workflow command for any test still running past a threshold (default 60 seconds) + +The extension activates when the test run is on GitHub Actions (`GITHUB_ACTIONS=true`) and the `--report-gh` switch is passed; it no-ops otherwise. When active, each feature is enabled by default and can be toggled individually: + +| Option | Description | Default | +|---|---|---| +| `--report-gh` | Master switch that turns the extension on (required, in addition to running on GitHub Actions) | off | +| `--report-gh-groups on\|off` | Per-assembly log groups | on | +| `--report-gh-annotations on\|off` | Failure annotations | on | +| `--report-gh-step-summary on\|off` | Markdown job summary | on | +| `--report-gh-slow-test-notices on\|off` | Slow-test notices | on | +| `--report-gh-slow-test-threshold ` | Time before a slow-test notice is emitted; accepts a bare number of seconds or a unit suffix such as `90s`, `2m`, `1.5h` | 60s | + +## Related packages + +- [Microsoft.Testing.Extensions.AzureDevOpsReport](https://www.nuget.org/packages/Microsoft.Testing.Extensions.AzureDevOpsReport): Azure DevOps reporting + +## Documentation + +For comprehensive documentation, see . + +## Feedback & contributing + +Microsoft.Testing.Platform is an open source project. Provide feedback or report issues in the [microsoft/testfx](https://github.com/microsoft/testfx/issues) GitHub repository. diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PublicAPI/PublicAPI.Shipped.txt b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PublicAPI/PublicAPI.Shipped.txt new file mode 100644 index 0000000000..7dc5c58110 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PublicAPI/PublicAPI.Shipped.txt @@ -0,0 +1 @@ +#nullable enable diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PublicAPI/PublicAPI.Unshipped.txt new file mode 100644 index 0000000000..cfdb07bbec --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PublicAPI/PublicAPI.Unshipped.txt @@ -0,0 +1,5 @@ +#nullable enable +[TPEXP]Microsoft.Testing.Extensions.GitHubActionsExtensions +Microsoft.Testing.Extensions.GitHubActionsReport.TestingPlatformBuilderHook +[TPEXP]static Microsoft.Testing.Extensions.GitHubActionsExtensions.AddGitHubActionsProvider(this Microsoft.Testing.Platform.Builder.ITestApplicationBuilder! builder) -> void +static Microsoft.Testing.Extensions.GitHubActionsReport.TestingPlatformBuilderHook.AddExtensions(Microsoft.Testing.Platform.Builder.ITestApplicationBuilder! testApplicationBuilder, string![]! _) -> void diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/GitHubActionsResources.resx b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/GitHubActionsResources.resx new file mode 100644 index 0000000000..ec65d7cb29 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/GitHubActionsResources.resx @@ -0,0 +1,121 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Test failed: {0} + {0} is the fully qualified test name. + + + The test failed without providing a failure message. + + + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"}{Locked="GITHUB_STEP_SUMMARY"} + + + Failed to write the GitHub Actions job summary to '{0}': {1} + {0} is the file path, {1} is the error message. + + + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + {Locked="s"}{Locked="m"}{Locked="h"} + + + Slow test: {0} + {0} is the fully qualified test name. + + + {0} still running after {1}s + {0} is the fully qualified test name, {1} is the elapsed seconds. + + + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + {Locked="s"}{Locked="m"} + + + GitHub Actions report generator + + + Tests: {0} ({1}) + {0} is the assembly name, {1} is the target framework moniker. + + + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.cs.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.cs.xlf new file mode 100644 index 0000000000..432611fc10 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.cs.xlf @@ -0,0 +1,87 @@ + + + + + + Test failed: {0} + Test failed: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + GitHub Actions report generator + GitHub Actions report generator + + + + Tests: {0} ({1}) + Tests: {0} ({1}) + {0} is the assembly name, {1} is the target framework moniker. + + + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + {Locked="s"}{Locked="m"} + + + The test failed without providing a failure message. + The test failed without providing a failure message. + + + + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + Slow test: {0} + Slow test: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + {0} still running after {1}s + {0} still running after {1}s + {0} is the fully qualified test name, {1} is the elapsed seconds. + + + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + {Locked="s"}{Locked="m"}{Locked="h"} + + + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"}{Locked="GITHUB_STEP_SUMMARY"} + + + Failed to write the GitHub Actions job summary to '{0}': {1} + Failed to write the GitHub Actions job summary to '{0}': {1} + {0} is the file path, {1} is the error message. + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.de.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.de.xlf new file mode 100644 index 0000000000..e320897cc8 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.de.xlf @@ -0,0 +1,87 @@ + + + + + + Test failed: {0} + Test failed: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + GitHub Actions report generator + GitHub Actions report generator + + + + Tests: {0} ({1}) + Tests: {0} ({1}) + {0} is the assembly name, {1} is the target framework moniker. + + + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + {Locked="s"}{Locked="m"} + + + The test failed without providing a failure message. + The test failed without providing a failure message. + + + + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + Slow test: {0} + Slow test: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + {0} still running after {1}s + {0} still running after {1}s + {0} is the fully qualified test name, {1} is the elapsed seconds. + + + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + {Locked="s"}{Locked="m"}{Locked="h"} + + + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"}{Locked="GITHUB_STEP_SUMMARY"} + + + Failed to write the GitHub Actions job summary to '{0}': {1} + Failed to write the GitHub Actions job summary to '{0}': {1} + {0} is the file path, {1} is the error message. + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.es.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.es.xlf new file mode 100644 index 0000000000..2b3c00a7b6 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.es.xlf @@ -0,0 +1,87 @@ + + + + + + Test failed: {0} + Test failed: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + GitHub Actions report generator + GitHub Actions report generator + + + + Tests: {0} ({1}) + Tests: {0} ({1}) + {0} is the assembly name, {1} is the target framework moniker. + + + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + {Locked="s"}{Locked="m"} + + + The test failed without providing a failure message. + The test failed without providing a failure message. + + + + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + Slow test: {0} + Slow test: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + {0} still running after {1}s + {0} still running after {1}s + {0} is the fully qualified test name, {1} is the elapsed seconds. + + + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + {Locked="s"}{Locked="m"}{Locked="h"} + + + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"}{Locked="GITHUB_STEP_SUMMARY"} + + + Failed to write the GitHub Actions job summary to '{0}': {1} + Failed to write the GitHub Actions job summary to '{0}': {1} + {0} is the file path, {1} is the error message. + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.fr.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.fr.xlf new file mode 100644 index 0000000000..dbf86ae3f1 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.fr.xlf @@ -0,0 +1,87 @@ + + + + + + Test failed: {0} + Test failed: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + GitHub Actions report generator + GitHub Actions report generator + + + + Tests: {0} ({1}) + Tests: {0} ({1}) + {0} is the assembly name, {1} is the target framework moniker. + + + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + {Locked="s"}{Locked="m"} + + + The test failed without providing a failure message. + The test failed without providing a failure message. + + + + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + Slow test: {0} + Slow test: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + {0} still running after {1}s + {0} still running after {1}s + {0} is the fully qualified test name, {1} is the elapsed seconds. + + + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + {Locked="s"}{Locked="m"}{Locked="h"} + + + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"}{Locked="GITHUB_STEP_SUMMARY"} + + + Failed to write the GitHub Actions job summary to '{0}': {1} + Failed to write the GitHub Actions job summary to '{0}': {1} + {0} is the file path, {1} is the error message. + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.it.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.it.xlf new file mode 100644 index 0000000000..35d93329b2 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.it.xlf @@ -0,0 +1,87 @@ + + + + + + Test failed: {0} + Test failed: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + GitHub Actions report generator + GitHub Actions report generator + + + + Tests: {0} ({1}) + Tests: {0} ({1}) + {0} is the assembly name, {1} is the target framework moniker. + + + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + {Locked="s"}{Locked="m"} + + + The test failed without providing a failure message. + The test failed without providing a failure message. + + + + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + Slow test: {0} + Slow test: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + {0} still running after {1}s + {0} still running after {1}s + {0} is the fully qualified test name, {1} is the elapsed seconds. + + + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + {Locked="s"}{Locked="m"}{Locked="h"} + + + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"}{Locked="GITHUB_STEP_SUMMARY"} + + + Failed to write the GitHub Actions job summary to '{0}': {1} + Failed to write the GitHub Actions job summary to '{0}': {1} + {0} is the file path, {1} is the error message. + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ja.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ja.xlf new file mode 100644 index 0000000000..7e70cb7cb2 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ja.xlf @@ -0,0 +1,87 @@ + + + + + + Test failed: {0} + Test failed: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + GitHub Actions report generator + GitHub Actions report generator + + + + Tests: {0} ({1}) + Tests: {0} ({1}) + {0} is the assembly name, {1} is the target framework moniker. + + + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + {Locked="s"}{Locked="m"} + + + The test failed without providing a failure message. + The test failed without providing a failure message. + + + + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + Slow test: {0} + Slow test: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + {0} still running after {1}s + {0} still running after {1}s + {0} is the fully qualified test name, {1} is the elapsed seconds. + + + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + {Locked="s"}{Locked="m"}{Locked="h"} + + + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"}{Locked="GITHUB_STEP_SUMMARY"} + + + Failed to write the GitHub Actions job summary to '{0}': {1} + Failed to write the GitHub Actions job summary to '{0}': {1} + {0} is the file path, {1} is the error message. + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ko.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ko.xlf new file mode 100644 index 0000000000..c696cb06d0 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ko.xlf @@ -0,0 +1,87 @@ + + + + + + Test failed: {0} + Test failed: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + GitHub Actions report generator + GitHub Actions report generator + + + + Tests: {0} ({1}) + Tests: {0} ({1}) + {0} is the assembly name, {1} is the target framework moniker. + + + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + {Locked="s"}{Locked="m"} + + + The test failed without providing a failure message. + The test failed without providing a failure message. + + + + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + Slow test: {0} + Slow test: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + {0} still running after {1}s + {0} still running after {1}s + {0} is the fully qualified test name, {1} is the elapsed seconds. + + + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + {Locked="s"}{Locked="m"}{Locked="h"} + + + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"}{Locked="GITHUB_STEP_SUMMARY"} + + + Failed to write the GitHub Actions job summary to '{0}': {1} + Failed to write the GitHub Actions job summary to '{0}': {1} + {0} is the file path, {1} is the error message. + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pl.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pl.xlf new file mode 100644 index 0000000000..0d44ee2503 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pl.xlf @@ -0,0 +1,87 @@ + + + + + + Test failed: {0} + Test failed: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + GitHub Actions report generator + GitHub Actions report generator + + + + Tests: {0} ({1}) + Tests: {0} ({1}) + {0} is the assembly name, {1} is the target framework moniker. + + + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + {Locked="s"}{Locked="m"} + + + The test failed without providing a failure message. + The test failed without providing a failure message. + + + + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + Slow test: {0} + Slow test: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + {0} still running after {1}s + {0} still running after {1}s + {0} is the fully qualified test name, {1} is the elapsed seconds. + + + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + {Locked="s"}{Locked="m"}{Locked="h"} + + + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"}{Locked="GITHUB_STEP_SUMMARY"} + + + Failed to write the GitHub Actions job summary to '{0}': {1} + Failed to write the GitHub Actions job summary to '{0}': {1} + {0} is the file path, {1} is the error message. + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pt-BR.xlf new file mode 100644 index 0000000000..5f1dab9dda --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pt-BR.xlf @@ -0,0 +1,87 @@ + + + + + + Test failed: {0} + Test failed: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + GitHub Actions report generator + GitHub Actions report generator + + + + Tests: {0} ({1}) + Tests: {0} ({1}) + {0} is the assembly name, {1} is the target framework moniker. + + + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + {Locked="s"}{Locked="m"} + + + The test failed without providing a failure message. + The test failed without providing a failure message. + + + + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + Slow test: {0} + Slow test: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + {0} still running after {1}s + {0} still running after {1}s + {0} is the fully qualified test name, {1} is the elapsed seconds. + + + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + {Locked="s"}{Locked="m"}{Locked="h"} + + + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"}{Locked="GITHUB_STEP_SUMMARY"} + + + Failed to write the GitHub Actions job summary to '{0}': {1} + Failed to write the GitHub Actions job summary to '{0}': {1} + {0} is the file path, {1} is the error message. + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ru.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ru.xlf new file mode 100644 index 0000000000..f3b69bb488 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ru.xlf @@ -0,0 +1,87 @@ + + + + + + Test failed: {0} + Test failed: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + GitHub Actions report generator + GitHub Actions report generator + + + + Tests: {0} ({1}) + Tests: {0} ({1}) + {0} is the assembly name, {1} is the target framework moniker. + + + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + {Locked="s"}{Locked="m"} + + + The test failed without providing a failure message. + The test failed without providing a failure message. + + + + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + Slow test: {0} + Slow test: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + {0} still running after {1}s + {0} still running after {1}s + {0} is the fully qualified test name, {1} is the elapsed seconds. + + + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + {Locked="s"}{Locked="m"}{Locked="h"} + + + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"}{Locked="GITHUB_STEP_SUMMARY"} + + + Failed to write the GitHub Actions job summary to '{0}': {1} + Failed to write the GitHub Actions job summary to '{0}': {1} + {0} is the file path, {1} is the error message. + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.tr.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.tr.xlf new file mode 100644 index 0000000000..83525819c6 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.tr.xlf @@ -0,0 +1,87 @@ + + + + + + Test failed: {0} + Test failed: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + GitHub Actions report generator + GitHub Actions report generator + + + + Tests: {0} ({1}) + Tests: {0} ({1}) + {0} is the assembly name, {1} is the target framework moniker. + + + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + {Locked="s"}{Locked="m"} + + + The test failed without providing a failure message. + The test failed without providing a failure message. + + + + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + Slow test: {0} + Slow test: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + {0} still running after {1}s + {0} still running after {1}s + {0} is the fully qualified test name, {1} is the elapsed seconds. + + + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + {Locked="s"}{Locked="m"}{Locked="h"} + + + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"}{Locked="GITHUB_STEP_SUMMARY"} + + + Failed to write the GitHub Actions job summary to '{0}': {1} + Failed to write the GitHub Actions job summary to '{0}': {1} + {0} is the file path, {1} is the error message. + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hans.xlf new file mode 100644 index 0000000000..395751aa81 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hans.xlf @@ -0,0 +1,87 @@ + + + + + + Test failed: {0} + Test failed: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + GitHub Actions report generator + GitHub Actions report generator + + + + Tests: {0} ({1}) + Tests: {0} ({1}) + {0} is the assembly name, {1} is the target framework moniker. + + + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + {Locked="s"}{Locked="m"} + + + The test failed without providing a failure message. + The test failed without providing a failure message. + + + + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + Slow test: {0} + Slow test: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + {0} still running after {1}s + {0} still running after {1}s + {0} is the fully qualified test name, {1} is the elapsed seconds. + + + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + {Locked="s"}{Locked="m"}{Locked="h"} + + + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"}{Locked="GITHUB_STEP_SUMMARY"} + + + Failed to write the GitHub Actions job summary to '{0}': {1} + Failed to write the GitHub Actions job summary to '{0}': {1} + {0} is the file path, {1} is the error message. + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hant.xlf new file mode 100644 index 0000000000..257c67e094 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hant.xlf @@ -0,0 +1,87 @@ + + + + + + Test failed: {0} + Test failed: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + GitHub Actions report generator + GitHub Actions report generator + + + + Tests: {0} ({1}) + Tests: {0} ({1}) + {0} is the assembly name, {1} is the target framework moniker. + + + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + Invalid value '{0}'. Valid values are 'on' (or 'true', 'enable', '1') or 'off' (or 'false', 'disable', '0'). + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + Invalid value '{0}'. The slow-test threshold must be a positive duration, e.g. '60', '90s', or '2m'. + {Locked="s"}{Locked="m"} + + + The test failed without providing a failure message. + The test failed without providing a failure message. + + + + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + + + + Slow test: {0} + Slow test: {0} + {0} is the fully qualified test name. + + + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"} + + + {0} still running after {1}s + {0} still running after {1}s + {0} is the fully qualified test name, {1} is the elapsed seconds. + + + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + {Locked="s"}{Locked="m"}{Locked="h"} + + + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + {Locked="on"}{Locked="off"}{Locked="true"}{Locked="enable"}{Locked="1"}{Locked="false"}{Locked="disable"}{Locked="0"}{Locked="GITHUB_STEP_SUMMARY"} + + + Failed to write the GitHub Actions job summary to '{0}': {1} + Failed to write the GitHub Actions job summary to '{0}': {1} + {0} is the file path, {1} is the error message. + + + + \ No newline at end of file diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/TestingPlatformBuilderHook.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/TestingPlatformBuilderHook.cs new file mode 100644 index 0000000000..849231f52a --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/TestingPlatformBuilderHook.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Builder; + +namespace Microsoft.Testing.Extensions.GitHubActionsReport; + +/// +/// This class is used by Microsoft.Testing.Platform.MSBuild to hook into the Testing Platform Builder to add GitHub Actions reporting support. +/// +public static class TestingPlatformBuilderHook +{ + /// + /// Adds GitHub Actions reporting support to the Testing Platform Builder. + /// + /// The test application builder. + /// The command line arguments. + public static void AddExtensions(ITestApplicationBuilder testApplicationBuilder, string[] _) + => testApplicationBuilder.AddGitHubActionsProvider(); +} diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/build/Microsoft.Testing.Extensions.GitHubActionsReport.props b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/build/Microsoft.Testing.Extensions.GitHubActionsReport.props new file mode 100644 index 0000000000..b640aa2cd6 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/build/Microsoft.Testing.Extensions.GitHubActionsReport.props @@ -0,0 +1,3 @@ + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/buildMultiTargeting/Microsoft.Testing.Extensions.GitHubActionsReport.props b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/buildMultiTargeting/Microsoft.Testing.Extensions.GitHubActionsReport.props new file mode 100644 index 0000000000..fa1c9640a6 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/buildMultiTargeting/Microsoft.Testing.Extensions.GitHubActionsReport.props @@ -0,0 +1,15 @@ + + + + + + Microsoft.Testing.Extensions.GitHubActionsReport + Microsoft.Testing.Extensions.GitHubActionsReport.TestingPlatformBuilderHook + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/buildTransitive/Microsoft.Testing.Extensions.GitHubActionsReport.props b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/buildTransitive/Microsoft.Testing.Extensions.GitHubActionsReport.props new file mode 100644 index 0000000000..c9ec314bf6 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/buildTransitive/Microsoft.Testing.Extensions.GitHubActionsReport.props @@ -0,0 +1,3 @@ + + + diff --git a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj index 593447ec6c..3fd6572bc4 100644 --- a/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj +++ b/src/Platform/Microsoft.Testing.Platform/Microsoft.Testing.Platform.csproj @@ -48,6 +48,7 @@ This package provides the core platform and the .NET implementation of the proto + diff --git a/src/Platform/SharedExtensionHelpers/ReportEngineBase.cs b/src/Platform/SharedExtensionHelpers/ReportEngineBase.cs index 52eefe194c..3f6327f7a8 100644 --- a/src/Platform/SharedExtensionHelpers/ReportEngineBase.cs +++ b/src/Platform/SharedExtensionHelpers/ReportEngineBase.cs @@ -81,7 +81,7 @@ internal static string BuildDefaultFileName(string testApplicationModule, string // TestResults folder overwrites the previous file (with a warning), matching // the behavior of an explicitly-provided file name. string moduleName = Path.GetFileNameWithoutExtension(testApplicationModule); - string targetFrameworkMoniker = TargetFrameworkMonikerHelper.GetTargetFrameworkMoniker(); + string targetFrameworkMoniker = TargetFrameworkMonikerHelper.GetTargetFrameworkMonikerIncludingPlatform(); string architecture = RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant(); string raw = $"{moduleName}_{targetFrameworkMoniker}_{architecture}.{extension}"; return ReplaceInvalidFileNameChars(raw); diff --git a/src/Platform/SharedExtensionHelpers/SlowTestReporterBase.cs b/src/Platform/SharedExtensionHelpers/SlowTestReporterBase.cs new file mode 100644 index 0000000000..d886c6df43 --- /dev/null +++ b/src/Platform/SharedExtensionHelpers/SlowTestReporterBase.cs @@ -0,0 +1,278 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Extensions.TestHost; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Extensions; + +/// +/// Shared plumbing for the pipeline-specific slow-test reporters (Azure DevOps and GitHub Actions). +/// Tracks in-progress tests, runs the background scan loop, and applies exponential backoff so a +/// genuinely stuck test does not spam the log. Host-specific concerns — how to resolve the emission +/// threshold and how to render the surfaced line — are supplied by derived types. +/// +/// +/// Once the platform-level IProgressEnricher hook (issue #9139) ships, the surfacing/backoff logic +/// here should migrate onto it and each host should only supply the threshold and decoration. +/// +internal abstract class SlowTestReporterBase : IDataConsumer, ITestSessionLifetimeHandler, IOutputDeviceDataProducer +{ + private static readonly TimeSpan ScanInterval = TimeSpan.FromSeconds(1); + + private readonly ITask _task; + private readonly IClock _clock; + private readonly ConcurrentDictionary _inProgress = new(StringComparer.Ordinal); + + private volatile bool _active; + private CancellationTokenSource? _loopCancellationTokenSource; + private Task? _loopTask; + + protected SlowTestReporterBase(IOutputDevice outputDevice, ITask task, IClock clock, ILogger logger) + { + OutputDevice = outputDevice; + _task = task; + _clock = clock; + Logger = logger; + } + + public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)]; + + public abstract string Uid { get; } + + public string Version => ExtensionVersion.DefaultSemVer; + + public abstract string DisplayName { get; } + + public abstract string Description { get; } + + /// + /// Gets a value indicating whether the reporter is enabled, based on the host-specific options. + /// + protected abstract bool IsEnabled { get; } + + protected IOutputDevice OutputDevice { get; } + + protected ILogger Logger { get; } + + public Task IsEnabledAsync() => Task.FromResult(IsEnabled); + + public Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) + { + try + { + testSessionContext.CancellationToken.ThrowIfCancellationRequested(); + + _active = false; + _inProgress.Clear(); + + if (!IsEnabled) + { + return Task.CompletedTask; + } + + // Host-specific activation gate (e.g. Azure DevOps requires TF_BUILD and warms up history state). + if (!OnActivating()) + { + return Task.CompletedTask; + } + + _loopCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(testSessionContext.CancellationToken); + _active = true; + _loopTask = _task.RunLongRunning(() => ScanLoopAsync(_loopCancellationTokenSource.Token), Uid, _loopCancellationTokenSource.Token); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + LogUnexpectedException(nameof(OnTestSessionStartingAsync), ex); + } + + return Task.CompletedTask; + } + + public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) + { + try + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!_active || value is not TestNodeUpdateMessage update) + { + return Task.CompletedTask; + } + + string uid = update.TestNode.Uid; + TestNodeStateProperty? state = update.TestNode.Properties.FirstOrDefault(); + if (state is InProgressTestNodeStateProperty) + { + string testName = GetTestName(update.TestNode); + TimeSpan threshold = ResolveThreshold(testName); + + // Use the first-seen start time: the platform can emit InProgress more than once for the same + // test (progress heartbeats), and resetting the start time on each would keep pushing the slow + // threshold out so a genuinely slow test would never surface. + _inProgress.TryAdd(uid, new InProgressTest(testName, _clock.UtcNow, threshold)); + } + else if (state is not null) + { + // Any non-in-progress state (passed/failed/skipped/error/timeout/cancelled) is terminal for surfacing. + _inProgress.TryRemove(uid, out _); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + LogUnexpectedException(nameof(ConsumeAsync), ex); + } + + return Task.CompletedTask; + } + + public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionContext) + { + _active = false; + + CancellationTokenSource? loopCancellationTokenSource = _loopCancellationTokenSource; + if (loopCancellationTokenSource is not null) + { +#pragma warning disable VSTHRD103 // CancelAsync is unavailable on all target frameworks. + loopCancellationTokenSource.Cancel(); +#pragma warning restore VSTHRD103 + } + + if (_loopTask is not null) + { + try + { + await _loopTask.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Expected during normal shutdown: cancelling _loopCancellationTokenSource above unblocks the + // scan loop, which surfaces as a cancellation here. Nothing to do — swallow and finish teardown. + } + catch (Exception ex) + { + LogUnexpectedException(nameof(OnTestSessionFinishingAsync), ex); + } + } + + loopCancellationTokenSource?.Dispose(); + _loopCancellationTokenSource = null; + _loopTask = null; + _inProgress.Clear(); + } + + /// + /// Resolves the stable, fully-qualified test name for a . + /// + protected abstract string GetTestName(TestNode testNode); + + /// + /// Resolves the elapsed time after which the given test should first surface a slow-test notice. + /// + protected abstract TimeSpan ResolveThreshold(string testName); + + /// + /// Renders and emits the host-specific slow-test line for a test that has passed its threshold. + /// + protected abstract Task EmitSlowTestAsync(string testName, TimeSpan elapsed, CancellationToken cancellationToken); + + /// + /// Host-specific activation gate invoked once the option-based check has passed. + /// Returns to keep the reporter dormant (e.g. when not running on the host CI). + /// + protected virtual bool OnActivating() => true; + + private async Task ScanLoopAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + await _task.Delay(ScanInterval, cancellationToken).ConfigureAwait(false); + await ScanOnceAsync(_clock.UtcNow, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; + } + } + } + + // Internal for unit testing: performs a single surfacing pass at the given 'now' so tests can drive + // the emission/backoff logic deterministically without relying on the timer-driven loop. + internal async Task ScanOnceAsync(DateTimeOffset now, CancellationToken cancellationToken) + { + foreach (KeyValuePair entry in _inProgress) + { + InProgressTest test = entry.Value; + TimeSpan elapsed = now - test.StartTime; + if (elapsed < test.NextEmitThreshold) + { + continue; + } + + // The enumeration is a moving snapshot of the ConcurrentDictionary; a test can complete (and be + // removed) between the snapshot and here. Skip it so we don't surface a slow-test notice for a + // test that has already finished. + if (!_inProgress.ContainsKey(entry.Key)) + { + continue; + } + + // Exponential backoff so a genuinely stuck test does not spam the log: T, 2T, 4T, ... + // Clamp at TimeSpan.MaxValue so a very long-running test cannot overflow Ticks * 2 into a + // negative value (which would make the backoff fire on every scan). + long currentTicks = test.NextEmitThreshold.Ticks; + test.NextEmitThreshold = currentTicks > TimeSpan.MaxValue.Ticks / 2 + ? TimeSpan.MaxValue + : TimeSpan.FromTicks(currentTicks * 2); + + try + { + await EmitSlowTestAsync(test.TestName, elapsed, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogUnexpectedException(nameof(ScanOnceAsync), ex); + } + } + } + + private void LogUnexpectedException(string callbackName, Exception ex) + { + if (Logger.IsEnabled(LogLevel.Warning)) + { + Logger.LogWarning($"Unexpected exception in {callbackName}: {ex}"); + } + } + + private sealed class InProgressTest + { + public InProgressTest(string testName, DateTimeOffset startTime, TimeSpan threshold) + { + TestName = testName; + StartTime = startTime; + NextEmitThreshold = threshold; + } + + public string TestName { get; } + + public DateTimeOffset StartTime { get; } + + public TimeSpan NextEmitThreshold { get; set; } + } +} diff --git a/src/Platform/SharedExtensionHelpers/StackTraceSourceLocationResolver.cs b/src/Platform/SharedExtensionHelpers/StackTraceSourceLocationResolver.cs new file mode 100644 index 0000000000..4f7bcede9f --- /dev/null +++ b/src/Platform/SharedExtensionHelpers/StackTraceSourceLocationResolver.cs @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; + +namespace Microsoft.Testing.Extensions; + +/// +/// Resolves the first user source location (workspace-relative file path + line) from an exception stack +/// trace, so a failure can be annotated on the reporting host (Azure DevOps ##vso[task.logissue] or +/// GitHub Actions ::error). Shared by the Azure DevOps and GitHub Actions reporters. +/// +internal static class StackTraceSourceLocationResolver +{ + // Source-linked (deterministic) builds emit paths rooted at '/_/' instead of the original absolute path. + private const string DeterministicBuildRoot = "/_/"; + + private static readonly char[] NewlineCharacters = ['\r', '\n']; + + // Fully-qualified type prefixes for MSTest assertion implementations. A stack frame whose 'code' starts + // with any of these is treated as framework internals and skipped when looking for the user's call site to + // annotate. Matching on the type name (rather than the source file name) is robust to partial-class splits + // (e.g. Assert.AreEqual.cs, Assert.IComparable.cs) and extension-based assertion implementations such as + // Assert.That in Assert.That.cs, and it avoids false positives on user files innocently named *Assert.cs. + // See https://github.com/microsoft/testfx/issues/6925. + private static readonly string[] AssertionImplementationCodePrefixes = + [ + "Microsoft.VisualStudio.TestTools.UnitTesting.Assert.", + "Microsoft.VisualStudio.TestTools.UnitTesting.AssertExtensions.", + "Microsoft.VisualStudio.TestTools.UnitTesting.CollectionAssert.", + "Microsoft.VisualStudio.TestTools.UnitTesting.StringAssert.", + ]; + + /// + /// Gets a value indicating whether MSTest assertion frames must be skipped manually on the current runtime. + /// + /// + /// MSTest's Assert, CollectionAssert, and StringAssert are all marked + /// [StackTraceHidden], which the CLR honors on .NET Core 2.1+ (i.e. every modern TFM) by omitting + /// those frames from altogether — so nothing needs skipping there. + /// Only .NET Framework ignores [StackTraceHidden] and still surfaces the assertion frames, which is + /// why the manual skip only earns its keep there. + /// + /// This is a runtime check on purpose: these extensions ship as netstandard2.0 and that build + /// is what loads under .NET Framework, so a compile-time #if NETFRAMEWORK would never be defined for + /// the running assembly. + /// + /// + public static bool SkipAssertionFramesForCurrentRuntime { get; } = + System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription.StartsWith(".NET Framework", StringComparison.OrdinalIgnoreCase); + + /// + /// Walks and returns the first frame that resolves to an existing file under + /// (or a deterministic-build path), as a workspace-relative, forward-slash path + /// plus its line number. Returns when no such frame can be resolved. + /// + /// The exception stack trace string, or . + /// The repository root used to relativize absolute paths (should end with a separator), or . + /// File system used to verify the candidate file exists on disk. + /// Logger for trace diagnostics. + /// + /// When , frames whose code matches a known MSTest assertion implementation are + /// skipped. Production callers pass . + /// + /// Optional additional per-frame predicate (e.g. host-specific user filters). + public static (string RelativeNormalizedPath, int LineNumber)? TryResolve( + string? stackTrace, + string? repoRoot, + IFileSystem fileSystem, + ILogger logger, + bool skipAssertionFrames, + Func? shouldSkipFrame = null) + { + if (RoslynString.IsNullOrEmpty(stackTrace) || RoslynString.IsNullOrEmpty(repoRoot)) + { + return null; + } + + foreach (string stackFrame in stackTrace!.Split(NewlineCharacters, StringSplitOptions.RemoveEmptyEntries)) + { + (string Code, string File, int LineNumber)? location = GetStackFrameLocation(stackFrame); + if (location is null) + { + continue; + } + + string file = location.Value.File; + string code = location.Value.Code; + + if ((skipAssertionFrames && IsAssertionImplementationFrame(code)) + || (shouldSkipFrame is not null && shouldSkipFrame(code))) + { + if (logger.IsEnabled(LogLevel.Trace)) + { + logger.LogTrace($"Skipping stack frame '{code}' while resolving the source location."); + } + + continue; + } + + string relativePath; + if (file.StartsWith(DeterministicBuildRoot, StringComparison.OrdinalIgnoreCase)) + { + relativePath = file.Substring(DeterministicBuildRoot.Length); + } + else if (file.StartsWith(repoRoot!, StringComparison.OrdinalIgnoreCase)) + { + relativePath = file.Substring(repoRoot!.Length); + } + else + { + continue; + } + + string fullPath = Path.Combine(repoRoot!, relativePath); + if (!fileSystem.ExistFile(fullPath)) + { + continue; + } + + // Annotations expect a workspace-relative path with forward slashes. + string relativeNormalizedPath = relativePath.Replace('\\', '/').TrimStart('/'); + if (logger.IsEnabled(LogLevel.Trace)) + { + logger.LogTrace($"Resolved source location '{relativeNormalizedPath}' (line {location.Value.LineNumber})."); + } + + return (relativeNormalizedPath, location.Value.LineNumber); + } + + return null; + } + + private static bool IsAssertionImplementationFrame(string code) + { + foreach (string prefix in AssertionImplementationCodePrefixes) + { + if (code.StartsWith(prefix, StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + + private static (string Code, string File, int LineNumber)? GetStackFrameLocation(string stackTraceLine) + { + Match match = StackTraceHelper.GetFrameRegex().Match(stackTraceLine); + if (!match.Success) + { + return null; + } + + string code = match.Groups["code"].Value; + if (RoslynString.IsNullOrWhiteSpace(code)) + { + return null; + } + + string file = match.Groups["file"].Value; + if (RoslynString.IsNullOrWhiteSpace(file)) + { + return null; + } + + int line = int.TryParse(match.Groups["line"].Value, out int value) ? value : 0; + return (code, file, line); + } +} diff --git a/src/Platform/SharedExtensionHelpers/TargetFrameworkMonikerHelper.cs b/src/Platform/SharedExtensionHelpers/TargetFrameworkMonikerHelper.cs index c17af84b8a..03c9af9dcd 100644 --- a/src/Platform/SharedExtensionHelpers/TargetFrameworkMonikerHelper.cs +++ b/src/Platform/SharedExtensionHelpers/TargetFrameworkMonikerHelper.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Platform.OutputDevice; @@ -7,7 +7,7 @@ namespace Microsoft.Testing.Extensions; internal static class TargetFrameworkMonikerHelper { - public static string GetTargetFrameworkMoniker() + public static string GetTargetFrameworkMonikerIncludingPlatform() => TargetFrameworkParser.GetShortTargetFrameworkIncludingPlatform(Assembly.GetEntryAssembly()) - ?? "unknown"; + ?? "unknown framework"; } diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/TestNodeIdentity.cs b/src/Platform/SharedExtensionHelpers/TestNodeIdentity.cs similarity index 91% rename from src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/TestNodeIdentity.cs rename to src/Platform/SharedExtensionHelpers/TestNodeIdentity.cs index 41bf73b562..8c8e967b0d 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/TestNodeIdentity.cs +++ b/src/Platform/SharedExtensionHelpers/TestNodeIdentity.cs @@ -4,15 +4,15 @@ using Microsoft.Testing.Platform; using Microsoft.Testing.Platform.Extensions.Messages; -namespace Microsoft.Testing.Extensions.AzureDevOpsReport; +namespace Microsoft.Testing.Extensions; internal static class TestNodeIdentity { private const string FullyQualifiedNamePropertyKey = "vstest.TestCase.FullyQualifiedName"; /// - /// Resolves the stable test name used to match a against Azure DevOps history - /// (which keys results by AutomatedTestName, i.e. the fully-qualified Namespace.Type.Method). + /// Resolves the stable, fully-qualified test name for a . Falls back to the + /// display name when the fully-qualified name property is unavailable. /// /// /// The identity is resolved in the same priority order used by the TRX and VSTest-bridge converters: diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs index 4bbac6557f..ad1da12473 100644 --- a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs +++ b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs @@ -201,6 +201,11 @@ public async Task RunTests_With_CentralPackageManagement_Standalone(string multi "--report-azdo", "--crashdump")); + yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, + "true", + "--report-gh", + "--crashdump")); + yield return new((buildConfig.MultiTfm, buildConfig.BuildConfiguration, "true", "--report-ctrf", @@ -269,7 +274,7 @@ public async Task RunTests_With_MSTestRunner_Standalone_EnableAll_Extensions(str foreach (string tfm in multiTfm.Split(";")) { var testHost = TestHost.LocateFrom(testAsset.TargetAssetPath, AssetName, tfm, buildConfiguration: buildConfiguration); - TestHostResult testHostResult = await testHost.ExecuteAsync(command: "--coverage --retry-failed-tests 3 --report-trx --crashdump --hangdump --report-azdo --report-html", cancellationToken: TestContext.CancellationToken); + TestHostResult testHostResult = await testHost.ExecuteAsync(command: "--coverage --retry-failed-tests 3 --report-trx --crashdump --hangdump --report-azdo --report-gh --report-html", cancellationToken: TestContext.CancellationToken); testHostResult.AssertOutputContainsSummary(0, 1, 0); } } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs index 3acec8b2c8..f28ac9cdd0 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs @@ -181,6 +181,18 @@ Enable generating a CTRF (Common Test Report Format) JSON report The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). Example: MyReport_{tfm}.ctrf.json + --report-gh + Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + --report-gh-annotations + Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + --report-gh-groups + Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + --report-gh-slow-test-notices + Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + --report-gh-slow-test-threshold + The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + --report-gh-step-summary + Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. --report-html Enable generating an HTML report --report-html-filename @@ -534,6 +546,35 @@ The file makes it possible to identify the tests that were running at the time o Description: The name of the generated CTRF report. May include a relative or absolute path; relative paths are resolved against the test results directory and missing directories are created. Supports the following placeholders: {pname} (test application name), {pid} (process ID), {asm} (entry assembly name), {tfm} (target framework moniker), {time} (timestamp). Example: MyReport_{tfm}.ctrf.json + GitHubActionsCommandLineProvider + Name: GitHub Actions report generator + Version: * + Description: GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + Options: + --report-gh + Arity: 0 + Hidden: False + Description: Enable GitHub Actions report generator to emit workflow commands so test runs produce a first-class experience on GitHub Actions. + --report-gh-annotations + Arity: 1 + Hidden: False + Description: Enable or disable GitHub Actions failure annotations. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + --report-gh-groups + Arity: 1 + Hidden: False + Description: Enable or disable per-assembly log groups. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + --report-gh-slow-test-notices + Arity: 1 + Hidden: False + Description: Enable or disable GitHub Actions slow-test notices. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. + --report-gh-slow-test-threshold + Arity: 1 + Hidden: False + Description: The duration a test may run before a GitHub Actions slow-test notice is emitted. Accepts a bare number of seconds or a value with a unit suffix such as '90s', '2m', or '1.5h'. Defaults to 60s. + --report-gh-step-summary + Arity: 1 + Hidden: False + Description: Enable or disable writing a markdown job summary to the GITHUB_STEP_SUMMARY file. Valid values are 'on' (also accepts 'true', 'enable', '1') or 'off' (also accepts 'false', 'disable', '0'). Defaults to 'on' when running on GitHub Actions. HangDumpCommandLineProvider Name: Hang dump Version: * @@ -729,6 +770,7 @@ public sealed class TestAssetFixture() : TestAssetFixtureBase() + @@ -795,6 +837,7 @@ public override (string ID, string Name, string Code) GetAssetsToGenerate() => ( .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) .PatchCodeWithReplace("$MicrosoftTestingExtensionsCtrfReportVersion$", MicrosoftTestingExtensionsCtrfReportVersion) .PatchCodeWithReplace("$MicrosoftTestingExtensionsJUnitReportVersion$", MicrosoftTestingExtensionsJUnitReportVersion) + .PatchCodeWithReplace("$MicrosoftTestingExtensionsGitHubActionsReportVersion$", MicrosoftTestingExtensionsGitHubActionsReportVersion) .PatchCodeWithReplace("$MicrosoftTestingExtensionsPackagedAppVersion$", MicrosoftTestingExtensionsPackagedAppVersion) .PatchCodeWithReplace("$MicrosoftTestingExtensionsVideoRecorderVersion$", MicrosoftTestingExtensionsVideoRecorderVersion) .PatchCodeWithReplace("$MicrosoftTestingExtensionsAzureFoundryVersion$", MicrosoftTestingExtensionsAzureFoundryVersion)); diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs index 4e94eef8ae..c690025824 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/Helpers/AcceptanceTestBase.cs @@ -20,6 +20,7 @@ static AcceptanceTestBase() MicrosoftTestingExtensionsLoggingVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "Microsoft.Testing.Extensions.Logging."); MicrosoftTestingExtensionsCtrfReportVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "Microsoft.Testing.Extensions.CtrfReport."); MicrosoftTestingExtensionsJUnitReportVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "Microsoft.Testing.Extensions.JUnitReport."); + MicrosoftTestingExtensionsGitHubActionsReportVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "Microsoft.Testing.Extensions.GitHubActionsReport."); MicrosoftTestingExtensionsPackagedAppVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "Microsoft.Testing.Extensions.PackagedApp."); MicrosoftTestingExtensionsVideoRecorderVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesShipping, "Microsoft.Testing.Extensions.VideoRecorder."); MicrosoftTestingExtensionsAzureFoundryVersion = ExtractVersionFromPackage(Constants.ArtifactsPackagesNonShipping, "Microsoft.Testing.Extensions.AzureFoundry."); @@ -48,6 +49,8 @@ static AcceptanceTestBase() public static string MicrosoftTestingExtensionsJUnitReportVersion { get; private set; } + public static string MicrosoftTestingExtensionsGitHubActionsReportVersion { get; private set; } + public static string MicrosoftTestingExtensionsPackagedAppVersion { get; private set; } public static string MicrosoftTestingExtensionsVideoRecorderVersion { get; private set; } diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs index 5f6bfed1d9..7783ac1927 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuild.KnownExtensionRegistration.cs @@ -22,6 +22,7 @@ public async Task Microsoft_Testing_Platform_Extensions_ShouldBe_Correctly_Regis .PatchCodeWithReplace("$TargetFrameworks$", tfm) .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion) .PatchCodeWithReplace("$MicrosoftTestingExtensionsCtrfReportVersion$", MicrosoftTestingExtensionsCtrfReportVersion) + .PatchCodeWithReplace("$MicrosoftTestingExtensionsGitHubActionsReportVersion$", MicrosoftTestingExtensionsGitHubActionsReportVersion) .PatchCodeWithReplace("$MicrosoftTestingExtensionsJUnitReportVersion$", MicrosoftTestingExtensionsJUnitReportVersion) .PatchCodeWithReplace("$MicrosoftTestingExtensionsVideoRecorderVersion$", MicrosoftTestingExtensionsVideoRecorderVersion)); DotnetMuxerResult result = await DotnetCli.RunAsync($"{(verb == Verb.publish ? $"publish -f {tfm}" : "build")} -c {compilationMode} -r {RID} {testAsset.TargetAssetPath} -v:n", cancellationToken: TestContext.CancellationToken); @@ -34,6 +35,7 @@ public async Task Microsoft_Testing_Platform_Extensions_ShouldBe_Correctly_Regis testHostResult.AssertOutputContains("--publish-azdo-run-name"); testHostResult.AssertOutputContains("--publish-azdo-test-results"); testHostResult.AssertOutputContains("--report-ctrf"); + testHostResult.AssertOutputContains("--report-gh"); testHostResult.AssertOutputContains("--report-html"); testHostResult.AssertOutputContains("--report-junit"); testHostResult.AssertOutputContains("--report-trx"); @@ -48,6 +50,7 @@ public async Task Microsoft_Testing_Platform_Extensions_ShouldBe_Correctly_Regis Assert.Contains("Microsoft.Testing.Extensions.AzureDevOpsReport.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.CrashDump.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.CtrfReport.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); + Assert.Contains("Microsoft.Testing.Extensions.GitHubActionsReport.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.HangDump.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.HotReload.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); Assert.Contains("Microsoft.Testing.Extensions.HtmlReport.TestingPlatformBuilderHook.AddExtensions", generatedSource.Text, generatedSource.Text); @@ -112,6 +115,7 @@ public async Task TestingPlatformBuilderHook_With_Conflicting_Metadata_Fails_Bui + diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsTestNodeIdentityTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsTestNodeIdentityTests.cs index 3fd223eec6..91c4758d45 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsTestNodeIdentityTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsTestNodeIdentityTests.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.Testing.Extensions.AzureDevOpsReport; using Microsoft.Testing.Platform.Extensions.Messages; namespace Microsoft.Testing.Extensions.UnitTests; diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsAnnotationReporterTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsAnnotationReporterTests.cs new file mode 100644 index 0000000000..4aeddaa797 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsAnnotationReporterTests.cs @@ -0,0 +1,137 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +extern alias ghactions; + +using ghactions::Microsoft.Testing.Extensions.GitHubActionsReport; + +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; + +using Moq; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public sealed class GitHubActionsAnnotationReporterTests +{ + [TestMethod] + public void GetErrorAnnotation_ReportsResolvedFileWithLineColTitleAndEscaping() + { + Exception error = CaptureException("this is an error\nwith\rnewline", out int throwLine); + + string text = GitHubActionsAnnotationReporter.GetErrorAnnotation( + "MyNamespace.MyTest", explanation: null, error, GitHubActionsRepositoryRoot.FindGitRoot(), CreateFileSystemWhereEveryFileExists(), new NoopLogger(), skipAssertionFrames: true); + + // The line is computed dynamically (from where the throw actually executes) rather than hard-coded, and the + // file existence is mocked, so the assertion does not depend on this file's exact layout or the physical + // repo checkout. + Assert.IsTrue(text.StartsWith("::error file=", StringComparison.Ordinal), text); + Assert.Contains($"GitHubActionsAnnotationReporterTests.cs,line={throwLine},col=1,title=Test failed%3A MyNamespace.MyTest::", text); + Assert.IsTrue(text.EndsWith("this is an error%0Awith%0Dnewline", StringComparison.Ordinal), text); + } + + [TestMethod] + public void GetErrorAnnotation_PrefersExplanationOverExceptionMessage() + { + Exception error = CaptureException("exception message", out int throwLine); + + string text = GitHubActionsAnnotationReporter.GetErrorAnnotation( + "MyNamespace.MyTest", "Some custom reason\nwith\rnewline", error, GitHubActionsRepositoryRoot.FindGitRoot(), CreateFileSystemWhereEveryFileExists(), new NoopLogger(), skipAssertionFrames: true); + + Assert.IsTrue(text.StartsWith("::error file=", StringComparison.Ordinal), text); + Assert.Contains($"GitHubActionsAnnotationReporterTests.cs,line={throwLine},col=1,title=Test failed%3A MyNamespace.MyTest::", text); + Assert.IsTrue(text.EndsWith("Some custom reason%0Awith%0Dnewline", StringComparison.Ordinal), text); + } + + [TestMethod] + public void GetErrorAnnotation_FallsBackToTitleOnly_WhenNoSourceLocation() + { + string text = GitHubActionsAnnotationReporter.GetErrorAnnotation("MyNamespace.MyTest", "boom", exception: null, repoRoot: null, CreateFileSystemWhereEveryFileExists(), new NoopLogger(), skipAssertionFrames: true); + + Assert.AreEqual("::error title=Test failed%3A MyNamespace.MyTest::boom", text); + } + + [TestMethod] + public void GetErrorAnnotation_RemapsDeterministicBuildRootPathToWorkspaceRelative() + { + // A frame emitted from a deterministic (source-linked) build carries the '/_/' root marker; the reporter + // must strip it and produce a forward-slash workspace-relative path regardless of the repo root value. + var exception = new StackTraceException(" at Contoso.Calc.Add() in /_/src/Calc.cs:line 12"); + + string text = GitHubActionsAnnotationReporter.GetErrorAnnotation( + "Contoso.CalcTests.Add", "boom", exception, repoRoot: "/repo/", CreateFileSystemWhereEveryFileExists(), new NoopLogger(), skipAssertionFrames: true); + + Assert.AreEqual("::error file=src/Calc.cs,line=12,col=1,title=Test failed%3A Contoso.CalcTests.Add::boom", text); + } + + [TestMethod] + public void GetErrorAnnotation_SkipsAssertionFramesAndAnnotatesUserCallSite() + { + // The top frame is an MSTest assertion implementation and must be skipped in favour of the user's call site. + var exception = new StackTraceException( + " at Microsoft.VisualStudio.TestTools.UnitTesting.Assert.Fail(string message) in /_/assert/Assert.cs:line 1\n" + + " at Contoso.MyTests.TheTest() in /_/src/MyTests.cs:line 7"); + + string text = GitHubActionsAnnotationReporter.GetErrorAnnotation( + "Contoso.MyTests.TheTest", "nope", exception, repoRoot: "/repo/", CreateFileSystemWhereEveryFileExists(), new NoopLogger(), skipAssertionFrames: true); + + Assert.AreEqual("::error file=src/MyTests.cs,line=7,col=1,title=Test failed%3A Contoso.MyTests.TheTest::nope", text); + } + + [TestMethod] + public void GetErrorAnnotation_UsesFallbackMessage_WhenNoExplanationOrException() + { + string text = GitHubActionsAnnotationReporter.GetErrorAnnotation("MyNamespace.MyTest", explanation: null, exception: null, repoRoot: null, CreateFileSystemWhereEveryFileExists(), new NoopLogger(), skipAssertionFrames: true); + + Assert.IsTrue(text.StartsWith("::error title=Test failed%3A MyNamespace.MyTest::", StringComparison.Ordinal), text); + } + + // Throws (and catches) an exception, reporting the exact line of the throw statement so tests can assert the + // resolved line without hard-coding a physical number that shifts whenever code above changes. + private static Exception CaptureException(string message, out int throwLine) + { + throwLine = 0; + try + { + throwLine = CurrentLine() + 1; + throw new Exception(message); + } + catch (Exception ex) + { + return ex; + } + } + + private static int CurrentLine([CallerLineNumber] int line = 0) => line; + + private static IFileSystem CreateFileSystemWhereEveryFileExists() + { + var fileSystem = new Mock(); + fileSystem.Setup(f => f.ExistFile(It.IsAny())).Returns(true); + return fileSystem.Object; + } + + // Exception whose StackTrace is a caller-supplied synthetic string, letting tests exercise the + // frame-parsing/path-remapping branches deterministically without relying on real PDB-derived paths. + private sealed class StackTraceException : Exception + { + private readonly string _stackTrace; + + public StackTraceException(string stackTrace) => _stackTrace = stackTrace; + + public override string? StackTrace => _stackTrace; + } + + private sealed class NoopLogger : ILogger + { + public bool IsEnabled(LogLevel logLevel) => false; + + public void Log(LogLevel logLevel, TState state, Exception? exception, Func formatter) + { + } + + public Task LogAsync(LogLevel logLevel, TState state, Exception? exception, Func formatter) + => Task.CompletedTask; + } +} diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsCommandLineProviderTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsCommandLineProviderTests.cs new file mode 100644 index 0000000000..c77e51922c --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsCommandLineProviderTests.cs @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +extern alias ghactions; + +using ghactions::Microsoft.Testing.Extensions.GitHubActionsReport; + +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions.CommandLine; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public sealed class GitHubActionsCommandLineProviderTests +{ + [TestMethod] + public async Task ValidateOptionArgumentsAsync_ReturnsInvalid_WhenGroupsValueIsNotOnOrOffAsync() + { + GitHubActionsCommandLineProvider provider = new(); + CommandLineOption option = provider.GetCommandLineOptions().Single(o => o.Name == GitHubActionsCommandLineOptions.GitHubActionsGroups); + ValidationResult validationResult = await provider.ValidateOptionArgumentsAsync(option, ["maybe"]).ConfigureAwait(false); + + Assert.IsFalse(validationResult.IsValid); + } + + [TestMethod] + public async Task ValidateOptionArgumentsAsync_ReturnsValid_WhenGroupsValueIsOffAsync() + { + GitHubActionsCommandLineProvider provider = new(); + CommandLineOption option = provider.GetCommandLineOptions().Single(o => o.Name == GitHubActionsCommandLineOptions.GitHubActionsGroups); + ValidationResult validationResult = await provider.ValidateOptionArgumentsAsync(option, ["off"]).ConfigureAwait(false); + + Assert.IsTrue(validationResult.IsValid); + } + + [TestMethod] + public async Task ValidateOptionArgumentsAsync_ReturnsInvalid_WhenAnnotationsValueIsNotOnOrOffAsync() + { + GitHubActionsCommandLineProvider provider = new(); + CommandLineOption option = provider.GetCommandLineOptions().Single(o => o.Name == GitHubActionsCommandLineOptions.GitHubActionsAnnotations); + ValidationResult validationResult = await provider.ValidateOptionArgumentsAsync(option, ["maybe"]).ConfigureAwait(false); + + Assert.IsFalse(validationResult.IsValid); + } + + [TestMethod] + public async Task ValidateOptionArgumentsAsync_ReturnsValid_WhenAnnotationsValueIsOffAsync() + { + GitHubActionsCommandLineProvider provider = new(); + CommandLineOption option = provider.GetCommandLineOptions().Single(o => o.Name == GitHubActionsCommandLineOptions.GitHubActionsAnnotations); + ValidationResult validationResult = await provider.ValidateOptionArgumentsAsync(option, ["off"]).ConfigureAwait(false); + + Assert.IsTrue(validationResult.IsValid); + } + + [TestMethod] + public async Task ValidateOptionArgumentsAsync_ReturnsInvalid_WhenStepSummaryValueIsNotOnOrOffAsync() + { + GitHubActionsCommandLineProvider provider = new(); + CommandLineOption option = provider.GetCommandLineOptions().Single(o => o.Name == GitHubActionsCommandLineOptions.GitHubActionsStepSummary); + ValidationResult validationResult = await provider.ValidateOptionArgumentsAsync(option, ["maybe"]).ConfigureAwait(false); + + Assert.IsFalse(validationResult.IsValid); + } + + [TestMethod] + public async Task ValidateOptionArgumentsAsync_ReturnsInvalid_WhenSlowTestNoticesValueIsNotOnOrOffAsync() + { + GitHubActionsCommandLineProvider provider = new(); + CommandLineOption option = provider.GetCommandLineOptions().Single(o => o.Name == GitHubActionsCommandLineOptions.GitHubActionsSlowTestNotices); + ValidationResult validationResult = await provider.ValidateOptionArgumentsAsync(option, ["maybe"]).ConfigureAwait(false); + + Assert.IsFalse(validationResult.IsValid); + } + + [TestMethod] + public async Task ValidateOptionArgumentsAsync_ReturnsInvalid_WhenSlowTestThresholdIsNotPositiveIntegerAsync() + { + GitHubActionsCommandLineProvider provider = new(); + CommandLineOption option = provider.GetCommandLineOptions().Single(o => o.Name == GitHubActionsCommandLineOptions.GitHubActionsSlowTestThreshold); + ValidationResult validationResult = await provider.ValidateOptionArgumentsAsync(option, ["0"]).ConfigureAwait(false); + + Assert.IsFalse(validationResult.IsValid); + } + + [TestMethod] + public async Task ValidateOptionArgumentsAsync_ReturnsValid_WhenSlowTestThresholdIsPositiveIntegerAsync() + { + GitHubActionsCommandLineProvider provider = new(); + CommandLineOption option = provider.GetCommandLineOptions().Single(o => o.Name == GitHubActionsCommandLineOptions.GitHubActionsSlowTestThreshold); + ValidationResult validationResult = await provider.ValidateOptionArgumentsAsync(option, ["30"]).ConfigureAwait(false); + + Assert.IsTrue(validationResult.IsValid); + } + + [TestMethod] + [DataRow("true")] + [DataRow("enable")] + [DataRow("1")] + [DataRow("false")] + [DataRow("disable")] + [DataRow("0")] + public async Task ValidateOptionArgumentsAsync_ReturnsValid_ForBooleanAliasesAsync(string value) + { + GitHubActionsCommandLineProvider provider = new(); + CommandLineOption option = provider.GetCommandLineOptions().Single(o => o.Name == GitHubActionsCommandLineOptions.GitHubActionsGroups); + ValidationResult validationResult = await provider.ValidateOptionArgumentsAsync(option, [value]).ConfigureAwait(false); + + Assert.IsTrue(validationResult.IsValid, value); + } + + [TestMethod] + [DataRow("90s")] + [DataRow("2m")] + [DataRow("1.5h")] + [DataRow("60")] + public async Task ValidateOptionArgumentsAsync_ReturnsValid_ForSlowTestThresholdDurationsAsync(string value) + { + GitHubActionsCommandLineProvider provider = new(); + CommandLineOption option = provider.GetCommandLineOptions().Single(o => o.Name == GitHubActionsCommandLineOptions.GitHubActionsSlowTestThreshold); + ValidationResult validationResult = await provider.ValidateOptionArgumentsAsync(option, [value]).ConfigureAwait(false); + + Assert.IsTrue(validationResult.IsValid, value); + } + + [TestMethod] + [DataRow("0s")] + [DataRow("abc")] + [DataRow("-5s")] + public async Task ValidateOptionArgumentsAsync_ReturnsInvalid_ForNonPositiveOrUnparseableThresholdAsync(string value) + { + GitHubActionsCommandLineProvider provider = new(); + CommandLineOption option = provider.GetCommandLineOptions().Single(o => o.Name == GitHubActionsCommandLineOptions.GitHubActionsSlowTestThreshold); + ValidationResult validationResult = await provider.ValidateOptionArgumentsAsync(option, [value]).ConfigureAwait(false); + + Assert.IsFalse(validationResult.IsValid, value); + } +} diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsEscaperTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsEscaperTests.cs new file mode 100644 index 0000000000..bd2afb4b46 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsEscaperTests.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +extern alias ghactions; + +using ghactions::Microsoft.Testing.Extensions.GitHubActionsReport; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public sealed class GitHubActionsEscaperTests +{ + [TestMethod] + public void EscapeData_EscapesPercentAndNewlines() + => Assert.AreEqual("a%25b%0Ac%0Dd", GitHubActionsEscaper.EscapeData("a%b\nc\rd")); + + [TestMethod] + public void EscapeData_LeavesPlainTextUntouched() + => Assert.AreEqual("Tests: MSTest.UnitTests (net9.0)", GitHubActionsEscaper.EscapeData("Tests: MSTest.UnitTests (net9.0)")); + + [TestMethod] + public void EscapeData_ReturnsEmptyForEmpty() + => Assert.AreEqual(string.Empty, GitHubActionsEscaper.EscapeData(string.Empty)); + + [TestMethod] + public void EscapeProperty_EscapesPercentNewlinesColonAndComma() + => Assert.AreEqual("a%25b%0Ac%0Dd%3Ae%2Cf", GitHubActionsEscaper.EscapeProperty("a%b\nc\rd:e,f")); + + [TestMethod] + public void EscapeProperty_LeavesPlainTextUntouched() + => Assert.AreEqual("Test failed", GitHubActionsEscaper.EscapeProperty("Test failed")); +} diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsReporterTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsReporterTests.cs new file mode 100644 index 0000000000..c2c4e7c098 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsReporterTests.cs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +extern alias ghactions; + +using ghactions::Microsoft.Testing.Extensions.GitHubActionsReport; + +using Microsoft.Testing.Extensions.UnitTests.Helpers; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Extensions.TestHost; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; + +using Moq; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public sealed class GitHubActionsReporterTests +{ + [TestMethod] + public async Task IsEnabledAsync_ReturnsTrue_WhenRunningOnGitHubActionsAsync() + { + GitHubActionsReporter reporter = CreateReporter(githubActions: true, options: []); + Assert.IsTrue(await reporter.IsEnabledAsync().ConfigureAwait(false)); + } + + [TestMethod] + public async Task IsEnabledAsync_ReturnsFalse_WhenNotOnGitHubActionsAsync() + { + GitHubActionsReporter reporter = CreateReporter(githubActions: false, options: []); + Assert.IsFalse(await reporter.IsEnabledAsync().ConfigureAwait(false)); + } + + [TestMethod] + public async Task IsEnabledAsync_ReturnsFalse_WhenGroupsExplicitlyOffAsync() + { + GitHubActionsReporter reporter = CreateReporter(githubActions: true, options: new Dictionary + { + [GitHubActionsCommandLineOptions.GitHubActionsGroups] = ["off"], + }); + Assert.IsFalse(await reporter.IsEnabledAsync().ConfigureAwait(false)); + } + + [TestMethod] + public async Task SessionLifetime_EmitsGroupAndEndGroupAsync() + { + List output = []; + GitHubActionsReporter reporter = CreateReporter(githubActions: true, options: [], output, assemblyName: "MSTest.UnitTests"); + + var context = new Mock(); + context.SetupGet(c => c.CancellationToken).Returns(CancellationToken.None); + + await reporter.OnTestSessionStartingAsync(context.Object).ConfigureAwait(false); + await reporter.OnTestSessionFinishingAsync(context.Object).ConfigureAwait(false); + + Assert.HasCount(2, output); + Assert.IsTrue(output[0].StartsWith("::group::Tests: MSTest.UnitTests", StringComparison.Ordinal), output[0]); + Assert.AreEqual("::endgroup::", output[1]); + } + + private static GitHubActionsReporter CreateReporter(bool githubActions, Dictionary options, List? output = null, string assemblyName = "Some.UnitTests") + { + var environment = new Mock(); + environment.Setup(e => e.GetEnvironmentVariable("GITHUB_ACTIONS")).Returns(githubActions ? "true" : null); + + // The extension is enabled only when both the GITHUB_ACTIONS env var and the --report-gh master switch + // are set, so always seed the master switch here; these tests exercise the env/knob behavior on top of it. + var commandLineOptions = new Dictionary(options, StringComparer.OrdinalIgnoreCase) + { + [GitHubActionsCommandLineOptions.GitHubActionsOptionName] = [], + }; + + var outputDevice = new Mock(); + outputDevice + .Setup(o => o.DisplayAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, data, _) => output?.Add(((FormattedTextOutputDeviceData)data).Text)) + .Returns(Task.CompletedTask); + + var moduleInfo = new Mock(); + moduleInfo.Setup(m => m.TryGetAssemblyName()).Returns(assemblyName); + + var logger = new Mock(); + var loggerFactory = new Mock(); + loggerFactory.Setup(f => f.CreateLogger(It.IsAny())).Returns(logger.Object); + + return new GitHubActionsReporter( + new TestCommandLineOptions(commandLineOptions), + environment.Object, + outputDevice.Object, + moduleInfo.Object, + loggerFactory.Object); + } +} diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSlowTestReporterTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSlowTestReporterTests.cs new file mode 100644 index 0000000000..bdee744e2b --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSlowTestReporterTests.cs @@ -0,0 +1,239 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +extern alias ghactions; + +using ghactions::Microsoft.Testing.Extensions.GitHubActionsReport; + +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; +using Microsoft.Testing.Platform.TestHost; + +using Moq; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public sealed class GitHubActionsSlowTestReporterTests +{ + private static readonly DateTimeOffset Start = new(2025, 05, 16, 12, 00, 00, TimeSpan.Zero); + + [TestMethod] + public void BuildNoticeLine_FormatsNoticeWithEscapedTitleAndMessage() + { + string line = GitHubActionsSlowTestReporter.BuildNoticeLine("Ns.MyTest", TimeSpan.FromSeconds(75)); + + Assert.AreEqual("::notice title=Slow test%3A Ns.MyTest::Ns.MyTest still running after 75s", line); + } + + [TestMethod] + public async Task ScanOnce_AfterThreshold_EmitsSingleNoticeAsync() + { + CapturingOutputDevice outputDevice = new(); + GitHubActionsSlowTestReporter reporter = CreateReporter(outputDevice, githubActions: true); + await reporter.OnTestSessionStartingAsync(new TestSessionContextStub()).ConfigureAwait(false); + + await reporter.ConsumeAsync(null!, CreateMessage("u1", "Ns.T1", new InProgressTestNodeStateProperty()), CancellationToken.None).ConfigureAwait(false); + + // Default threshold is 60s; nothing before it. + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(30), CancellationToken.None).ConfigureAwait(false); + Assert.IsEmpty(outputDevice.Lines); + + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(90), CancellationToken.None).ConfigureAwait(false); + Assert.HasCount(1, outputDevice.Lines); + Assert.Contains("::notice", outputDevice.Lines[0]); + Assert.Contains("Ns.T1 still running after", outputDevice.Lines[0]); + } + + [TestMethod] + public async Task ScanOnce_UsesExponentialBackoffAsync() + { + CapturingOutputDevice outputDevice = new(); + GitHubActionsSlowTestReporter reporter = CreateReporter(outputDevice, githubActions: true); + await reporter.OnTestSessionStartingAsync(new TestSessionContextStub()).ConfigureAwait(false); + await reporter.ConsumeAsync(null!, CreateMessage("u1", "Ns.T1", new InProgressTestNodeStateProperty()), CancellationToken.None).ConfigureAwait(false); + + // First emit at >= 60s; threshold then doubles to 120s, so 90s does not emit again. + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(70), CancellationToken.None).ConfigureAwait(false); + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(90), CancellationToken.None).ConfigureAwait(false); + Assert.HasCount(1, outputDevice.Lines); + + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(130), CancellationToken.None).ConfigureAwait(false); + Assert.HasCount(2, outputDevice.Lines); + } + + [TestMethod] + public async Task ConsumeAsync_TerminalState_StopsTrackingAsync() + { + CapturingOutputDevice outputDevice = new(); + GitHubActionsSlowTestReporter reporter = CreateReporter(outputDevice, githubActions: true); + await reporter.OnTestSessionStartingAsync(new TestSessionContextStub()).ConfigureAwait(false); + + await reporter.ConsumeAsync(null!, CreateMessage("u1", "Ns.T1", new InProgressTestNodeStateProperty()), CancellationToken.None).ConfigureAwait(false); + await reporter.ConsumeAsync(null!, CreateMessage("u1", "Ns.T1", new PassedTestNodeStateProperty()), CancellationToken.None).ConfigureAwait(false); + + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(120), CancellationToken.None).ConfigureAwait(false); + Assert.IsEmpty(outputDevice.Lines); + } + + [TestMethod] + public async Task ScanOnce_WhenDisabled_DoesNotEmitAsync() + { + CapturingOutputDevice outputDevice = new(); + GitHubActionsSlowTestReporter reporter = CreateReporter(outputDevice, githubActions: false); + await reporter.OnTestSessionStartingAsync(new TestSessionContextStub()).ConfigureAwait(false); + + await reporter.ConsumeAsync(null!, CreateMessage("u1", "Ns.T1", new InProgressTestNodeStateProperty()), CancellationToken.None).ConfigureAwait(false); + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(120), CancellationToken.None).ConfigureAwait(false); + + Assert.IsEmpty(outputDevice.Lines); + } + + [TestMethod] + public async Task ScanOnce_RespectsConfiguredThresholdAsync() + { + CapturingOutputDevice outputDevice = new(); + Dictionary options = new(StringComparer.OrdinalIgnoreCase) + { + [GitHubActionsCommandLineOptions.GitHubActionsSlowTestThreshold] = ["2"], + }; + GitHubActionsSlowTestReporter reporter = CreateReporter(outputDevice, githubActions: true, options); + await reporter.OnTestSessionStartingAsync(new TestSessionContextStub()).ConfigureAwait(false); + await reporter.ConsumeAsync(null!, CreateMessage("u1", "Ns.T1", new InProgressTestNodeStateProperty()), CancellationToken.None).ConfigureAwait(false); + + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(1), CancellationToken.None).ConfigureAwait(false); + Assert.IsEmpty(outputDevice.Lines); + + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(3), CancellationToken.None).ConfigureAwait(false); + Assert.HasCount(1, outputDevice.Lines); + } + + private static GitHubActionsSlowTestReporter CreateReporter(CapturingOutputDevice outputDevice, bool githubActions, Dictionary? options = null) + { + Mock environmentMock = new(); + environmentMock.Setup(x => x.GetEnvironmentVariable("GITHUB_ACTIONS")).Returns(githubActions ? "true" : null); + + // The extension is enabled only when both the GITHUB_ACTIONS env var and the --report-gh master switch + // are set, so always seed the master switch here; these tests exercise the env/knob behavior on top of it. + Dictionary commandLineOptions = options is null + ? new(StringComparer.OrdinalIgnoreCase) + : new(options, StringComparer.OrdinalIgnoreCase); + commandLineOptions[GitHubActionsCommandLineOptions.GitHubActionsOptionName] = []; + + return new GitHubActionsSlowTestReporter( + new FakeCommandLineOptions(commandLineOptions), + environmentMock.Object, + outputDevice, + new NonRunningTask(), + new FixedClock(Start), + new StubLoggerFactory()); + } + + private static TestNodeUpdateMessage CreateMessage(string uid, string fullyQualifiedName, TestNodeStateProperty state) + { + PropertyBag propertyBag = new(); + propertyBag.Add(state); + propertyBag.Add(new SerializableKeyValuePairStringProperty("vstest.TestCase.FullyQualifiedName", fullyQualifiedName)); + + return new TestNodeUpdateMessage(new SessionUid("session"), new TestNode + { + Uid = uid, + DisplayName = fullyQualifiedName, + Properties = propertyBag, + }); + } + + private sealed class FakeCommandLineOptions(IReadOnlyDictionary options) : ICommandLineOptions + { + private readonly IReadOnlyDictionary _options = options; + + public bool IsOptionSet(string optionName) + => _options.ContainsKey(optionName); + + public bool TryGetOptionArgumentList(string optionName, [NotNullWhen(true)] out string[]? arguments) + { + if (_options.TryGetValue(optionName, out string[]? values)) + { + arguments = values; + return true; + } + + arguments = null; + return false; + } + } + + private sealed class CapturingOutputDevice : IOutputDevice + { + public List Lines { get; } = []; + + public Task DisplayAsync(IOutputDeviceDataProducer producer, IOutputDeviceData data, CancellationToken cancellationToken) + { + Lines.Add(((TextOutputDeviceData)data).Text); + return Task.CompletedTask; + } + } + + private sealed class FixedClock(DateTimeOffset now) : IClock + { + public DateTimeOffset UtcNow { get; } = now; + } + + private sealed class NonRunningTask : ITask + { + public Task Delay(int millisecondDelay) + => Task.CompletedTask; + + public Task Delay(TimeSpan timeSpan, CancellationToken cancellationToken) + => Task.CompletedTask; + + public Task Run(Action action) + { + action(); + return Task.CompletedTask; + } + + public Task Run(Func function, CancellationToken cancellationToken) + => function(); + + public Task Run(Func?> function, CancellationToken cancellationToken) + => function()!; + + public Task RunLongRunning(Func action, string name, CancellationToken cancellationToken) + => Task.CompletedTask; + + public Task WhenAll(params Task[] tasks) + => Task.WhenAll(tasks); + } + + private sealed class TestSessionContextStub : ITestSessionContext + { + public SessionUid SessionUid { get; } = new("session"); + + public CancellationToken CancellationToken { get; } = CancellationToken.None; + } + + private sealed class StubLoggerFactory : ILoggerFactory + { + public ILogger CreateLogger(string categoryName) + => new NullLogger(); + + private sealed class NullLogger : ILogger + { + public bool IsEnabled(LogLevel logLevel) + => false; + + public void Log(LogLevel logLevel, TState state, Exception? exception, Func formatter) + { + } + + public Task LogAsync(LogLevel logLevel, TState state, Exception? exception, Func formatter) + => Task.CompletedTask; + } + } +} diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs new file mode 100644 index 0000000000..06ac66715e --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsSummaryReporterTests.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +extern alias ghactions; + +using ghactions::Microsoft.Testing.Extensions.GitHubActionsReport; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public sealed class GitHubActionsSummaryReporterTests +{ + [TestMethod] + public void BuildMarkdown_AllPassing_UsesSuccessIconAndTotals() + { + GitHubActionsSummaryReporter.TestRecord[] records = + [ + new("Add", "CalculatorTests.Add", GitHubActionsSummaryReporter.TerminalKind.Passed, TimeSpan.FromMilliseconds(10)), + new("Sub", "CalculatorTests.Sub", GitHubActionsSummaryReporter.TerminalKind.Passed, TimeSpan.FromMilliseconds(20)), + new("Skip", "CalculatorTests.Skip", GitHubActionsSummaryReporter.TerminalKind.Skipped, TimeSpan.Zero), + ]; + + string markdown = GitHubActionsSummaryReporter.BuildMarkdown(records, "CalculatorTests", "net9.0"); + + Assert.Contains("## ✅ Test Run Summary — CalculatorTests (net9.0)", markdown); + Assert.Contains("| 3 | 2 | 0 | 1 | 30ms |", markdown); + Assert.DoesNotContain("### ❌ Failures", markdown); + } + + [TestMethod] + public void BuildMarkdown_WithFailures_UsesFailureIconAndListsFailures() + { + GitHubActionsSummaryReporter.TestRecord[] records = + [ + new("Pass", "StringUtilsTests.Pass", GitHubActionsSummaryReporter.TerminalKind.Passed, TimeSpan.FromMilliseconds(5)), + new("Boom", "StringUtilsTests.Boom", GitHubActionsSummaryReporter.TerminalKind.Failed, TimeSpan.FromMilliseconds(7)), + ]; + + string markdown = GitHubActionsSummaryReporter.BuildMarkdown(records, "StringUtilsTests", "net9.0"); + + Assert.Contains("## ❌ Test Run Summary — StringUtilsTests (net9.0)", markdown); + Assert.Contains("### ❌ Failures (1)", markdown); + Assert.Contains("- `StringUtilsTests.Boom`", markdown); + } + + [TestMethod] + public void BuildMarkdown_EmitsSlowestTestsSortedByDuration() + { + GitHubActionsSummaryReporter.TestRecord[] records = + [ + new("Fast", "T.Fast", GitHubActionsSummaryReporter.TerminalKind.Passed, TimeSpan.FromMilliseconds(10)), + new("Slow", "T.Slow", GitHubActionsSummaryReporter.TerminalKind.Passed, TimeSpan.FromSeconds(65)), + ]; + + string markdown = GitHubActionsSummaryReporter.BuildMarkdown(records, "T", "net9.0"); + + Assert.Contains("### ⏱ Slowest tests", markdown); + int slowIndex = markdown.IndexOf("- `T.Slow` — 1m 05s", StringComparison.Ordinal); + int fastIndex = markdown.IndexOf("- `T.Fast` — 10ms", StringComparison.Ordinal); + Assert.IsGreaterThanOrEqualTo(0, slowIndex, markdown); + Assert.IsGreaterThanOrEqualTo(0, fastIndex, markdown); + + // Slowest-first ordering: the slow test must be listed before the fast one, i.e. at a smaller index. + // IsLessThan(upperBound, value) asserts value < upperBound, so this asserts slowIndex < fastIndex. + Assert.IsLessThan(fastIndex, slowIndex, markdown); + } + + [TestMethod] + public void BuildMarkdown_NoTests_StillEmitsHeaderAndZeroTotals() + { + string markdown = GitHubActionsSummaryReporter.BuildMarkdown([], "Empty", "net9.0"); + + Assert.Contains("## ✅ Test Run Summary — Empty (net9.0)", markdown); + Assert.Contains("| 0 | 0 | 0 | 0 | 0ms |", markdown); + } +} diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj index 52c551cfa3..488ab3aa63 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/Microsoft.Testing.Extensions.UnitTests.csproj @@ -36,6 +36,10 @@ TargetFramework=netstandard2.0 + + ghactions + TargetFramework=netstandard2.0 + TargetFramework=netstandard2.0