From 18d3c114454a897a728c71417de57b3fedf29193 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Sat, 16 May 2026 20:13:51 +0200 Subject: [PATCH] Add Azure DevOps flaky history reporting Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Directory.Packages.props | 1 + .../AzureDevOpsCommandLineOptions.cs | 3 + .../AzureDevOpsCommandLineProvider.cs | 58 ++- .../AzureDevOpsExtensions.cs | 37 +- .../AzureDevOpsHistoryClient.cs | 392 +++++++++++++++ .../AzureDevOpsHistoryClientJsonContext.cs | 10 + .../AzureDevOpsHistoryService.cs | 266 ++++++++++ .../AzureDevOpsReporter.cs | 193 +++++--- .../FlakyStats.cs | 21 + .../IAzureDevOpsHistoryService.cs | 13 + ...esting.Extensions.AzureDevOpsReport.csproj | 5 + .../QuarantineFile.cs | 88 ++++ .../Resources/AzureDevOpsResources.resx | 72 ++- .../Resources/xlf/AzureDevOpsResources.cs.xlf | 108 +++- .../Resources/xlf/AzureDevOpsResources.de.xlf | 108 +++- .../Resources/xlf/AzureDevOpsResources.es.xlf | 108 +++- .../Resources/xlf/AzureDevOpsResources.fr.xlf | 108 +++- .../Resources/xlf/AzureDevOpsResources.it.xlf | 108 +++- .../Resources/xlf/AzureDevOpsResources.ja.xlf | 108 +++- .../Resources/xlf/AzureDevOpsResources.ko.xlf | 108 +++- .../Resources/xlf/AzureDevOpsResources.pl.xlf | 108 +++- .../xlf/AzureDevOpsResources.pt-BR.xlf | 108 +++- .../Resources/xlf/AzureDevOpsResources.ru.xlf | 108 +++- .../Resources/xlf/AzureDevOpsResources.tr.xlf | 108 +++- .../xlf/AzureDevOpsResources.zh-Hans.xlf | 108 +++- .../xlf/AzureDevOpsResources.zh-Hant.xlf | 108 +++- .../AzureDevOpsCommandLineTests.cs | 107 ++++ .../HelpInfoAllExtensionsTests.cs | 36 ++ .../AzureDevOpsCommandLineProviderTests.cs | 97 ++++ .../AzureDevOpsHistoryClientTests.cs | 112 +++++ .../AzureDevOpsHistoryServiceTests.cs | 461 ++++++++++++++++++ 31 files changed, 3238 insertions(+), 138 deletions(-) create mode 100644 src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryClient.cs create mode 100644 src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryClientJsonContext.cs create mode 100644 src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryService.cs create mode 100644 src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/FlakyStats.cs create mode 100644 src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/IAzureDevOpsHistoryService.cs create mode 100644 src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/QuarantineFile.cs create mode 100644 test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/AzureDevOpsCommandLineTests.cs create mode 100644 test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsCommandLineProviderTests.cs create mode 100644 test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsHistoryClientTests.cs create mode 100644 test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsHistoryServiceTests.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 25e873c44f..00779f72ca 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -59,6 +59,7 @@ + diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineOptions.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineOptions.cs index bfe0b50025..54a7b1f96d 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineOptions.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineOptions.cs @@ -6,5 +6,8 @@ namespace Microsoft.Testing.Extensions.Reporting; internal static class AzureDevOpsCommandLineOptions { public const string AzureDevOpsOptionName = "report-azdo"; + public const string AzureDevOpsDemoteKnownFlaky = "report-azdo-demote-known-flaky"; + public const string AzureDevOpsFlakyHistory = "report-azdo-flaky-history"; + public const string AzureDevOpsQuarantineFile = "report-azdo-quarantine-file"; public const string AzureDevOpsReportSeverity = "report-azdo-severity"; } diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineProvider.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineProvider.cs index 85598bc581..8ae4c21ede 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineProvider.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineProvider.cs @@ -26,31 +26,61 @@ public IReadOnlyCollection GetCommandLineOptions() => [ new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName, AzureDevOpsResources.OptionDescription, ArgumentArity.Zero, false), + new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsDemoteKnownFlaky, AzureDevOpsResources.DemoteKnownFlakyOptionDescription, ArgumentArity.Zero, false), + new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory, AzureDevOpsResources.FlakyHistoryOptionDescription, ArgumentArity.ExactlyOne, false), + new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsQuarantineFile, AzureDevOpsResources.QuarantineFileOptionDescription, ArgumentArity.ExactlyOne, false), new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity, AzureDevOpsResources.SeverityOptionDescription, ArgumentArity.ExactlyOne, false), ]; public Task ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments) + => commandOption.Name switch + { + AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory => ValidateFlakyHistoryArgumentsAsync(arguments), + AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity => ValidateSeverityArgumentsAsync(arguments), + _ => ValidationResult.ValidTask, + }; + + public Task ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions) { - if (commandOption.Name == AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity) + string? errorMessage = null; + if (!commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName)) { - if (!SeverityOptions.Contains(arguments[0], StringComparer.OrdinalIgnoreCase)) + if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsDemoteKnownFlaky)) + { + errorMessage = AzureDevOpsResources.AzureDevOpsDemoteKnownFlakyRequiresAzureDevOps; + } + else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory)) + { + errorMessage = AzureDevOpsResources.AzureDevOpsFlakyHistoryRequiresAzureDevOps; + } + else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsQuarantineFile)) + { + errorMessage = AzureDevOpsResources.AzureDevOpsQuarantineFileRequiresAzureDevOps; + } + else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity)) { - return ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSeverity, arguments[0])); + errorMessage = AzureDevOpsResources.AzureDevOpsReportSeverityRequiresAzureDevOps; } } - - return ValidationResult.ValidTask; - } - - public Task ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions) - { - if (!commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName) && - commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity)) + else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsDemoteKnownFlaky) + && !commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory)) { - // If report-azdo is not set, but report-azdo-severity is set, it's invalid. - return ValidationResult.InvalidTask(AzureDevOpsResources.AzureDevOpsReportSeverityRequiresAzureDevOps); + errorMessage = AzureDevOpsResources.AzureDevOpsDemoteKnownFlakyRequiresFlakyHistory; } - return ValidationResult.ValidTask; + return errorMessage is null + ? ValidationResult.ValidTask + : ValidationResult.InvalidTask(errorMessage); } + + private static Task ValidateFlakyHistoryArgumentsAsync(string[] arguments) + => int.TryParse(arguments[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out int days) + && days is >= 1 and <= 90 + ? ValidationResult.ValidTask + : ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidFlakyHistoryDays, arguments[0])); + + private static Task ValidateSeverityArgumentsAsync(string[] arguments) + => SeverityOptions.Contains(arguments[0], StringComparer.OrdinalIgnoreCase) + ? ValidationResult.ValidTask + : ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSeverity, arguments[0])); } diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsExtensions.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsExtensions.cs index 728a257520..584383f9ec 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsExtensions.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsExtensions.cs @@ -4,7 +4,6 @@ using Microsoft.Testing.Extensions.AzureDevOpsReport; using Microsoft.Testing.Extensions.Reporting; using Microsoft.Testing.Platform.Builder; -using Microsoft.Testing.Platform.Extensions; using Microsoft.Testing.Platform.Services; namespace Microsoft.Testing.Extensions; @@ -20,17 +19,33 @@ public static class AzureDevOpsExtensions /// The test application builder. public static void AddAzureDevOpsProvider(this ITestApplicationBuilder builder) { - var compositeTestSessionAzDoService = - new CompositeExtensionFactory(serviceProvider => - new AzureDevOpsReporter( - serviceProvider.GetCommandLineOptions(), - serviceProvider.GetEnvironment(), - serviceProvider.GetFileSystem(), - serviceProvider.GetOutputDevice(), - serviceProvider.GetLoggerFactory())); + builder.TestHost.AddDataConsumer(serviceProvider => + new AzureDevOpsReporter( + serviceProvider.GetCommandLineOptions(), + serviceProvider.GetEnvironment(), + serviceProvider.GetFileSystem(), + serviceProvider.GetOutputDevice(), + serviceProvider.GetLoggerFactory(), + GetOrCreateHistoryService(serviceProvider))); + builder.TestHost.AddTestSessionLifetimeHandler(serviceProvider => (AzureDevOpsHistoryService)GetOrCreateHistoryService(serviceProvider)); + builder.CommandLine.AddProvider(() => new AzureDevOpsCommandLineProvider()); + } - builder.TestHost.AddDataConsumer(compositeTestSessionAzDoService); + private static IAzureDevOpsHistoryService GetOrCreateHistoryService(IServiceProvider serviceProvider) + { + if (serviceProvider.GetService() is IAzureDevOpsHistoryService historyService) + { + return historyService; + } - builder.CommandLine.AddProvider(() => new AzureDevOpsCommandLineProvider()); + historyService = new AzureDevOpsHistoryService( + serviceProvider.GetCommandLineOptions(), + serviceProvider.GetEnvironment(), + serviceProvider.GetClock(), + new AzureDevOpsHistoryClient(serviceProvider.GetTask(), serviceProvider.GetClock()), + serviceProvider.GetTask(), + serviceProvider.GetLoggerFactory()); + ((ServiceProvider)serviceProvider).AddService(historyService, throwIfSameInstanceExit: false); + return historyService; } } diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryClient.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryClient.cs new file mode 100644 index 0000000000..ef33aade64 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryClient.cs @@ -0,0 +1,392 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Net; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; + +using Microsoft.Testing.Platform; +using Microsoft.Testing.Platform.Helpers; + +namespace Microsoft.Testing.Extensions.AzureDevOpsReport; + +internal interface IAzureDevOpsHistoryClient +{ + Task> GetRunsAsync(AzureDevOpsHistoryQuery query, int maximumRunCount, CancellationToken cancellationToken); + + Task GetResultsAsync(AzureDevOpsHistoryQuery query, string runUrl, int skip, int top, string? continuationToken, CancellationToken cancellationToken); +} + +internal sealed class AzureDevOpsHistoryClient : IAzureDevOpsHistoryClient +{ + private const int BaseDelayMs = 500; + private const int ErrorContentMaxLength = 500; + private const int MaxAttempts = 3; + private const int RunsPageSize = 200; + private const string ApiVersion = "7.1"; + private const string ContinuationTokenHeaderName = "x-ms-continuationtoken"; + private static readonly System.Net.Http.Headers.MediaTypeWithQualityHeaderValue JsonMediaType = new("application/json"); + private static readonly System.Net.Http.Headers.ProductInfoHeaderValue UserAgent = new("Microsoft.Testing.Extensions.AzureDevOpsReport", ExtensionVersion.DefaultSemVer); + private static readonly TimeSpan RequestTimeout = TimeSpan.FromSeconds(20); + private static readonly HttpClient SharedHttpClient = new(); + + private readonly ITask _task; + private readonly IClock _clock; + private readonly HttpClient _httpClient; + + public AzureDevOpsHistoryClient(ITask task, IClock clock, HttpClient? httpClient = null) + { + _task = task; + _clock = clock; + _httpClient = httpClient ?? SharedHttpClient; + } + + public async Task> GetRunsAsync(AzureDevOpsHistoryQuery query, int maximumRunCount, CancellationToken cancellationToken) + { + if (maximumRunCount <= 0) + { + return []; + } + + List runs = []; + string? continuationToken = null; + string? previousContinuationToken = null; + int skip = 0; + + while (runs.Count < maximumRunCount) + { + int top = Math.Min(RunsPageSize, maximumRunCount - runs.Count); + AzureDevOpsTestRunsPage page = await SendWithRetryAsync( + () => CreateRunsRequest(query, skip, top, continuationToken), + ParseRunsAsync, + cancellationToken).ConfigureAwait(false); + + if (page.Runs.Count == 0) + { + return runs; + } + + foreach (AzureDevOpsTestRun run in page.Runs) + { + runs.Add(run); + if (runs.Count == maximumRunCount) + { + return runs; + } + } + + if (page.ContinuationToken is null) + { + if (continuationToken is not null || page.Runs.Count < top) + { + return runs; + } + + skip += page.Runs.Count; + continue; + } + + if (page.ContinuationToken == previousContinuationToken) + { + return runs; + } + + previousContinuationToken = page.ContinuationToken; + continuationToken = page.ContinuationToken; + } + + return runs; + } + + public Task GetResultsAsync(AzureDevOpsHistoryQuery query, string runUrl, int skip, int top, string? continuationToken, CancellationToken cancellationToken) + => SendWithRetryAsync(() => CreateResultsRequest(query, runUrl, skip, top, continuationToken), ParseResultsAsync, cancellationToken); + + internal static string CreateRunsRequestUri(AzureDevOpsHistoryQuery query, int skip, int top, string? continuationToken = null) + { + string collectionUri = EnsureTrailingSlash(query.CollectionUri); + System.Text.StringBuilder builder = new(); + builder + .Append(collectionUri) + .Append(Uri.EscapeDataString(query.TeamProject)) + .Append("/_apis/test/Runs") + .Append("?minLastUpdatedDate=") + .Append(Uri.EscapeDataString(query.MinLastUpdatedDate.ToString("O", CultureInfo.InvariantCulture))) + .Append("&maxLastUpdatedDate=") + .Append(Uri.EscapeDataString(query.MaxLastUpdatedDate.ToString("O", CultureInfo.InvariantCulture))) + .Append("&definitions=") + .Append(Uri.EscapeDataString(query.BuildDefinitionId)) + .Append("&automated=true") + .Append("&$top=") + .Append(top.ToString(CultureInfo.InvariantCulture)); + + if (continuationToken is null) + { + builder.Append("&$skip=").Append(skip.ToString(CultureInfo.InvariantCulture)); + } + else + { + builder.Append("&continuationToken=").Append(Uri.EscapeDataString(continuationToken)); + } + + builder.Append("&api-version=").Append(ApiVersion); + return builder.ToString(); + } + + private static HttpRequestMessage CreateRunsRequest(AzureDevOpsHistoryQuery query, int skip, int top, string? continuationToken) + => CreateRequest(query.AccessToken, CreateRunsRequestUri(query, skip, top, continuationToken)); + + private static HttpRequestMessage CreateResultsRequest(AzureDevOpsHistoryQuery query, string runUrl, int skip, int top, string? continuationToken) + => CreateRequest(query.AccessToken, CreateResultsRequestUri(runUrl, skip, top, continuationToken)); + + private static string CreateResultsRequestUri(string runUrl, int skip, int top, string? continuationToken) + { + System.Text.StringBuilder builder = new(); + builder + .Append(runUrl.TrimEnd('/')) + .Append("/results?outcomes=Failed,Passed") + .Append("&$top=") + .Append(top.ToString(CultureInfo.InvariantCulture)); + + if (continuationToken is null) + { + builder.Append("&$skip=").Append(skip.ToString(CultureInfo.InvariantCulture)); + } + else + { + builder.Append("&continuationToken=").Append(Uri.EscapeDataString(continuationToken)); + } + + builder.Append("&api-version=").Append(ApiVersion); + return builder.ToString(); + } + + private static HttpRequestMessage CreateRequest(string accessToken, string requestUri) + { + HttpRequestMessage request = new(HttpMethod.Get, requestUri); + string encodedToken = Convert.ToBase64String(System.Text.Encoding.ASCII.GetBytes($":{accessToken}")); + request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", encodedToken); + request.Headers.Accept.Add(JsonMediaType); + request.Headers.UserAgent.Add(UserAgent); + return request; + } + + private async Task SendWithRetryAsync(Func requestFactory, Func> responseParser, CancellationToken cancellationToken) + { + for (int attempt = 1; ; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + using HttpRequestMessage request = requestFactory(); + using var timeoutCancellationTokenSource = new CancellationTokenSource(RequestTimeout); + using var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellationTokenSource.Token); + using HttpResponseMessage response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, linkedCancellationTokenSource.Token).ConfigureAwait(false); + if (response.IsSuccessStatusCode) + { + return await responseParser(response).ConfigureAwait(false); + } + + if (attempt >= MaxAttempts || !IsTransient(response.StatusCode)) + { + string responseContent = await ReadResponseContentAsync(response).ConfigureAwait(false); + string reasonPhrase = RoslynString.IsNullOrWhiteSpace(response.ReasonPhrase) ? response.StatusCode.ToString() : response.ReasonPhrase!; + string responseSuffix = responseContent.Length == 0 ? string.Empty : $" {responseContent}"; + throw new HttpRequestException($"Azure DevOps history request failed with status code {(int)response.StatusCode} ({reasonPhrase}).{responseSuffix}"); + } + + await _task.Delay(GetRetryDelay(response, attempt), cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && attempt < MaxAttempts) + { + await _task.Delay(GetRetryDelay(response: null, attempt), cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException ex) when (!cancellationToken.IsCancellationRequested) + { + throw new HttpRequestException("Azure DevOps history request timed out.", ex); + } + catch (HttpRequestException) when (attempt < MaxAttempts) + { + await _task.Delay(GetRetryDelay(response: null, attempt), cancellationToken).ConfigureAwait(false); + } + } + } + + private TimeSpan GetRetryDelay(HttpResponseMessage? response, int attempt) + { + if (response?.Headers.RetryAfter?.Delta is TimeSpan retryAfterDelta) + { + return retryAfterDelta; + } + + if (response?.Headers.RetryAfter?.Date is DateTimeOffset retryAfterDate) + { + TimeSpan delay = retryAfterDate - _clock.UtcNow; + if (delay > TimeSpan.Zero) + { + return delay; + } + } + + return TimeSpan.FromMilliseconds(BaseDelayMs * Math.Pow(2, attempt - 1)); + } + + private static bool IsTransient(HttpStatusCode statusCode) + => statusCode is HttpStatusCode.RequestTimeout + or HttpStatusCode.InternalServerError + or HttpStatusCode.BadGateway + or HttpStatusCode.ServiceUnavailable + or HttpStatusCode.GatewayTimeout + || (int)statusCode == 429; + + private static async Task ParseRunsAsync(HttpResponseMessage response) + { + AzureDevOpsRunsResponse? payload = await DeserializeResponseAsync(response, AzureDevOpsHistoryClientJsonContext.Default.AzureDevOpsRunsResponse).ConfigureAwait(false); + return payload?.Value is null + ? new AzureDevOpsTestRunsPage([], GetContinuationToken(response)) + : new AzureDevOpsTestRunsPage( + [.. payload.Value + .Where(static run => !RoslynString.IsNullOrWhiteSpace(run.Url)) + .Select(static run => new AzureDevOpsTestRun(run.Url!))], + GetContinuationToken(response)); + } + + private static async Task ParseResultsAsync(HttpResponseMessage response) + { + AzureDevOpsResultsResponse? payload = await DeserializeResponseAsync(response, AzureDevOpsHistoryClientJsonContext.Default.AzureDevOpsResultsResponse).ConfigureAwait(false); + string? continuationToken = GetContinuationToken(response); + + return payload?.Value is null + ? new AzureDevOpsTestResultsPage([], continuationToken) + : new AzureDevOpsTestResultsPage( + [.. payload.Value + .Where(static result => !RoslynString.IsNullOrWhiteSpace(result.AutomatedTestName) && !RoslynString.IsNullOrWhiteSpace(result.Outcome)) + .Select(static result => new AzureDevOpsTestResult(result.AutomatedTestName!, result.Outcome!))], + continuationToken); + } + + private static async Task DeserializeResponseAsync(HttpResponseMessage response, JsonTypeInfo jsonTypeInfo) + { +#pragma warning disable CA2016 // CancellationToken overload is unavailable on all target frameworks. + string content = await response.Content.ReadAsStringAsync().ConfigureAwait(false); +#pragma warning restore CA2016 + return System.Text.Json.JsonSerializer.Deserialize(content, jsonTypeInfo); + } + + private static string EnsureTrailingSlash(string value) + => value.EndsWith("/", StringComparison.Ordinal) ? value : value + "/"; + + private static string? GetContinuationToken(HttpResponseMessage response) + => response.Headers.TryGetValues(ContinuationTokenHeaderName, out IEnumerable? values) + ? values.FirstOrDefault() + : null; + + private static async Task ReadResponseContentAsync(HttpResponseMessage response) + { +#pragma warning disable CA2016 // CancellationToken overload is unavailable on all target frameworks. + string responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false); +#pragma warning restore CA2016 + return responseContent.Length <= ErrorContentMaxLength + ? responseContent + : responseContent.Substring(0, ErrorContentMaxLength); + } +} + +internal sealed class AzureDevOpsHistoryQuery +{ + public AzureDevOpsHistoryQuery(string collectionUri, string teamProject, string accessToken, string buildDefinitionId, DateTimeOffset minLastUpdatedDate, DateTimeOffset maxLastUpdatedDate) + { + CollectionUri = collectionUri; + TeamProject = teamProject; + AccessToken = accessToken; + BuildDefinitionId = buildDefinitionId; + MinLastUpdatedDate = minLastUpdatedDate; + MaxLastUpdatedDate = maxLastUpdatedDate; + } + + public string CollectionUri { get; } + + public string TeamProject { get; } + + public string AccessToken { get; } + + public string BuildDefinitionId { get; } + + public DateTimeOffset MinLastUpdatedDate { get; } + + public DateTimeOffset MaxLastUpdatedDate { get; } +} + +internal sealed class AzureDevOpsRunResponse +{ + [JsonPropertyName("url")] + public string? Url { get; set; } +} + +internal sealed class AzureDevOpsRunsResponse +{ + [JsonPropertyName("value")] + public AzureDevOpsRunResponse[]? Value { get; set; } +} + +internal sealed class AzureDevOpsTestRun +{ + public AzureDevOpsTestRun(string url) + => Url = url; + + public string Url { get; } +} + +internal sealed class AzureDevOpsTestRunsPage +{ + public AzureDevOpsTestRunsPage(IReadOnlyList runs, string? continuationToken) + { + Runs = runs; + ContinuationToken = continuationToken; + } + + public IReadOnlyList Runs { get; } + + public string? ContinuationToken { get; } +} + +internal sealed class AzureDevOpsResultResponse +{ + [JsonPropertyName("automatedTestName")] + public string? AutomatedTestName { get; set; } + + [JsonPropertyName("outcome")] + public string? Outcome { get; set; } +} + +internal sealed class AzureDevOpsResultsResponse +{ + [JsonPropertyName("value")] + public AzureDevOpsResultResponse[]? Value { get; set; } +} + +internal sealed class AzureDevOpsTestResult +{ + public AzureDevOpsTestResult(string automatedTestName, string outcome) + { + AutomatedTestName = automatedTestName; + Outcome = outcome; + } + + public string AutomatedTestName { get; } + + public string Outcome { get; } +} + +internal sealed class AzureDevOpsTestResultsPage +{ + public AzureDevOpsTestResultsPage(IReadOnlyList results, string? continuationToken) + { + Results = results; + ContinuationToken = continuationToken; + } + + public IReadOnlyList Results { get; } + + public string? ContinuationToken { get; } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryClientJsonContext.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryClientJsonContext.cs new file mode 100644 index 0000000000..f811c20d40 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryClientJsonContext.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Text.Json.Serialization; + +namespace Microsoft.Testing.Extensions.AzureDevOpsReport; + +[JsonSerializable(typeof(AzureDevOpsResultsResponse))] +[JsonSerializable(typeof(AzureDevOpsRunsResponse))] +internal sealed partial class AzureDevOpsHistoryClientJsonContext : JsonSerializerContext; diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryService.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryService.cs new file mode 100644 index 0000000000..c777d82372 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryService.cs @@ -0,0 +1,266 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.AzureDevOpsReport.Resources; +using Microsoft.Testing.Extensions.Reporting; +using Microsoft.Testing.Platform; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions.TestHost; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Extensions.AzureDevOpsReport; + +internal sealed class AzureDevOpsHistoryService : ITestSessionLifetimeHandler, IAzureDevOpsHistoryService +{ + // NOTE: Inspect one extra run so the reporter can log when Azure DevOps history exceeds the inspection cap. + private const int MaxRunsToInspect = 500; + + // NOTE: Request the largest practical result page to minimize Azure DevOps round-trips during session start. + private const int ResultsPageSize = 1000; + + // NOTE: Bound per-run paging so a single large run cannot keep session startup busy indefinitely. + private const int MaxResultPagesPerRun = 50; + + private static readonly TimeSpan HistoryLoadBudget = TimeSpan.FromSeconds(30); + private static readonly IReadOnlyDictionary EmptyStatsByTest = new Dictionary(StringComparer.OrdinalIgnoreCase); + + private readonly ICommandLineOptions _commandLineOptions; + private readonly IEnvironment _environment; + private readonly IClock _clock; + private readonly IAzureDevOpsHistoryClient _historyClient; + private readonly ITask _task; + private readonly ILogger _logger; + private int _historyWindowInDays; + private IReadOnlyDictionary _statsByTest = EmptyStatsByTest; + + public AzureDevOpsHistoryService( + ICommandLineOptions commandLineOptions, + IEnvironment environment, + IClock clock, + IAzureDevOpsHistoryClient historyClient, + ITask task, + ILoggerFactory loggerFactory) + { + _commandLineOptions = commandLineOptions; + _environment = environment; + _clock = clock; + _historyClient = historyClient; + _task = task; + _logger = loggerFactory.CreateLogger(); + } + + public string Uid => nameof(AzureDevOpsHistoryService); + + public string Version => ExtensionVersion.DefaultSemVer; + + public string DisplayName => AzureDevOpsResources.DisplayName; + + public string Description => AzureDevOpsResources.Description; + + public int HistoryWindowInDays => Volatile.Read(ref _historyWindowInDays); + + public Task IsEnabledAsync() + => Task.FromResult(_commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName) + && _commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory)); + + public async Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) + { + if (!_commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName) + || !TryGetHistoryWindowInDays(out int historyWindowInDays)) + { + return; + } + + if (!string.Equals(_environment.GetEnvironmentVariable("TF_BUILD"), "true", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + if (!TryCreateQuery(historyWindowInDays, out AzureDevOpsHistoryQuery? query)) + { + _logger.LogWarning(AzureDevOpsResources.FlakyHistoryMissingEnvironmentWarning); + return; + } + + try + { + using var budgetCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(testSessionContext.CancellationToken); + Task loadTask = LoadHistoryAsync(query, historyWindowInDays, budgetCancellationTokenSource.Token); + Task budgetTask = _task.Delay(HistoryLoadBudget, testSessionContext.CancellationToken); + + if (await Task.WhenAny(loadTask, budgetTask).ConfigureAwait(false) != loadTask) + { + testSessionContext.CancellationToken.ThrowIfCancellationRequested(); +#pragma warning disable VSTHRD103 // CancelAsync is unavailable on all target frameworks. + budgetCancellationTokenSource.Cancel(); +#pragma warning restore VSTHRD103 + ResetHistoryState(); + _logger.LogInformation(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.FlakyHistoryLoadTimedOutInfo, (int)HistoryLoadBudget.TotalSeconds)); + + try + { + await loadTask.ConfigureAwait(false); + } + catch (OperationCanceledException) when (!testSessionContext.CancellationToken.IsCancellationRequested) + { + } + + return; + } + + await loadTask.ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + ResetHistoryState(); + _logger.LogWarning(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.FlakyHistoryLoadFailedWarning, ex.Message)); + } + } + + public Task OnTestSessionFinishingAsync(ITestSessionContext testSessionContext) + => Task.CompletedTask; + + public bool TryGetStats(string testName, out FlakyStats stats) + { + if (RoslynString.IsNullOrWhiteSpace(testName)) + { + stats = default; + return false; + } + + // NOTE: Callers must only query stats after test-session start completes; the published snapshot is empty until then. + return Volatile.Read(ref _statsByTest).TryGetValue(testName, out stats); + } + + public bool IsLikelyFlaky(string testName, double threshold) + => TryGetStats(testName, out FlakyStats stats) + && stats.TotalCount > 0 + && stats.FailureRate >= threshold; + + private async Task LoadHistoryAsync(AzureDevOpsHistoryQuery query, int historyWindowInDays, CancellationToken cancellationToken) + { + IReadOnlyList runs = await _historyClient.GetRunsAsync(query, MaxRunsToInspect + 1, cancellationToken).ConfigureAwait(false); + if (runs.Count > MaxRunsToInspect) + { + _logger.LogInformation(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.FlakyHistoryRunsCappedInfo, runs.Count, MaxRunsToInspect)); + runs = [.. runs.Take(MaxRunsToInspect)]; + } + + var counts = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (AzureDevOpsTestRun run in runs) + { + await AggregateRunResultsAsync(query, run, counts, cancellationToken).ConfigureAwait(false); + } + + PublishHistoryStats(historyWindowInDays, counts); + } + + private async Task AggregateRunResultsAsync( + AzureDevOpsHistoryQuery query, + AzureDevOpsTestRun run, + Dictionary counts, + CancellationToken cancellationToken) + { + string? continuationToken = null; + string? previousContinuationToken = null; + bool hasSeenContinuationToken = false; + int skip = 0; + + for (int pageNumber = 0; pageNumber < MaxResultPagesPerRun; pageNumber++) + { + AzureDevOpsTestResultsPage page = await _historyClient.GetResultsAsync(query, run.Url, skip, ResultsPageSize, continuationToken, cancellationToken).ConfigureAwait(false); + foreach (AzureDevOpsTestResult result in page.Results) + { + if (!counts.TryGetValue(result.AutomatedTestName, out (int PassCount, int FailCount) currentCount)) + { + currentCount = default; + } + + counts[result.AutomatedTestName] = result.Outcome switch + { + "Passed" => (currentCount.PassCount + 1, currentCount.FailCount), + "Failed" => (currentCount.PassCount, currentCount.FailCount + 1), + _ => currentCount, + }; + } + + if (page.Results.Count == 0) + { + return; + } + + if (page.ContinuationToken is null) + { + if (hasSeenContinuationToken || page.Results.Count < ResultsPageSize) + { + return; + } + + skip += page.Results.Count; + continue; + } + + if (page.ContinuationToken == previousContinuationToken) + { + _logger.LogWarning(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.FlakyHistoryResultsPagingStoppedWarning, run.Url, MaxResultPagesPerRun)); + return; + } + + hasSeenContinuationToken = true; + previousContinuationToken = page.ContinuationToken; + continuationToken = page.ContinuationToken; + } + + _logger.LogWarning(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.FlakyHistoryResultsPagingStoppedWarning, run.Url, MaxResultPagesPerRun)); + } + + private void PublishHistoryStats(int historyWindowInDays, Dictionary counts) + { + Volatile.Write(ref _historyWindowInDays, historyWindowInDays); + IReadOnlyDictionary publishedStats = counts.ToDictionary( + static keyValuePair => keyValuePair.Key, + static keyValuePair => new FlakyStats(keyValuePair.Value.PassCount, keyValuePair.Value.FailCount), + StringComparer.OrdinalIgnoreCase); + + // NOTE: Volatile.Write on a reference atomically publishes the fully materialized dictionary to concurrent readers. + Volatile.Write(ref _statsByTest, publishedStats); + } + + private void ResetHistoryState() + { + Volatile.Write(ref _historyWindowInDays, 0); + + // NOTE: Volatile.Write on a reference atomically publishes the empty snapshot when history loading is skipped or fails. + Volatile.Write(ref _statsByTest, EmptyStatsByTest); + } + + private bool TryGetHistoryWindowInDays(out int historyWindowInDays) + { + historyWindowInDays = 0; + return _commandLineOptions.TryGetOptionArgumentList(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory, out string[]? arguments) + && arguments is [string value] + && int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out historyWindowInDays); + } + + private bool TryCreateQuery(int historyWindowInDays, [NotNullWhen(true)] out AzureDevOpsHistoryQuery? query) + { + string? collectionUri = _environment.GetEnvironmentVariable("SYSTEM_COLLECTIONURI"); + string? teamProject = _environment.GetEnvironmentVariable("SYSTEM_TEAMPROJECT"); + string? accessToken = _environment.GetEnvironmentVariable("SYSTEM_ACCESSTOKEN"); + string? buildDefinitionId = _environment.GetEnvironmentVariable("BUILD_DEFINITIONID"); + if (RoslynString.IsNullOrWhiteSpace(collectionUri) + || RoslynString.IsNullOrWhiteSpace(teamProject) + || RoslynString.IsNullOrWhiteSpace(accessToken) + || RoslynString.IsNullOrWhiteSpace(buildDefinitionId)) + { + query = null; + return false; + } + + DateTimeOffset now = _clock.UtcNow; + query = new AzureDevOpsHistoryQuery(collectionUri, teamProject, accessToken, buildDefinitionId, now.AddDays(-historyWindowInDays), now); + return true; + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsReporter.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsReporter.cs index aebb6b47e8..2f2ed5111f 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsReporter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsReporter.cs @@ -1,4 +1,4 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. +// Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.Testing.Extensions.AzureDevOpsReport.Resources; @@ -19,28 +19,40 @@ internal sealed class AzureDevOpsReporter : IDataConsumer, IOutputDeviceDataProducer { + private const double KnownFlakyFailureRateThreshold = 0.25; private const string DeterministicBuildRoot = "/_/"; + private const string FullyQualifiedNamePropertyKey = "vstest.TestCase.FullyQualifiedName"; + private const int MinSamplesForRegressionAnnotation = 5; + private const string QuarantineBuildTagLine = "##vso[build.addbuildtag]has-quarantined-test-failure"; + private const string WarningSeverity = "warning"; + private static readonly char[] NewlineCharacters = ['\r', '\n']; private readonly IOutputDevice _outputDisplay; private readonly ILogger _logger; - private static readonly char[] NewlineCharacters = ['\r', '\n']; private readonly ICommandLineOptions _commandLine; private readonly IEnvironment _environment; private readonly IFileSystem _fileSystem; + private readonly IAzureDevOpsHistoryService _historyService; private readonly string _targetFrameworkMoniker; - private string _severity = "error"; + private string? _severity; + private bool _demoteKnownFlaky; + private QuarantineFile? _quarantineFile; + private bool _hasLoadedEnabledConfiguration; + private int _quarantineBuildTagEmitted; public AzureDevOpsReporter( ICommandLineOptions commandLine, IEnvironment environment, IFileSystem fileSystem, IOutputDevice outputDisplay, - ILoggerFactory loggerFactory) + ILoggerFactory loggerFactory, + IAzureDevOpsHistoryService historyService) { _commandLine = commandLine; _environment = environment; _fileSystem = fileSystem; _outputDisplay = outputDisplay; + _historyService = historyService; _logger = loggerFactory.CreateLogger(); _targetFrameworkMoniker = GetTargetFrameworkMoniker(); } @@ -77,34 +89,18 @@ public Task IsEnabledAsync() } bool isEnabledByEnvVariable = string.Equals(_environment.GetEnvironmentVariable("TF_BUILD"), "true", StringComparison.OrdinalIgnoreCase); - if (_logger.IsEnabled(LogLevel.Trace)) - { - _logger.LogTrace($"TF_BUILD environment variable is {(isEnabledByEnvVariable ? "enabled. Will report errors to Azure DevOps, because we are running in CI." : "disabled. Will not report errors to Azure DevOps.")}."); - } - - if (!isEnabledByEnvVariable) + if (isEnabledByEnvVariable) { - return Task.FromResult(false); + EnsureEnabledConfigurationLoaded(); } - bool found = _commandLine.TryGetOptionArgumentList(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity, out string[]? arguments); - if (found && arguments?.Length > 0) - { - _severity = arguments[0].ToLowerInvariant(); - if (_logger.IsEnabled(LogLevel.Trace)) - { - _logger.LogTrace($"Severity is set to '{_severity}', by --report-azdo-severity parameter."); - } - } - else + if (_logger.IsEnabled(LogLevel.Trace)) { - if (_logger.IsEnabled(LogLevel.Trace)) - { - _logger.LogTrace($"Severity is set to '{_severity}', you can override it by using --report-azdo-severity parameter."); - } + _logger.LogTrace($"TF_BUILD environment variable is {(isEnabledByEnvVariable ? "enabled. Will report errors to Azure DevOps, because we are running in CI." : "disabled. Will not report errors to Azure DevOps.")}"); + _logger.LogTrace($"Severity is set to '{_severity ?? "error"}', you can override it by using --report-azdo-severity parameter."); } - return Task.FromResult(true); + return Task.FromResult(isEnabledByEnvVariable); } public async Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) @@ -116,38 +112,47 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella return; } + EnsureEnabledConfigurationLoaded(); TestNodeStateProperty? nodeState = nodeUpdateMessage.TestNode.Properties.SingleOrDefault(); - string testDisplayName = nodeUpdateMessage.TestNode.DisplayName; + string testName = GetTestName(nodeUpdateMessage.TestNode); switch (nodeState) { case FailedTestNodeStateProperty failed: - await WriteExceptionAsync(testDisplayName, failed.Explanation, failed.Exception, cancellationToken).ConfigureAwait(false); + await WriteExceptionAsync(testDisplayName, testName, failed.Explanation, failed.Exception, cancellationToken).ConfigureAwait(false); break; case ErrorTestNodeStateProperty error: - await WriteExceptionAsync(testDisplayName, error.Explanation, error.Exception, cancellationToken).ConfigureAwait(false); + await WriteExceptionAsync(testDisplayName, testName, error.Explanation, error.Exception, cancellationToken).ConfigureAwait(false); break; #pragma warning disable CS0618, MTP0001 // Type or member is obsolete case CancelledTestNodeStateProperty cancelled: #pragma warning restore CS0618, MTP0001 // Type or member is obsolete - await WriteExceptionAsync(testDisplayName, cancelled.Explanation, cancelled.Exception, cancellationToken).ConfigureAwait(false); + await WriteExceptionAsync(testDisplayName, testName, cancelled.Explanation, cancelled.Exception, cancellationToken).ConfigureAwait(false); break; case TimeoutTestNodeStateProperty timeout: - await WriteExceptionAsync(testDisplayName, timeout.Explanation, timeout.Exception, cancellationToken).ConfigureAwait(false); + await WriteExceptionAsync(testDisplayName, testName, timeout.Explanation, timeout.Exception, cancellationToken).ConfigureAwait(false); break; } } - private async Task WriteExceptionAsync(string testDisplayName, string? explanation, Exception? exception, CancellationToken cancellationToken) + private async Task WriteExceptionAsync(string testDisplayName, string testName, string? explanation, Exception? exception, CancellationToken cancellationToken) { if (_logger.IsEnabled(LogLevel.Trace)) { _logger.LogTrace("Failure received."); } - string? line = GetErrorText(testDisplayName, explanation, exception, _severity, _fileSystem, _logger, _targetFrameworkMoniker); - if (line == null) + bool isQuarantined = _quarantineFile?.Matches(testName) == true; + if (isQuarantined && Interlocked.Exchange(ref _quarantineBuildTagEmitted, 1) == 0) + { + await _outputDisplay.DisplayAsync(this, new FormattedTextOutputDeviceData(QuarantineBuildTagLine), cancellationToken).ConfigureAwait(false); + } + + string severity = GetSeverity(testName, isQuarantined); + string annotationSuffix = BuildAnnotationSuffix(testName, isQuarantined); + string? line = GetErrorText(testDisplayName, explanation, exception, severity, _fileSystem, _logger, _targetFrameworkMoniker, annotationSuffix); + if (line is null) { if (_logger.IsEnabled(LogLevel.Trace)) { @@ -166,8 +171,11 @@ private async Task WriteExceptionAsync(string testDisplayName, string? explanati } internal static /* for testing */ string? GetErrorText(string testDisplayName, string? explanation, Exception? exception, string severity, IFileSystem fileSystem, ILogger logger, string targetFrameworkMoniker) + => GetErrorText(testDisplayName, explanation, exception, severity, fileSystem, logger, targetFrameworkMoniker, additionalMessageSuffix: null); + + internal static /* for testing */ string? GetErrorText(string testDisplayName, string? explanation, Exception? exception, string severity, IFileSystem fileSystem, ILogger logger, string targetFrameworkMoniker, string? additionalMessageSuffix) { - if (exception == null || exception.StackTrace == null) + if (exception is null || exception.StackTrace is null) { if (logger.IsEnabled(LogLevel.Trace)) { @@ -178,8 +186,7 @@ private async Task WriteExceptionAsync(string testDisplayName, string? explanati } string message = explanation ?? exception.Message; - - if (message == null) + if (message is null) { if (logger.IsEnabled(LogLevel.Trace)) { @@ -199,7 +206,7 @@ private async Task WriteExceptionAsync(string testDisplayName, string? explanati foreach (string? stackFrame in stackTrace.Split(NewlineCharacters, StringSplitOptions.RemoveEmptyEntries)) { (string Code, string File, int LineNumber)? location = GetStackFrameLocation(stackFrame); - if (location == null) + if (location is null) { if (logger.IsEnabled(LogLevel.Trace)) { @@ -210,8 +217,6 @@ private async Task WriteExceptionAsync(string testDisplayName, string? explanati } string file = location.Value.File; - - // TODO: We need better rule for stackframes to opt out from being interesting. if (file.EndsWith("Assert.cs", StringComparison.Ordinal)) { if (logger.IsEnabled(LogLevel.Trace)) @@ -222,7 +227,6 @@ private async Task WriteExceptionAsync(string testDisplayName, string? explanati continue; } - // Deterministic build paths start with "/_/" string relativePath; if (file.StartsWith(DeterministicBuildRoot, StringComparison.OrdinalIgnoreCase)) { @@ -232,7 +236,6 @@ private async Task WriteExceptionAsync(string testDisplayName, string? explanati } relativePath = file.Substring(DeterministicBuildRoot.Length); - if (logger.IsEnabled(LogLevel.Trace)) { logger.LogTrace($"Using relative path '{relativePath}'."); @@ -253,7 +256,6 @@ private async Task WriteExceptionAsync(string testDisplayName, string? explanati } else { - // Path does not belong to current repo, keep it null. if (logger.IsEnabled(LogLevel.Trace)) { logger.LogTrace($"Path '{file}' does not belong to current repo '{repoRoot}'. Continue to next."); @@ -262,17 +264,9 @@ private async Task WriteExceptionAsync(string testDisplayName, string? explanati continue; } - // Combine with repo root, to be able to resolve deterministic build paths. string fullPath = Path.Combine(repoRoot, relativePath); if (!fileSystem.ExistFile(fullPath)) { - // Path does not belong to current repository or does not exist, no need to report it because it will not show up in the PR error, we will only see it details of the run, which is the same - // as not reporting it this way. Maybe there can be 2 modes, but right now we want this to be usable for GitHub + AzDo, not for pure AzDo. - // - // In case of deterministic build, all the paths will be relative, so if library carries symbols and matches our path we would see the error as coming from our file - // even though it would not. That change is slim and something we have to live with. - // - // Deterministic build will also have paths normalized to /, luckily File.Exist does not care about the slash direction (on Windows). if (logger.IsEnabled(LogLevel.Trace)) { logger.LogTrace($"Path '{fullPath}' does not exist on disk. Continue to next."); @@ -281,36 +275,121 @@ private async Task WriteExceptionAsync(string testDisplayName, string? explanati continue; } - // The slashes must be / for GitHub to render the error placement correctly. string relativeNormalizedPath = relativePath.Replace('\\', '/'); if (logger.IsEnabled(LogLevel.Trace)) { logger.LogTrace($"Normalized path for GitHub '{relativeNormalizedPath}'."); } - string formattedMessage = $"[{testDisplayName}] [{targetFrameworkMoniker}] {message}"; + string formattedMessage = $"[{testDisplayName}] [{targetFrameworkMoniker}] {message}{additionalMessageSuffix}"; string line = $"##vso[task.logissue type={severity};sourcepath={relativeNormalizedPath};linenumber={location.Value.LineNumber};columnnumber=1]{AzDoEscaper.Escape(formattedMessage)}"; if (logger.IsEnabled(LogLevel.Trace)) { logger.LogTrace($"Reported full message '{line}'."); } - // Report the error only for the first stack frame that is useful. return line; } if (logger.IsEnabled(LogLevel.Trace)) { - logger.LogTrace($"No stack trace line matched criteria, no failure line was reported."); + logger.LogTrace("No stack trace line matched criteria, no failure line was reported."); } return null; } + private string GetConfiguredSeverity() + => _commandLine.TryGetOptionArgumentList(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity, out string[]? arguments) + && arguments is [string configuredSeverity] + ? configuredSeverity.ToLowerInvariant() + : "error"; + + private QuarantineFile? LoadQuarantineFile() + { + if (!_commandLine.TryGetOptionArgumentList(AzureDevOpsCommandLineOptions.AzureDevOpsQuarantineFile, out string[]? arguments) + || arguments is not [string quarantineFilePath]) + { + return null; + } + + // NOTE: The value is treated as an explicit filesystem path supplied by the caller; this extension only validates existence and readability. + if (!_fileSystem.ExistFile(quarantineFilePath)) + { + _logger.LogWarning(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.QuarantineFileMissingWarning, quarantineFilePath)); + return null; + } + + try + { + return new QuarantineFile(quarantineFilePath, _fileSystem, _logger); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.QuarantineFileLoadFailedWarning, quarantineFilePath, ex.Message)); + return null; + } + } + + private string GetSeverity(string testName, bool isQuarantined) + => isQuarantined || (_demoteKnownFlaky && _historyService.IsLikelyFlaky(testName, KnownFlakyFailureRateThreshold)) + ? WarningSeverity + : _severity ?? "error"; + + private string BuildAnnotationSuffix(string testName, bool isQuarantined) + { + string? historyAnnotation = GetHistoryAnnotation(testName); + if (historyAnnotation is null && !isQuarantined) + { + return string.Empty; + } + + StringBuilder builder = new(); + if (historyAnnotation is not null) + { + builder.Append(' ').Append(historyAnnotation); + } + + if (isQuarantined) + { + builder.Append(' ').Append(AzureDevOpsResources.QuarantinedAnnotation); + } + + return builder.ToString(); + } + + private string? GetHistoryAnnotation(string testName) + => !_historyService.TryGetStats(testName, out FlakyStats stats) || stats.TotalCount == 0 + ? null + : stats.FailCount == 0 + ? stats.TotalCount >= MinSamplesForRegressionAnnotation + ? AzureDevOpsResources.FlakyHistoryRegressionAnnotation + : null + : string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.FlakyHistoryFailureAnnotation, stats.FailCount, stats.TotalCount, _historyService.HistoryWindowInDays); + + private void EnsureEnabledConfigurationLoaded() + { + if (_hasLoadedEnabledConfiguration) + { + return; + } + + _severity = GetConfiguredSeverity(); + _demoteKnownFlaky = _commandLine.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsDemoteKnownFlaky); + _quarantineFile = LoadQuarantineFile(); + _hasLoadedEnabledConfiguration = true; + } + private static string GetTargetFrameworkMoniker() => TargetFrameworkParser.GetShortTargetFramework(Assembly.GetEntryAssembly()?.GetCustomAttribute()?.FrameworkDisplayName) ?? TargetFrameworkParser.GetShortTargetFramework(RuntimeInformation.FrameworkDescription); + private static string GetTestName(TestNode testNode) + => testNode.Properties + .OfType() + .FirstOrDefault(static property => property.Key == FullyQualifiedNamePropertyKey)?.Value + ?? testNode.DisplayName; + private static (string Code, string File, int LineNumber)? GetStackFrameLocation(string stackTraceLine) { Match match = StackTraceHelper.GetFrameRegex().Match(stackTraceLine); @@ -320,8 +399,7 @@ private static (string Code, string File, int LineNumber)? GetStackFrameLocation } string code = match.Groups["code"].Value; - bool weHaveFilePathAndCodeLine = !RoslynString.IsNullOrWhiteSpace(code); - if (!weHaveFilePathAndCodeLine) + if (RoslynString.IsNullOrWhiteSpace(code)) { return null; } @@ -333,7 +411,6 @@ private static (string Code, string File, int LineNumber)? GetStackFrameLocation } int line = int.TryParse(match.Groups["line"].Value, out int value) ? value : 0; - return (code, file, line); } } diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/FlakyStats.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/FlakyStats.cs new file mode 100644 index 0000000000..c277acd7de --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/FlakyStats.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Extensions.AzureDevOpsReport; + +internal readonly struct FlakyStats +{ + public FlakyStats(int passCount, int failCount) + { + PassCount = passCount; + FailCount = failCount; + } + + public int PassCount { get; } + + public int FailCount { get; } + + public int TotalCount => PassCount + FailCount; + + public double FailureRate => TotalCount == 0 ? 0 : (double)FailCount / TotalCount; +} diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/IAzureDevOpsHistoryService.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/IAzureDevOpsHistoryService.cs new file mode 100644 index 0000000000..c87575d9f6 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/IAzureDevOpsHistoryService.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Extensions.AzureDevOpsReport; + +internal interface IAzureDevOpsHistoryService +{ + int HistoryWindowInDays { get; } + + bool TryGetStats(string testName, out FlakyStats stats); + + bool IsLikelyFlaky(string testName, double threshold); +} diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Microsoft.Testing.Extensions.AzureDevOpsReport.csproj b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Microsoft.Testing.Extensions.AzureDevOpsReport.csproj index 025d2c16f3..584c40e84b 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Microsoft.Testing.Extensions.AzureDevOpsReport.csproj +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Microsoft.Testing.Extensions.AzureDevOpsReport.csproj @@ -47,12 +47,17 @@ This package extends Microsoft Testing Platform to provide a Azure DevOps report + + + + + diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/QuarantineFile.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/QuarantineFile.cs new file mode 100644 index 0000000000..c35d07c6fa --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/QuarantineFile.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.AzureDevOpsReport.Resources; +using Microsoft.Testing.Platform; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; + +namespace Microsoft.Testing.Extensions.AzureDevOpsReport; + +internal sealed class QuarantineFile +{ + private const int MaxPatternCount = 10_000; + private const int MaxPatternLength = 4 * 1024; + + // TODO: Consider an opt-in modifier for case-insensitive quarantine matching if customer scenarios require it. + private static readonly RegexOptions RegexOptions = RegexOptions.CultureInvariant; + + private readonly Regex[] _patterns; + + public QuarantineFile(string path, IFileSystem fileSystem, ILogger logger) + : this(ParsePatterns(fileSystem.ReadAllText(path), logger)) + { + } + + internal QuarantineFile(IEnumerable patterns) + => _patterns = [.. patterns.Select(CreatePatternRegex)]; + + public bool Matches(string testFqn) + { + if (RoslynString.IsNullOrWhiteSpace(testFqn)) + { + return false; + } + + string normalizedTestFqn = Normalize(testFqn); + foreach (Regex pattern in _patterns) + { + if (pattern.IsMatch(normalizedTestFqn)) + { + return true; + } + } + + return false; + } + + private static IEnumerable ParsePatterns(string fileContent, ILogger logger) + { + int patternCount = 0; + foreach (string line in fileContent.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries)) + { + string trimmedLine = line.Trim(); + if (trimmedLine.Length == 0 || trimmedLine.StartsWith("#", StringComparison.Ordinal)) + { + continue; + } + + string normalizedPattern = Normalize(trimmedLine); + if (normalizedPattern.Length > MaxPatternLength) + { + logger.LogWarning(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.QuarantinePatternTooLongWarning, MaxPatternLength)); + continue; + } + + if (patternCount >= MaxPatternCount) + { + logger.LogWarning(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.QuarantinePatternsCappedWarning, MaxPatternCount)); + yield break; + } + + patternCount++; + yield return normalizedPattern; + } + } + + private static Regex CreatePatternRegex(string pattern) + { + string escapedPattern = Regex.Escape(pattern) + .Replace("\\*", ".*") + .Replace("\\?", "."); + + return new Regex($"^{escapedPattern}$", RegexOptions); + } + + private static string Normalize(string value) + => value.Trim(); +} diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/AzureDevOpsResources.resx b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/AzureDevOpsResources.resx index 516d5d4382..e86e656c4a 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/AzureDevOpsResources.resx +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/AzureDevOpsResources.resx @@ -1,4 +1,4 @@ - +