Skip to content

Add Azure DevOps flaky-history annotations and quarantine awareness #8298

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Evangelink wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Add Azure DevOps flaky-history annotations and quarantine awareness #8298

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Directory.Packages.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
<PackageVersion Include="Microsoft.Win32.Registry" Version="5.0.0" />
<PackageVersion Include="Microsoft.WindowsAppSDK" Version="1.8.251003001" />
<PackageVersion Include="OpenTelemetry" Version="1.15.3" />
<PackageVersion Include="System.Text.Json" Version="8.0.5" />
<PackageVersion Include="System.Threading.Tasks.Extensions" Version="$(SystemThreadingTasksExtensionsVersion)" />
</ItemGroup>
<ItemGroup Label="Test dependencies">
Expand Down
3 changes: 3 additions & 0 deletions src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,8 @@ namespace Microsoft.Testing.Extensions.Reporting;
internal static class AzureDevOpsCommandLineOptions
{
public const string AzureDevOpsOptionName = "report-azdo";
public const string AzureDevOpsDemoteKnownFlaky = "report-azdo-demote-known-flaky";
public const string AzureDevOpsFlakyHistory = "report-azdo-flaky-history";
public const string AzureDevOpsQuarantineFile = "report-azdo-quarantine-file";
public const string AzureDevOpsReportSeverity = "report-azdo-severity";
}
58 changes: 44 additions & 14 deletions ...Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,31 +26,61 @@ public IReadOnlyCollection<CommandLineOption> GetCommandLineOptions()
=>
[
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName, AzureDevOpsResources.OptionDescription, ArgumentArity.Zero, false),
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsDemoteKnownFlaky, AzureDevOpsResources.DemoteKnownFlakyOptionDescription, ArgumentArity.Zero, false),
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory, AzureDevOpsResources.FlakyHistoryOptionDescription, ArgumentArity.ExactlyOne, false),
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsQuarantineFile, AzureDevOpsResources.QuarantineFileOptionDescription, ArgumentArity.ExactlyOne, false),
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity, AzureDevOpsResources.SeverityOptionDescription, ArgumentArity.ExactlyOne, false),
];

public Task<ValidationResult> ValidateOptionArgumentsAsync(CommandLineOption commandOption, string[] arguments)
=> commandOption.Name switch
{
AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory => ValidateFlakyHistoryArgumentsAsync(arguments),
AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity => ValidateSeverityArgumentsAsync(arguments),
_ => ValidationResult.ValidTask,
};

public Task<ValidationResult> ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions)
{
if (commandOption.Name == AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity)
string? errorMessage = null;
if (!commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName))
{
if (!SeverityOptions.Contains(arguments[0], StringComparer.OrdinalIgnoreCase))
if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsDemoteKnownFlaky))
{
errorMessage = AzureDevOpsResources.AzureDevOpsDemoteKnownFlakyRequiresAzureDevOps;
}
else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory))
{
errorMessage = AzureDevOpsResources.AzureDevOpsFlakyHistoryRequiresAzureDevOps;
}
else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsQuarantineFile))
{
errorMessage = AzureDevOpsResources.AzureDevOpsQuarantineFileRequiresAzureDevOps;
}
else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity))
{
return ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSeverity, arguments[0]));
errorMessage = AzureDevOpsResources.AzureDevOpsReportSeverityRequiresAzureDevOps;
}
}

return ValidationResult.ValidTask;
}

public Task<ValidationResult> ValidateCommandLineOptionsAsync(ICommandLineOptions commandLineOptions)
{
if (!commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName) &&
commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity))
else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsDemoteKnownFlaky)
&& !commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory))
{
// If report-azdo is not set, but report-azdo-severity is set, it's invalid.
return ValidationResult.InvalidTask(AzureDevOpsResources.AzureDevOpsReportSeverityRequiresAzureDevOps);
errorMessage = AzureDevOpsResources.AzureDevOpsDemoteKnownFlakyRequiresFlakyHistory;
}

return ValidationResult.ValidTask;
return errorMessage is null
? ValidationResult.ValidTask
: ValidationResult.InvalidTask(errorMessage);
}

private static Task<ValidationResult> ValidateFlakyHistoryArgumentsAsync(string[] arguments)
=> int.TryParse(arguments[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out int days)
&& days is >= 1 and <= 90
? ValidationResult.ValidTask
: ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidFlakyHistoryDays, arguments[0]));

private static Task<ValidationResult> ValidateSeverityArgumentsAsync(string[] arguments)
=> SeverityOptions.Contains(arguments[0], StringComparer.OrdinalIgnoreCase)
? ValidationResult.ValidTask
: ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSeverity, arguments[0]));
}
37 changes: 26 additions & 11 deletions src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
using Microsoft.Testing.Extensions.AzureDevOpsReport;
using Microsoft.Testing.Extensions.Reporting;
using Microsoft.Testing.Platform.Builder;
using Microsoft.Testing.Platform.Extensions;
using Microsoft.Testing.Platform.Services;

namespace Microsoft.Testing.Extensions;
Expand All @@ -20,17 +19,33 @@ public static class AzureDevOpsExtensions
/// <param name="builder">The test application builder.</param>
public static void AddAzureDevOpsProvider(this ITestApplicationBuilder builder)
{
var compositeTestSessionAzDoService =
new CompositeExtensionFactory<AzureDevOpsReporter>(serviceProvider =>
new AzureDevOpsReporter(
serviceProvider.GetCommandLineOptions(),
serviceProvider.GetEnvironment(),
serviceProvider.GetFileSystem(),
serviceProvider.GetOutputDevice(),
serviceProvider.GetLoggerFactory()));
builder.TestHost.AddDataConsumer(serviceProvider =>
new AzureDevOpsReporter(
serviceProvider.GetCommandLineOptions(),
serviceProvider.GetEnvironment(),
serviceProvider.GetFileSystem(),
serviceProvider.GetOutputDevice(),
serviceProvider.GetLoggerFactory(),
GetOrCreateHistoryService(serviceProvider)));
builder.TestHost.AddTestSessionLifetimeHandler(serviceProvider => (AzureDevOpsHistoryService)GetOrCreateHistoryService(serviceProvider));
builder.CommandLine.AddProvider(() => new AzureDevOpsCommandLineProvider());
}

builder.TestHost.AddDataConsumer(compositeTestSessionAzDoService);
private static IAzureDevOpsHistoryService GetOrCreateHistoryService(IServiceProvider serviceProvider)
{
if (serviceProvider.GetService<IAzureDevOpsHistoryService>() is IAzureDevOpsHistoryService historyService)
{
return historyService;
}

builder.CommandLine.AddProvider(() => new AzureDevOpsCommandLineProvider());
historyService = new AzureDevOpsHistoryService(
serviceProvider.GetCommandLineOptions(),
serviceProvider.GetEnvironment(),
serviceProvider.GetClock(),
new AzureDevOpsHistoryClient(serviceProvider.GetTask(), serviceProvider.GetClock()),
serviceProvider.GetTask(),
serviceProvider.GetLoggerFactory());
((ServiceProvider)serviceProvider).AddService(historyService, throwIfSameInstanceExit: false);
return historyService;
}
}
Loading