From 4de59ce874b0fb3969b968912653641d690d13a0 Mon Sep 17 00:00:00 2001 From: RakeshwarRK <156244203+RakeshwarRK@users.noreply.github.com> Date: Wed, 18 Mar 2026 21:17:21 -0700 Subject: [PATCH] Add CPU core affinity support for ASP.NET workloads - Refactor monolithic AspNetBenchExecutor into standalone executors: AspNetServerExecutor, AspNetOrchardServerExecutor, BombardierExecutor, WrkExecutor - Add WorkloadAffinitySupport shared helper for process affinity binding (numactl on Linux, processor bitmask on Windows) - Add BindToCores/CoreAffinity parameters to all 4 executors - Add PERF-ASPNETBENCH-AFFINITY.json profile (server 0-7, client 8-15) - Add Nginx/Wrk executor support with new profiles - Update documentation with CPU Core Affinity section Co-Authored-By: Claude Opus 4.6 --- .../AspNetBenchProfileTests.cs | 65 +- .../NginxWrkProfileTests.cs | 89 +++ .../AspNetBench/AspNetBenchExecutorTests.cs | 200 ----- .../AspNetOrchardServerExecutorTests.cs | 245 +++++++ .../AspNetBench/AspNetServerExecutorTests.cs | 269 +++++++ .../Examples/Nginx/NginxVersionExample.txt | 4 + .../Examples/Wrk/wrkErrorExample1.txt | 10 + .../Examples/Wrk/wrkErrorExample2.txt | 14 + .../Examples/Wrk/wrkStandardExample1.txt | 96 +++ .../Examples/Wrk/wrkStandardExample2.txt | 204 ++++++ .../Examples/Wrk/wrkStandardExample3.txt | 14 + .../Nginx/NginxServerExecutorTest.cs | 542 ++++++++++++++ .../Wrk/Wrk2ExecutorTest.cs | 437 +++++++++++ .../Wrk/WrkExecutorTest.cs | 687 ++++++++++++++++++ .../Wrk/WrkMetricsParserTest.cs | 174 +++++ .../ASPNET/AspNetBenchBaseExecutor.cs | 336 --------- .../ASPNET/AspNetBenchClientExecutor.cs | 64 -- .../ASPNET/AspNetBenchExecutor.cs | 250 ------- .../ASPNET/AspNetBenchServerExecutor.cs | 62 -- .../ASPNET/AspNetOrchardServerExecutor.cs | 398 ++++++++++ .../ASPNET/AspNetServerExecutor.cs | 425 +++++++++++ .../Bombardier/BombardierExecutor.cs | 506 +++++++++++++ .../Nginx/NginxCommand.cs | 38 + .../Nginx/NginxExtensions.cs | 66 ++ .../Nginx/NginxServerExecutor.cs | 371 ++++++++++ .../WorkloadAffinitySupport.cs | 109 +++ .../VirtualClient.Actions/Wrk/Wrk2Executor.cs | 56 ++ .../VirtualClient.Actions/Wrk/WrkExecutor.cs | 563 ++++++++++++++ .../VirtualClient.Actions/Wrk/runwrk.sh | 2 + .../profiles/PERF-ASPNET-ORCHARD-WRK.json | 161 ++++ .../profiles/PERF-ASPNET-TEJSON-WRK.json | 150 ++++ .../profiles/PERF-ASPNETBENCH-AFFINITY.json | 94 +++ .../profiles/PERF-ASPNETBENCH-MULTI.json | 37 +- .../profiles/PERF-ASPNETBENCH.json | 27 +- .../profiles/PERF-WEB-NGINX-WRK-RP.json | 404 ++++++++++ .../profiles/PERF-WEB-NGINX-WRK.json | 382 ++++++++++ .../profiles/PERF-WEB-NGINX-WRK2-RP.json | 421 +++++++++++ .../profiles/PERF-WEB-NGINX-WRK2.json | 399 ++++++++++ .../aspnetbench/aspnetbench-profiles.md | 166 ++++- .../docs/workloads/aspnetbench/aspnetbench.md | 88 ++- 40 files changed, 7647 insertions(+), 978 deletions(-) create mode 100644 src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs delete mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetBenchExecutorTests.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Nginx/NginxVersionExample.txt create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample1.txt create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample2.txt create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample1.txt create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample2.txt create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample3.txt create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/Wrk2ExecutorTest.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs create mode 100644 src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs delete mode 100644 src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchBaseExecutor.cs delete mode 100644 src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchClientExecutor.cs delete mode 100644 src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchExecutor.cs delete mode 100644 src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchServerExecutor.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Nginx/NginxCommand.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/WorkloadAffinitySupport.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Wrk/Wrk2Executor.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs create mode 100644 src/VirtualClient/VirtualClient.Actions/Wrk/runwrk.sh create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-ORCHARD-WRK.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-TEJSON-WRK.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-AFFINITY.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK-RP.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2-RP.json create mode 100644 src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2.json diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs index 046a97deb8..239caf3528 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/AspNetBenchProfileTests.cs @@ -6,6 +6,7 @@ namespace VirtualClient.Actions using System; using System.Collections.Generic; using System.Linq; + using System.Net; using System.Threading; using System.Threading.Tasks; using NUnit.Framework; @@ -17,19 +18,32 @@ namespace VirtualClient.Actions public class AspNetBenchProfileTests { private DependencyFixture mockFixture; + private string clientAgentId; + private string serverAgentId; [OneTimeSetUp] public void SetupFixture() { - this.mockFixture = new DependencyFixture(); + this.clientAgentId = $"{Environment.MachineName}-Client"; + this.serverAgentId = $"{Environment.MachineName}-Server"; + ComponentTypeCache.Instance.LoadComponentTypes(TestDependencies.TestDirectory); } + [SetUp] + public void Setup() + { + this.mockFixture = new DependencyFixture(); + } + [Test] [TestCase("PERF-ASPNETBENCH.json")] public void AspNetBenchWorkloadProfileParametersAreInlinedCorrectly(string profile) { - this.mockFixture.Setup(PlatformID.Unix); + this.mockFixture.Setup(PlatformID.Unix, agentId: this.clientAgentId).SetupLayout( + new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), + new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) { WorkloadAssert.ParameterReferencesInlined(executor.Profile); @@ -42,10 +56,6 @@ public async Task AspNetBenchWorkloadProfileExecutesTheExpectedWorkloadsOnWindow { IEnumerable expectedCommands = this.GetProfileExpectedCommands(PlatformID.Win32NT); this.SetupDefaultMockBehaviors(PlatformID.Win32NT); - // Setup the expectations for the workload - // - Workload package is installed and exists. - // - Workload binaries/executables exist on the file system. - // - The workload generates valid results. this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => { @@ -58,6 +68,9 @@ public async Task AspNetBenchWorkloadProfileExecutesTheExpectedWorkloadsOnWindow return process; }; + // Setup API client for client-server communication + this.SetupApiClient(this.serverAgentId, "1.2.3.4"); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) { executor.ExecuteDependencies = false; @@ -73,10 +86,6 @@ public async Task AspNetBenchWorkloadProfileExecutesTheExpectedWorkloadsOnUnixPl { IEnumerable expectedCommands = this.GetProfileExpectedCommands(PlatformID.Unix); this.SetupDefaultMockBehaviors(PlatformID.Unix); - // Setup the expectations for the workload - // - Workload package is installed and exists. - // - Workload binaries/executables exist on the file system. - // - The workload generates valid results. this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => { @@ -89,6 +98,9 @@ public async Task AspNetBenchWorkloadProfileExecutesTheExpectedWorkloadsOnUnixPl return process; }; + // Setup API client for client-server communication + this.SetupApiClient(this.serverAgentId, "1.2.3.4"); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) { executor.ExecuteDependencies = false; @@ -107,8 +119,10 @@ private IEnumerable GetProfileExpectedCommands(PlatformID platform) commands = new List { @"dotnet\.exe build -c Release -p:BenchmarksTargetFramework=net8.0", - @"dotnet\.exe .+Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept:.+ keep-alive", - @"bombardier\.exe --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:9876/json --print r --format json" + @"pkill dotnet", + @"fuser -n tcp -k 9876", + @"dotnet\.exe .+Benchmarks.dll --nonInteractive true --scenarios json --urls http://\*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept:.+ keep-alive", + @"bombardier\.exe --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://1\.2\.3\.4:9876/json --print r --format json" }; break; @@ -117,8 +131,10 @@ private IEnumerable GetProfileExpectedCommands(PlatformID platform) { @"chmod \+x .+bombardier", @"dotnet build -c Release -p:BenchmarksTargetFramework=net8.0", - @"dotnet .+Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept:.+ keep-alive", - @"bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:9876/json --print r --format json" + @"pkill dotnet", + @"fuser -n tcp -k 9876", + @"dotnet .+Benchmarks.dll --nonInteractive true --scenarios json --urls http://\*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept:.+ keep-alive", + @"bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://1\.2\.3\.4:9876/json --print r --format json" }; break; } @@ -130,14 +146,19 @@ private void SetupDefaultMockBehaviors(PlatformID platform) { if (platform == PlatformID.Win32NT) { - this.mockFixture.Setup(PlatformID.Win32NT); + this.mockFixture.Setup(PlatformID.Win32NT, agentId: this.clientAgentId).SetupLayout( + new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), + new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); + this.mockFixture.SetupPackage("aspnetbenchmarks", expectedFiles: @"aspnetbench"); this.mockFixture.SetupPackage("bombardier", expectedFiles: @"win-x64\bombardier.exe"); this.mockFixture.SetupPackage("dotnetsdk", expectedFiles: @"packages\dotnet\dotnet.exe"); } else { - this.mockFixture.Setup(PlatformID.Unix); + this.mockFixture.Setup(PlatformID.Unix, agentId: this.clientAgentId).SetupLayout( + new ClientInstance(this.clientAgentId, "1.2.3.5", ClientRole.Client), + new ClientInstance(this.serverAgentId, "1.2.3.4", ClientRole.Server)); this.mockFixture.SetupPackage("aspnetbenchmarks", expectedFiles: @"aspnetbench"); this.mockFixture.SetupPackage("bombardier", expectedFiles: @"linux-x64\bombardier"); @@ -146,5 +167,17 @@ private void SetupDefaultMockBehaviors(PlatformID platform) this.mockFixture.SetupDisks(withRemoteDisks: false); } + + private void SetupApiClient(string serverName, string serverIPAddress) + { + IPAddress.TryParse(serverIPAddress, out IPAddress ipAddress); + IApiClient apiClient = this.mockFixture.ApiClientManager.GetOrCreateApiClient(serverName, ipAddress); + + State state = new State(); + state.Online(true); + + apiClient.CreateStateAsync(nameof(State), state, CancellationToken.None) + .GetAwaiter().GetResult(); + } } } diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs new file mode 100644 index 0000000000..5387340e85 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/NginxWrkProfileTests.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using NUnit.Framework; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Functional")] + public class NginxWrkProfileTests + { + private DependencyFixture mockFixture; + + [OneTimeSetUp] + public void SetupFixture() + { + this.mockFixture = new DependencyFixture(); + ComponentTypeCache.Instance.LoadComponentTypes(TestDependencies.TestDirectory); + } + + [Test] + [TestCase("PERF-WEB-NGINX-WRK.json")] + [TestCase("PERF-WEB-NGINX-WRK2.json")] + public void NginxWrkProfileParametersAreInlinedCorrectly(string profile) + { + this.mockFixture.Setup(PlatformID.Unix); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + WorkloadAssert.ParameterReferencesInlined(executor.Profile); + } + } + + [Test] + [TestCase("PERF-WEB-NGINX-WRK.json")] + [TestCase("PERF-WEB-NGINX-WRK2.json")] + public void NginxWrkProfileParametersAreAvailable(string profile) + { + this.mockFixture.Setup(PlatformID.Unix); + + var serverPrams = new List { "PackageName", "Role", "Timeout" }; + + var reverseProxyPrams = new List { "PackageName", "Role", "Timeout" }; + + var clientPrams = new List { "PackageName", "Role", "Timeout", "TestDuration", "FileSizeInKB", "Connection", "ThreadCount", "CommandArguments", "MetricScenario", "Scenario" }; + + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) + { + foreach (var actionBlock in executor.Profile.Actions) + { + string role = actionBlock.Parameters["Role"].ToString(); + + if (role.Equals("server", StringComparison.OrdinalIgnoreCase)) + { + foreach (var pram in serverPrams) + { + if (!actionBlock.Parameters.ContainsKey(pram)) + { + Assert.False(true, $"{actionBlock.Type} does not have {pram} parameter."); + } + } + } + else if (role.Equals("reverseproxy", StringComparison.OrdinalIgnoreCase)) + { + foreach (var pram in reverseProxyPrams) + { + if (!actionBlock.Parameters.ContainsKey(pram)) + { + Assert.False(true, $"{actionBlock.Type} does not have {pram} parameter."); + } + } + } + else + { + foreach (var pram in clientPrams) + { + if (!actionBlock.Parameters.ContainsKey(pram)) + { + Assert.False(true, $"{actionBlock.Type} does not have {pram} parameter."); + } + } + } + } + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetBenchExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetBenchExecutorTests.cs deleted file mode 100644 index b95aa5a1bd..0000000000 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetBenchExecutorTests.cs +++ /dev/null @@ -1,200 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.Diagnostics; - using System.IO; - using System.Linq; - using System.Reflection; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Extensions.DependencyInjection; - using Moq; - using NUnit.Framework; - using VirtualClient.Common; - using VirtualClient.Common.Telemetry; - using VirtualClient.Contracts; - - [TestFixture] - [Category("Unit")] - public class AspNetBenchExecutorTests : MockFixture - { - public void SetupTest(PlatformID platform) - { - if (platform == PlatformID.Win32NT) - { - this.Setup(PlatformID.Win32NT); - - DependencyPath mockAspNetBenchPackage = new DependencyPath("aspnetbenchmarks", this.PlatformSpecifics.GetPackagePath("aspnetbenchmarks")); - DependencyPath mockDotNetPackage = new DependencyPath("dotnetsdk", this.PlatformSpecifics.GetPackagePath("dotnet")); - DependencyPath mockBombardierPackage = new DependencyPath("bombardier", this.PlatformSpecifics.GetPackagePath("bombardier")); - this.PackageManager.OnGetPackage(mockAspNetBenchPackage.Name).ReturnsAsync(mockAspNetBenchPackage); - this.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); - this.PackageManager.OnGetPackage(mockBombardierPackage.Name).ReturnsAsync(mockBombardierPackage); - } - else - { - this.Setup(PlatformID.Unix); - - DependencyPath mockAspNetBenchPackage = new DependencyPath("aspnetbenchmarks", this.PlatformSpecifics.GetPackagePath("aspnetbenchmarks")); - DependencyPath mockDotNetPackage = new DependencyPath("dotnetsdk", this.PlatformSpecifics.GetPackagePath("dotnet")); - DependencyPath mockBombardierPackage = new DependencyPath("bombardier", this.PlatformSpecifics.GetPackagePath("bombardier")); - this.PackageManager.OnGetPackage(mockAspNetBenchPackage.Name).ReturnsAsync(mockAspNetBenchPackage); - this.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); - this.PackageManager.OnGetPackage(mockBombardierPackage.Name).ReturnsAsync(mockBombardierPackage); - } - - this.File.Reset(); - this.File.Setup(f => f.Exists(It.IsAny())) - .Returns(true); - this.Directory.Setup(f => f.Exists(It.IsAny())) - .Returns(true); - this.FileSystem.SetupGet(fs => fs.File).Returns(this.File.Object); - - this.Parameters = new Dictionary() - { - { nameof(AspNetBenchExecutor.PackageName), "aspnetbenchmarks" }, - { nameof(AspNetBenchExecutor.DotNetSdkPackageName), "dotnetsdk" }, - { nameof(AspNetBenchExecutor.BombardierPackageName), "bombardier" }, - { nameof(AspNetBenchExecutor.TargetFramework), "net123.321" }, - { nameof(AspNetBenchExecutor.Port), "12321" } - }; - } - - [Test] - public void AspNetBenchExecutorThrowsIfCannotFindAspNetBenchPackage() - { - this.SetupTest(PlatformID.Win32NT); - this.PackageManager.OnGetPackage("aspnetbenchmarks").ReturnsAsync(value: null); - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); - } - } - - [Test] - public void AspNetBenchExecutorThrowsIfCannotFindBombardierPackage() - { - this.SetupTest(PlatformID.Unix); - this.PackageManager.OnGetPackage("bombardier").ReturnsAsync(value: null); - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); - } - } - - [Test] - public void AspNetBenchExecutorThrowsIfCannotFindDotNetSDKPackage() - { - this.SetupTest(PlatformID.Unix); - this.PackageManager.OnGetPackage("dotnetsdk").ReturnsAsync(value: null); - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); - } - } - - [Test] - public async Task AspNetBenchExecutorRunsTheExpectedWorkloadCommandInLinux() - { - this.SetupTest(PlatformID.Unix); - - string packageDirectory = this.GetPackagePath(); - ProcessStartInfo expectedInfo = new ProcessStartInfo(); - List expectedCommands = new List() - { - $@"sudo chmod +x ""{packageDirectory}/bombardier/linux-x64/bombardier""", - $@"sudo {packageDirectory}/dotnet/dotnet build -c Release -p:BenchmarksTargetFramework=net123.321", - $@"sudo {packageDirectory}/dotnet/dotnet {packageDirectory}/aspnetbenchmarks/src/Benchmarks/bin/Release/net123.321/Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:12321 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive""", - $@"sudo {packageDirectory}/bombardier/linux-x64/bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:12321/json --print r --format json" - }; - - int commandExecuted = 0; - this.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => - { - if (expectedCommands.Any(c => c == $"{exe} {arguments}")) - { - commandExecuted++; - } - - IProcessProxy process = new InMemoryProcess() - { - ExitCode = 0, - OnStart = () => true, - OnHasExited = () => true - }; - string exampleResults = MockFixture.ReadFile(MockFixture.ExamplesDirectory, "Bombardier", "BombardierExample.txt"); - process.StandardOutput.Append(exampleResults); - return process; - }; - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); - } - - Assert.AreEqual(4, commandExecuted); - } - - [Test] - public async Task AspNetBenchExecutorRunsTheExpectedWorkloadCommandInWindows() - { - this.SetupTest(PlatformID.Win32NT); - - string packageDirectory = this.GetPackagePath(); - ProcessStartInfo expectedInfo = new ProcessStartInfo(); - - List expectedCommands = new List() - { - $@"{packageDirectory}\dotnet\dotnet.exe build -c Release -p:BenchmarksTargetFramework=net123.321", - $@"{packageDirectory}\dotnet\dotnet.exe {packageDirectory}\aspnetbenchmarks\src\Benchmarks\bin\Release\net123.321\Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:12321 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive""", - $@"{packageDirectory}\bombardier\win-x64\bombardier.exe --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:12321/json --print r --format json" - }; - - int commandExecuted = 0; - this.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => - { - if (expectedCommands.Any(c => c == $"{exe} {arguments}")) - { - commandExecuted++; - } - - IProcessProxy process = new InMemoryProcess() - { - ExitCode = 0, - OnStart = () => true, - OnHasExited = () => true - }; - string exampleResults = MockFixture.ReadFile(MockFixture.ExamplesDirectory, "Bombardier", "BombardierExample.txt"); - process.StandardOutput.Append(exampleResults); - return process; - }; - - using (TestAspNetBenchExecutor executor = new TestAspNetBenchExecutor(this.Dependencies, this.Parameters)) - { - await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); - } - - Assert.AreEqual(3, commandExecuted); - } - - private class TestAspNetBenchExecutor : AspNetBenchExecutor - { - public TestAspNetBenchExecutor(IServiceCollection dependencies, IDictionary parameters) - : base(dependencies, parameters) - { - } - - public new Task ExecuteAsync(EventContext context, CancellationToken cancellationToken) - { - return base.ExecuteAsync(context, cancellationToken); - } - } - } -} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs new file mode 100644 index 0000000000..668802be28 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetOrchardServerExecutorTests.cs @@ -0,0 +1,245 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + using System.Net; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class AspNetOrchardServerExecutorTests + { + private MockFixture mockFixture; + private DependencyPath mockOrchardCorePackage; + private DependencyPath mockDotNetPackage; + + [Test] + public void AspNetOrchardServerOrchardExecutorThrowsIfCannotFindAspNetOrchardPackage() + { + this.SetupDefaultMockBehaviors(PlatformID.Win32NT); + this.mockFixture.PackageManager.OnGetPackage("orchardcore").ReturnsAsync(value: null); + + using (TestAspNetOrchardServerExecutor executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void AspNetOrchardServerOrchardExecutorThrowsIfCannotFindDotNetSDKPackage() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.PackageManager.OnGetPackage("dotnetsdk").ReturnsAsync(value: null); + + using (TestAspNetOrchardServerExecutor executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public async Task AspNetOrchardServerExecutorRunsTheExpectedWorkloadCommandInLinux() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + + string packageDirectory = this.mockFixture.GetPackagePath(); + ProcessStartInfo expectedInfo = new ProcessStartInfo(); + List expectedCommands = new List() + { + "pkill OrchardCore", + "fuser -n tcp -k 5014", + $@"{packageDirectory}/dotnet/dotnet publish -c Release --sc -f net9.0 {packageDirectory}/orchardcore/src/OrchardCore.Cms.Web/OrchardCore.Cms.Web.csproj", + $@"nohup {packageDirectory}/orchardcore/src/OrchardCore.Cms.Web/bin/Release/net9.0/linux-x64/publish/OrchardCore.Cms.Web --urls http://*:5014" + }; + + int commandExecuted = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => + { + string command = $"{exe} {arguments}"; + if (expectedCommands.Any(c => c == command)) + { + expectedCommands.Remove(command); + } + commandExecuted++; + + IProcessProxy process = new InMemoryProcess() + { + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + return process; + }; + + using (TestAspNetOrchardServerExecutor executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + } + + Assert.AreEqual(4, commandExecuted); + Assert.IsEmpty(expectedCommands); + } + + [Test] + public async Task AspNetOrchardServerExecutorRunsTheExpectedWorkloadCommandInWindows() + { + this.SetupDefaultMockBehaviors(PlatformID.Win32NT); + + string packageDirectory = this.mockFixture.GetPackagePath(); + ProcessStartInfo expectedInfo = new ProcessStartInfo(); + + List expectedCommands = new List() + { + "pkill OrchardCore", + "fuser -n tcp -k 5014", + $@"{packageDirectory}\dotnet\dotnet.exe publish -c Release --sc -f net9.0 {packageDirectory}\orchardcore\src\OrchardCore.Cms.Web\OrchardCore.Cms.Web.csproj", + $@"nohup {packageDirectory}\orchardcore\src\OrchardCore.Cms.Web\bin\Release\net9.0\win-x64\publish\OrchardCore.Cms.Web --urls http://*:5014", + }; + + int commandExecuted = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => + { + string command = $"{exe} {arguments}"; + if (expectedCommands.Any(c => c == command)) + { + expectedCommands.Remove(command); + } + commandExecuted++; + + IProcessProxy process = new InMemoryProcess() + { + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + return process; + }; + + using (TestAspNetOrchardServerExecutor executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + } + + Assert.AreEqual(4, commandExecuted); + Assert.IsEmpty(expectedCommands); + } + + [Test] + public async Task AspNetServerExecutorInitializeAsyncSetsCorrectPaths() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.Layout = new EnvironmentLayout(new List + { + new ClientInstance("Server", "1.2.3.4", ClientRole.Server), + new ClientInstance("Client", "5.6.7.8", ClientRole.Client) + }); + + using (TestAspNetOrchardServerExecutor executor = new TestAspNetOrchardServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.InitializeAsync(EventContext.None, CancellationToken.None); + + // Verify correct paths are set + string expectedAspNetDir = this.mockFixture.Combine( + this.mockOrchardCorePackage.Path, + "src", + "OrchardCore.Cms.Web"); + + Assert.AreEqual(expectedAspNetDir, executor.AspnetOrchardDirectory); + + string expectedDotNetPath = this.mockFixture.Combine( + this.mockDotNetPackage.Path, + "dotnet"); + + Assert.AreEqual(expectedDotNetPath, executor.DotNetExePath); + + // Verify API client is initialized + Assert.IsNotNull(executor.ServerApi); + } + } + + private class TestAspNetOrchardServerExecutor : AspNetOrchardServerExecutor + { + public TestAspNetOrchardServerExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + } + + public string AspnetOrchardDirectory + { + get + { + var field = typeof(AspNetOrchardServerExecutor).GetField("aspnetOrchardDirectory", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public string DotNetExePath + { + get + { + var field = typeof(AspNetOrchardServerExecutor).GetField("dotnetExePath", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public new Task InitializeAsync(EventContext context, CancellationToken cancellationToken) + { + return base.InitializeAsync(context, cancellationToken); + } + } + + private void SetupDefaultMockBehaviors(PlatformID platform) + { + if (platform == PlatformID.Win32NT) + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Win32NT); + this.mockOrchardCorePackage = new DependencyPath("orchardcore", this.mockFixture.PlatformSpecifics.GetPackagePath("orchardcore")); + this.mockDotNetPackage = new DependencyPath("dotnetsdk", this.mockFixture.PlatformSpecifics.GetPackagePath("dotnet")); + this.mockFixture.PackageManager.OnGetPackage(mockOrchardCorePackage.Name).ReturnsAsync(mockOrchardCorePackage); + this.mockFixture.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); + } + else + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Unix); + this.mockOrchardCorePackage = new DependencyPath("orchardcore", this.mockFixture.PlatformSpecifics.GetPackagePath("orchardcore")); + this.mockDotNetPackage = new DependencyPath("dotnetsdk", this.mockFixture.PlatformSpecifics.GetPackagePath("dotnet")); + this.mockFixture.PackageManager.OnGetPackage(mockOrchardCorePackage.Name).ReturnsAsync(mockOrchardCorePackage); + this.mockFixture.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); + } + + this.mockFixture.File.Reset(); + this.mockFixture.File.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.mockFixture.Directory.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.mockFixture.FileSystem.SetupGet(fs => fs.File).Returns(this.mockFixture.File.Object); + + this.mockFixture.Parameters = new Dictionary() + { + { nameof(AspNetOrchardServerExecutor.PackageName), "orchardcore" }, + { nameof(AspNetOrchardServerExecutor.DotNetSdkPackageName), "dotnetsdk" }, + { nameof(AspNetOrchardServerExecutor.TargetFramework), "net9.0" }, + { nameof(AspNetOrchardServerExecutor.ServerPort), "5014" } + }; + + this.mockFixture.ApiClient.OnUpdateState(nameof(State)) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK)); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs new file mode 100644 index 0000000000..cc304ac37d --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/AspNetBench/AspNetServerExecutorTests.cs @@ -0,0 +1,269 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.IO; + using System.Linq; + using System.Net; + using System.Reflection; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + using VirtualClient.Actions.Memtier; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class AspNetServerExecutorTests + { + private MockFixture mockFixture; + private DependencyPath mockAspNetBenchPackage; + private DependencyPath mockDotNetPackage; + + [Test] + public void AspNetServerExecutorThrowsIfCannotFindAspNetBenchPackage() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.PackageManager.OnGetPackage("aspnetbenchmarks").ReturnsAsync(value: null); + + using (TestAspNetServerExecutor executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public void AspNetServerExecutorThrowsIfCannotFindDotNetSDKPackage() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.PackageManager.OnGetPackage("dotnetsdk").ReturnsAsync(value: null); + + using (TestAspNetServerExecutor executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + Assert.ThrowsAsync(() => executor.ExecuteAsync(CancellationToken.None)); + } + } + + [Test] + public async Task AspNetServerExecutorRunsTheExpectedWorkloadCommandInLinux() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + + string packageDirectory = this.mockFixture.GetPackagePath(); + ProcessStartInfo expectedInfo = new ProcessStartInfo(); + List expectedCommands = new List() + { + "pkill dotnet", + "fuser -n tcp -k 9876", + $@"{packageDirectory}/dotnet/dotnet build -c Release -p:BenchmarksTargetFramework=net8.0", + $@"{packageDirectory}/dotnet/dotnet {packageDirectory}/aspnetbenchmarks/src/Benchmarks/bin/Release/net8.0/Benchmarks.dll --nonInteractive true --scenarios json --urls http://*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive""" + }; + + int commandExecuted = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => + { + string command = $"{exe} {arguments}"; + if (expectedCommands.Any(c => c == command)) + { + expectedCommands.Remove(command); + } + commandExecuted++; + + IProcessProxy process = new InMemoryProcess() + { + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + string resultsPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Bombardier", "BombardierExample.txt"); + process.StandardOutput.Append(File.ReadAllText(resultsPath)); + return process; + }; + + using (TestAspNetServerExecutor executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + } + + Assert.AreEqual(4, commandExecuted); // Updated to 4 + Assert.IsEmpty(expectedCommands); + } + + [Test] + public async Task AspNetServerExecutorRunsTheExpectedWorkloadCommandInWindows() + { + this.SetupDefaultMockBehaviors(PlatformID.Win32NT); + + string packageDirectory = this.mockFixture.GetPackagePath(); + ProcessStartInfo expectedInfo = new ProcessStartInfo(); + + List expectedCommands = new List() + { + "pkill dotnet", + "fuser -n tcp -k 9876", + $@"{packageDirectory}\dotnet\dotnet.exe build -c Release -p:BenchmarksTargetFramework=net8.0", + $@"{packageDirectory}\dotnet\dotnet.exe {packageDirectory}\aspnetbenchmarks\src\Benchmarks\bin\Release\net8.0\Benchmarks.dll --nonInteractive true --scenarios json --urls http://*:9876 --server Kestrel --kestrelTransport Sockets --protocol http --header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive""", + }; + + int commandExecuted = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => + { + string command = $"{exe} {arguments}"; + if (expectedCommands.Any(c => c == command)) + { + expectedCommands.Remove(command); + } + commandExecuted++; + + IProcessProxy process = new InMemoryProcess() + { + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + string resultsPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Bombardier", "BombardierExample.txt"); + process.StandardOutput.Append(File.ReadAllText(resultsPath)); + return process; + }; + + using (TestAspNetServerExecutor executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false); + } + + Assert.AreEqual(4, commandExecuted); // Updated to 4 + Assert.IsEmpty(expectedCommands); + } + + [Test] + public async Task AspNetServerExecutorInitializeAsyncSetsCorrectPaths() + { + this.SetupDefaultMockBehaviors(PlatformID.Unix); + this.mockFixture.Layout = new EnvironmentLayout(new List + { + new ClientInstance("Server", "1.2.3.4", ClientRole.Server), + new ClientInstance("Client", "5.6.7.8", ClientRole.Client) + }); + + using (TestAspNetServerExecutor executor = new TestAspNetServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await executor.InitializeAsync(EventContext.None, CancellationToken.None); + + // Verify correct paths are set + string expectedAspNetDir = this.mockFixture.Combine( + this.mockAspNetBenchPackage.Path, + "src", + "Benchmarks"); + + Assert.AreEqual(expectedAspNetDir, executor.AspNetBenchDirectory); + + string expectedDotNetPath = this.mockFixture.Combine( + this.mockDotNetPackage.Path, + "dotnet"); + + Assert.AreEqual(expectedDotNetPath, executor.DotNetExePath); + + // Verify API client is initialized + Assert.IsNotNull(executor.ServerApi); + } + } + + private class TestAspNetServerExecutor : AspNetServerExecutor + { + public TestAspNetServerExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + } + + public string AspNetBenchDirectory + { + get + { + var field = typeof(AspNetServerExecutor).GetField("aspnetBenchDirectory", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public string AspNetBenchDllPath + { + get + { + var field = typeof(AspNetServerExecutor).GetField("aspnetBenchDllPath", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public string DotNetExePath + { + get + { + var field = typeof(AspNetServerExecutor).GetField("dotnetExePath", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + return field?.GetValue(this) as string; + } + } + + public new Task InitializeAsync(EventContext context, CancellationToken cancellationToken) + { + return base.InitializeAsync(context, cancellationToken); + } + } + + private void SetupDefaultMockBehaviors(PlatformID platform) + { + if (platform == PlatformID.Win32NT) + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Win32NT); + this.mockAspNetBenchPackage = new DependencyPath("aspnetbenchmarks", this.mockFixture.PlatformSpecifics.GetPackagePath("aspnetbenchmarks")); + this.mockDotNetPackage = new DependencyPath("dotnetsdk", this.mockFixture.PlatformSpecifics.GetPackagePath("dotnet")); + + this.mockFixture.PackageManager.OnGetPackage(mockAspNetBenchPackage.Name).ReturnsAsync(mockAspNetBenchPackage); + this.mockFixture.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); + + this.mockFixture.ApiClient.OnUpdateState(nameof(ServerState)) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK)); + } + else + { + this.mockFixture = new MockFixture(); + this.mockFixture.Setup(PlatformID.Unix); + + this.mockAspNetBenchPackage = new DependencyPath("aspnetbenchmarks", this.mockFixture.PlatformSpecifics.GetPackagePath("aspnetbenchmarks")); + this.mockDotNetPackage = new DependencyPath("dotnetsdk", this.mockFixture.PlatformSpecifics.GetPackagePath("dotnet")); + this.mockFixture.PackageManager.OnGetPackage(mockAspNetBenchPackage.Name).ReturnsAsync(mockAspNetBenchPackage); + this.mockFixture.PackageManager.OnGetPackage(mockDotNetPackage.Name).ReturnsAsync(mockDotNetPackage); + } + + this.mockFixture.File.Reset(); + this.mockFixture.File.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.mockFixture.Directory.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.mockFixture.FileSystem.SetupGet(fs => fs.File).Returns(this.mockFixture.File.Object); + + this.mockFixture.Parameters = new Dictionary() + { + { nameof(AspNetServerExecutor.PackageName), "aspnetbenchmarks" }, + { nameof(AspNetServerExecutor.DotNetSdkPackageName), "dotnetsdk" }, + { nameof(AspNetServerExecutor.TargetFramework), "net8.0" }, + { nameof(AspNetServerExecutor.ServerPort), "9876" }, + { nameof(AspNetServerExecutor.AspNetCoreThreadCount), "1" }, + { nameof(AspNetServerExecutor.DotNetSystemNetSocketsThreadCount), "1" } + }; + + this.mockFixture.ApiClient.OnUpdateState(nameof(State)) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(HttpStatusCode.OK)); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Nginx/NginxVersionExample.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Nginx/NginxVersionExample.txt new file mode 100644 index 0000000000..fba9d3c630 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Nginx/NginxVersionExample.txt @@ -0,0 +1,4 @@ +nginx version: nginx/1.18.0 (Ubuntu) +built with OpenSSL 1.1.1f 31 Mar 2020 +TLS SNI support enabled +configure arguments: --with-cc-opt='-g -O2 -fdebug-prefix-map=/build/nginx-lUTckl/nginx-1.18.0=. -fstack-protector-strong -Wformat -Werror=format-security -fPIC -Wdate-time -D_FORTIFY_SOURCE=2' --with-ld-opt='-Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-z,now -fPIC' --prefix=/usr/share/nginx --conf-path=/etc/nginx/nginx.conf --http-log-path=/var/log/nginx/access.log --error-log-path=/var/log/nginx/error.log --lock-path=/var/lock/nginx.lock --pid-path=/run/nginx.pid --modules-path=/usr/lib/nginx/modules --http-client-body-temp-path=/var/lib/nginx/body --http-fastcgi-temp-path=/var/lib/nginx/fastcgi --http-proxy-temp-path=/var/lib/nginx/proxy --http-scgi-temp-path=/var/lib/nginx/scgi --http-uwsgi-temp-path=/var/lib/nginx/uwsgi --with-debug --with-compat --with-pcre-jit --with-http_ssl_module --with-http_stub_status_module --with-http_realip_module --with-http_auth_request_module --with-http_v2_module --with-http_dav_module --with-http_slice_module --with-threads --with-http_addition_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_image_filter_module=dynamic --with-http_sub_module --with-http_xslt_module=dynamic --with-stream=dynamic --with-stream_ssl_module --with-mail=dynamic --with-mail_ssl_module diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample1.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample1.txt new file mode 100644 index 0000000000..86f07d2132 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample1.txt @@ -0,0 +1,10 @@ +Running 2m test @ http://10.1.0.15/api_new/10kb + 1 threads and 10000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 102.45ms 17.10ms 134.53ms 76.19% + Req/Sec 0.67 1.15 2.00 66.67% + 21 requests in 2.50m, 6.69KB read + Socket errors: connect 0, read 1645610, write 16, timeout 0 + Non-2xx or 3xx responses: 21 +Requests/sec: 0.14 +Transfer/sec: 45.63B diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample2.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample2.txt new file mode 100644 index 0000000000..7c724b5a07 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkErrorExample2.txt @@ -0,0 +1,14 @@ +Running 30s test @ http://10.1.0.13/index.html + 5 threads and 100 connections + Thread calibration: mean lat.: 1.019ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 1.009ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.988ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.963ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 0.958ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.98ms 468.49us 8.97ms 70.07% + Req/Sec 420.73 49.56 0.89k 84.45% + 59956 requests in 30.00s, 18.64MB read + Non-2xx or 3xx responses: 59956 +Requests/sec: 1998.42 +Transfer/sec: 636.13KB diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample1.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample1.txt new file mode 100644 index 0000000000..560336900d --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample1.txt @@ -0,0 +1,96 @@ +Running 1m test @ https://10.1.0.26/api_new/1kb + 2 threads and 20 connections + Thread calibration: mean lat.: 4197.437ms, rate sampling interval: 15204ms + Thread calibration: mean lat.: 4197.977ms, rate sampling interval: 15147ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 29.27s 12.10s 50.20s 57.59% + Req/Sec 8.17k 163.44 8.32k 66.67% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 29.56s + 75.000% 39.75s + 90.000% 45.97s + 99.000% 49.74s + 99.900% 50.17s + 99.990% 50.23s + 99.999% 50.23s +100.000% 50.23s + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 8372.223 0.000000 2 1.00 + 12525.567 0.100000 81614 1.11 + 16605.183 0.200000 163195 1.25 + 20660.223 0.300000 244674 1.43 + 24805.375 0.400000 326214 1.67 + 29556.735 0.500000 407688 2.00 + 31604.735 0.550000 448517 2.22 + 33619.967 0.600000 489638 2.50 + 35618.815 0.650000 530556 2.86 + 37683.199 0.700000 570904 3.33 + 39747.583 0.750000 611770 4.00 + 40796.159 0.775000 632420 4.44 + 41811.967 0.800000 652450 5.00 + 42827.775 0.825000 672721 5.71 + 43876.351 0.850000 693461 6.67 + 44892.159 0.875000 713577 8.00 + 45449.215 0.887500 723987 8.89 + 45973.503 0.900000 734157 10.00 + 46497.791 0.912500 744472 11.43 + 47054.847 0.925000 754722 13.33 + 47579.135 0.937500 764883 16.00 + 47841.279 0.943750 769668 17.78 + 48103.423 0.950000 774858 20.00 + 48365.567 0.956250 780158 22.86 + 48627.711 0.962500 785303 26.67 + 48857.087 0.968750 789786 32.00 + 48988.159 0.971875 792512 35.56 + 49119.231 0.975000 795257 40.00 + 49250.303 0.978125 797991 45.71 + 49381.375 0.981250 800387 53.33 + 49512.447 0.984375 802822 64.00 + 49577.983 0.985938 804076 71.11 + 49643.519 0.987500 805311 80.00 + 49709.055 0.989062 806567 91.43 + 49774.591 0.990625 807793 106.67 + 49840.127 0.992188 809006 128.00 + 49872.895 0.992969 809631 142.22 + 49905.663 0.993750 810256 160.00 + 49938.431 0.994531 810831 182.86 + 50003.967 0.995313 811970 213.33 + 50036.735 0.996094 812575 256.00 + 50036.735 0.996484 812575 284.44 + 50069.503 0.996875 813155 320.00 + 50069.503 0.997266 813155 365.71 + 50102.271 0.997656 813642 426.67 + 50135.039 0.998047 814120 512.00 + 50135.039 0.998242 814120 568.89 + 50135.039 0.998437 814120 640.00 + 50167.807 0.998633 814706 731.43 + 50167.807 0.998828 814706 853.33 + 50167.807 0.999023 814706 1024.00 + 50167.807 0.999121 814706 1137.78 + 50167.807 0.999219 814706 1280.00 + 50167.807 0.999316 814706 1462.86 + 50200.575 0.999414 815155 1706.67 + 50200.575 0.999512 815155 2048.00 + 50200.575 0.999561 815155 2275.56 + 50200.575 0.999609 815155 2560.00 + 50200.575 0.999658 815155 2925.71 + 50200.575 0.999707 815155 3413.33 + 50200.575 0.999756 815155 4096.00 + 50200.575 0.999780 815155 4551.11 + 50200.575 0.999805 815155 5120.00 + 50200.575 0.999829 815155 5851.43 + 50200.575 0.999854 815155 6826.67 + 50200.575 0.999878 815155 8192.00 + 50200.575 0.999890 815155 9102.22 + 50233.343 0.999902 815242 10240.00 + 50233.343 1.000000 815242 inf +#[Mean = 29266.261, StdDeviation = 12102.812] +#[Max = 50200.576, Total count = 815242] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 978315 requests in 1.00m, 1.17GB read +Requests/sec: 16305.17 +Transfer/sec: 20.01MB diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample2.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample2.txt new file mode 100644 index 0000000000..5a83982898 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample2.txt @@ -0,0 +1,204 @@ +Running 30s test @ http://10.1.0.13/index.html + 2 threads and 100 connections + Thread calibration: mean lat.: 1.137ms, rate sampling interval: 10ms + Thread calibration: mean lat.: 2.015ms, rate sampling interval: 10ms + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.54ms 0.86ms 9.76ms 70.18% + Req/Sec 1.05k 1.39k 5.55k 89.70% + Latency Distribution (HdrHistogram - Recorded Latency) + 50.000% 1.43ms + 75.000% 1.98ms + 90.000% 2.68ms + 99.000% 3.96ms + 99.900% 6.93ms + 99.990% 8.99ms + 99.999% 9.77ms +100.000% 9.77ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.175 0.000000 1 1.00 + 0.566 0.100000 3954 1.11 + 0.776 0.200000 7913 1.25 + 1.003 0.300000 11866 1.43 + 1.226 0.400000 15800 1.67 + 1.427 0.500000 19773 2.00 + 1.517 0.550000 21738 2.22 + 1.610 0.600000 23711 2.50 + 1.715 0.650000 25678 2.86 + 1.837 0.700000 27663 3.33 + 1.982 0.750000 29634 4.00 + 2.067 0.775000 30635 4.44 + 2.163 0.800000 31614 5.00 + 2.261 0.825000 32598 5.71 + 2.381 0.850000 33586 6.67 + 2.519 0.875000 34567 8.00 + 2.601 0.887500 35060 8.89 + 2.683 0.900000 35553 10.00 + 2.769 0.912500 36051 11.43 + 2.867 0.925000 36538 13.33 + 2.981 0.937500 37038 16.00 + 3.047 0.943750 37281 17.78 + 3.105 0.950000 37534 20.00 + 3.163 0.956250 37773 22.86 + 3.235 0.962500 38022 26.67 + 3.327 0.968750 38271 32.00 + 3.375 0.971875 38392 35.56 + 3.435 0.975000 38515 40.00 + 3.507 0.978125 38636 45.71 + 3.589 0.981250 38765 53.33 + 3.689 0.984375 38886 64.00 + 3.749 0.985938 38946 71.11 + 3.825 0.987500 39008 80.00 + 3.911 0.989062 39068 91.43 + 4.011 0.990625 39130 106.67 + 4.167 0.992188 39193 128.00 + 4.271 0.992969 39223 142.22 + 4.383 0.993750 39255 160.00 + 4.535 0.994531 39284 182.86 + 4.719 0.995313 39315 213.33 + 4.967 0.996094 39346 256.00 + 5.087 0.996484 39364 284.44 + 5.383 0.996875 39377 320.00 + 5.623 0.997266 39392 365.71 + 5.851 0.997656 39408 426.67 + 6.087 0.998047 39423 512.00 + 6.195 0.998242 39431 568.89 + 6.331 0.998437 39439 640.00 + 6.543 0.998633 39446 731.43 + 6.663 0.998828 39454 853.33 + 7.011 0.999023 39462 1024.00 + 7.351 0.999121 39466 1137.78 + 7.511 0.999219 39470 1280.00 + 7.603 0.999316 39473 1462.86 + 7.687 0.999414 39477 1706.67 + 7.951 0.999512 39481 2048.00 + 8.099 0.999561 39484 2275.56 + 8.115 0.999609 39485 2560.00 + 8.207 0.999658 39487 2925.71 + 8.335 0.999707 39489 3413.33 + 8.599 0.999756 39491 4096.00 + 8.631 0.999780 39492 4551.11 + 8.783 0.999805 39493 5120.00 + 8.927 0.999829 39494 5851.43 + 8.983 0.999854 39495 6826.67 + 8.991 0.999878 39496 8192.00 + 8.991 0.999890 39496 9102.22 + 9.231 0.999902 39497 10240.00 + 9.231 0.999915 39497 11702.86 + 9.271 0.999927 39498 13653.33 + 9.271 0.999939 39498 16384.00 + 9.271 0.999945 39498 18204.44 + 9.503 0.999951 39499 20480.00 + 9.503 0.999957 39499 23405.71 + 9.503 0.999963 39499 27306.67 + 9.503 0.999969 39499 32768.00 + 9.503 0.999973 39499 36408.89 + 9.767 0.999976 39500 40960.00 + 9.767 1.000000 39500 inf +#[Mean = 1.537, StdDeviation = 0.861] +#[Max = 9.760, Total count = 39500] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + + Latency Distribution (HdrHistogram - Uncorrected Latency (measured without taking delayed starts into account)) + 50.000% 483.00us + 75.000% 1.12ms + 90.000% 1.71ms + 99.000% 2.87ms + 99.900% 5.76ms + 99.990% 8.02ms + 99.999% 8.41ms +100.000% 8.41ms + + Detailed Percentile spectrum: + Value Percentile TotalCount 1/(1-Percentile) + + 0.135 0.000000 1 1.00 + 0.242 0.100000 4008 1.11 + 0.287 0.200000 7981 1.25 + 0.335 0.300000 11912 1.43 + 0.392 0.400000 15829 1.67 + 0.483 0.500000 19763 2.00 + 0.552 0.550000 21732 2.22 + 0.652 0.600000 23706 2.50 + 0.802 0.650000 25676 2.86 + 0.959 0.700000 27650 3.33 + 1.115 0.750000 29633 4.00 + 1.200 0.775000 30626 4.44 + 1.285 0.800000 31612 5.00 + 1.375 0.825000 32597 5.71 + 1.475 0.850000 33586 6.67 + 1.586 0.875000 34567 8.00 + 1.645 0.887500 35064 8.89 + 1.713 0.900000 35550 10.00 + 1.788 0.912500 36050 11.43 + 1.872 0.925000 36541 13.33 + 1.964 0.937500 37032 16.00 + 2.018 0.943750 37282 17.78 + 2.081 0.950000 37532 20.00 + 2.147 0.956250 37779 22.86 + 2.219 0.962500 38023 26.67 + 2.301 0.968750 38268 32.00 + 2.353 0.971875 38395 35.56 + 2.397 0.975000 38514 40.00 + 2.459 0.978125 38638 45.71 + 2.531 0.981250 38761 53.33 + 2.625 0.984375 38888 64.00 + 2.679 0.985938 38945 71.11 + 2.741 0.987500 39007 80.00 + 2.821 0.989062 39068 91.43 + 2.907 0.990625 39130 106.67 + 3.029 0.992188 39193 128.00 + 3.119 0.992969 39223 142.22 + 3.209 0.993750 39254 160.00 + 3.371 0.994531 39284 182.86 + 3.583 0.995313 39315 213.33 + 3.793 0.996094 39346 256.00 + 3.943 0.996484 39363 284.44 + 4.175 0.996875 39377 320.00 + 4.423 0.997266 39392 365.71 + 4.703 0.997656 39408 426.67 + 5.007 0.998047 39423 512.00 + 5.163 0.998242 39431 568.89 + 5.223 0.998437 39439 640.00 + 5.355 0.998633 39446 731.43 + 5.651 0.998828 39454 853.33 + 5.783 0.999023 39462 1024.00 + 5.887 0.999121 39466 1137.78 + 5.971 0.999219 39470 1280.00 + 6.075 0.999316 39473 1462.86 + 6.147 0.999414 39477 1706.67 + 6.715 0.999512 39481 2048.00 + 6.963 0.999561 39483 2275.56 + 7.059 0.999609 39485 2560.00 + 7.195 0.999658 39487 2925.71 + 7.295 0.999707 39489 3413.33 + 7.423 0.999756 39491 4096.00 + 7.571 0.999780 39492 4551.11 + 7.687 0.999805 39493 5120.00 + 7.739 0.999829 39494 5851.43 + 7.823 0.999854 39495 6826.67 + 8.015 0.999878 39496 8192.00 + 8.015 0.999890 39496 9102.22 + 8.163 0.999902 39497 10240.00 + 8.163 0.999915 39497 11702.86 + 8.223 0.999927 39498 13653.33 + 8.223 0.999939 39498 16384.00 + 8.223 0.999945 39498 18204.44 + 8.399 0.999951 39499 20480.00 + 8.399 0.999957 39499 23405.71 + 8.399 0.999963 39499 27306.67 + 8.399 0.999969 39499 32768.00 + 8.399 0.999973 39499 36408.89 + 8.407 0.999976 39500 40960.00 + 8.407 1.000000 39500 inf +#[Mean = 0.782, StdDeviation = 0.678] +#[Max = 8.400, Total count = 39500] +#[Buckets = 27, SubBuckets = 2048] +---------------------------------------------------------- + 58902 requests in 30.00s, 18.31MB read + Non-2xx or 3xx responses: 58902 +Requests/sec: 1963.39 +Transfer/sec: 624.98KB diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample3.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample3.txt new file mode 100644 index 0000000000..015ecfc6bd --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Wrk/wrkStandardExample3.txt @@ -0,0 +1,14 @@ +Running 2m test @ http://10.9.0.7/api_new/10kb + 32 threads and 5000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 4.12ms 2.38ms 219.68ms 73.08% + Req/Sec 38.62k 3.74k 82.02k 72.05% + Latency Distribution + 50% 3.85ms + 75% 5.05ms + 90% 7.27ms + 99% 11.16ms + 184596465 requests in 2.50m, 56.05GB read + Non-2xx or 3xx responses: 184596465 +Requests/sec: 1229838.61 +Transfer/sec: 382.35MB diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs new file mode 100644 index 0000000000..33df870ace --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Nginx/NginxServerExecutorTest.cs @@ -0,0 +1,542 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net.Http; + using System.Reflection; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Common; + using VirtualClient.Contracts; + using Microsoft.CodeAnalysis; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Moq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Telemetry; + using Architecture = System.Runtime.InteropServices.Architecture; + + [TestFixture] + [NUnit.Framework.Category("Unit")] + public class NginxServerExecutorTest + { + private MockFixture mockFixture; + private InMemoryProcess memoryProcess; + private Item serverState; + private TimeSpan timeout = TimeSpan.FromMinutes(10); + private string packageName = "nginxconfiguration"; + + [SetUp] + public void SetupTests() + { + this.serverState = new Item(nameof(State), new State()); + this.mockFixture = new MockFixture(); + this.memoryProcess = new InMemoryProcess + { + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true, + StandardError = new ConcurrentBuffer(new StringBuilder($"nginx version: v1\n")) + }; + } + + [Test] + [TestCase(PlatformID.Win32NT, Architecture.X64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public void NginxServerExecutorThrowsErrorIfPlatformIsWrong(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters.Add(nameof(this.packageName), this.packageName); + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void NginxServerExecutorThrowsErrorPackageIsMissing(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxServerExecutorInitializesAsExpected(PlatformID platform, Architecture architecture) + { + /* + When Nginx Server Initialize, these are the expectations: + 1.Set up Api Client for client and server. + 2.Verify packages are installed and shell scripts exist inside. + 3.Set up reset(execute: "setup - reset.sh") - only applicable for first time + a.This will ensure, server's sysctl configuration is saved so when virtual client exits, it is able to reset the server back to its original state. + 4. Set up content. (execute: "setup-content.sh FileSizeInKB") + a.This will create a file that can be used during testing. + 5. Set up config (execute: "setup-config.sh) + a.This will make changes to server configuration. + 6. Delete any states that is saved in Server. + */ + + this.mockFixture.Setup(platform, architecture); + this.mockFixture.Parameters.Add(nameof(this.packageName), this.packageName); + this.mockFixture.Parameters.Add("FileSizeInKb", 5); + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + + // pkg setup + DependencyFixture fixture = new DependencyFixture(); + fixture.Setup(platform, architecture); + fixture.SetupPackage(this.packageName); + string packagePath = executor.PlatformSpecifics.ToPlatformSpecificPath(fixture.PackageManager.FirstOrDefault(), platform, architecture).Path; + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(y => y == this.packageName), It.IsAny())) + .ReturnsAsync(fixture.PackageManager.FirstOrDefault()); + + string resetFilePath = executor.PlatformSpecifics.Combine(packagePath, "reset.sh"); + string resetOutput = Guid.NewGuid().ToString(); + int processCount = 0; + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (processCount == 0) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-reset.sh"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(resetOutput)); + } + else if (processCount == 1) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-content.sh 5"); + } + else if (processCount == 2) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-config.sh auto Client 1.2.3.5"); + } + else + { + Assert.Fail("Only 3 process expected."); + } + + processCount++; + return this.memoryProcess; + }; + + string[] expectedFiles = new string[] + { + executor.PlatformSpecifics.Combine(packagePath, "setup-reset.sh"), + executor.PlatformSpecifics.Combine(packagePath, "setup-content.sh"), + executor.PlatformSpecifics.Combine(packagePath,"setup-config.sh") + }; + + this.mockFixture.FileSystem.Setup(x => x.File.Exists(It.Is(x => x == resetFilePath))).Returns(false); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.Is(x => x != resetFilePath))) + .Returns(true) + .Callback((string fileName) => + { + if (!expectedFiles.Any(y => y == fileName)) + { + Assert.Fail($"Unexpected File Name: {fileName}. \n{string.Join("\n", expectedFiles)}"); + } + }); + + // Create file for reset + Mock mockFileStream = new Mock(); + this.mockFixture.FileStream.Setup(f => f.New(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(mockFileStream.Object) + .Callback((string path, FileMode mode, FileAccess access, FileShare share) => + { + Assert.AreEqual(resetFilePath, path); + Assert.IsTrue(mode == FileMode.Create); + Assert.IsTrue(access == FileAccess.ReadWrite); + Assert.IsTrue(share == FileShare.ReadWrite); + }); + + mockFileStream + .Setup(x => x.Write(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((byte[] data, int offset, int count) => + { + byte[] byteData = Encoding.Default.GetBytes(resetOutput); + Assert.AreEqual(offset, 0); + Assert.AreEqual(count, byteData.Length); + Assert.AreEqual(data, byteData); + }); + + + + await executor.InitializeAsync().ConfigureAwait(false); + Assert.AreEqual(processCount, 3); + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(4)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxServerExecutorInitializesAsExpectedForSecondTime(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters.Add(nameof(this.packageName), this.packageName); + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + DependencyFixture fixture = new DependencyFixture(); + fixture.Setup(platform, architecture); + fixture.SetupPackage(this.packageName); + string packagePath = executor.PlatformSpecifics.ToPlatformSpecificPath(fixture.PackageManager.FirstOrDefault(), platform, architecture).Path; + + int processCount = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (processCount == 0) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-content.sh 1"); + } + else if (processCount == 1) + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(workingDir, packagePath); + Assert.AreEqual(arguments, "bash setup-config.sh auto Client 1.2.3.5"); + } + else + { + Assert.Fail("Only 2 process expected."); + } + + processCount++; + return this.memoryProcess; + }; + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(y => y == this.packageName), It.IsAny())) + .ReturnsAsync(fixture.PackageManager.FirstOrDefault()); + + var expectedFiles = new string[] + { + executor.PlatformSpecifics.Combine(packagePath, "setup-reset.sh"), + executor.PlatformSpecifics.Combine(packagePath, "setup-content.sh"), + executor.PlatformSpecifics.Combine(packagePath,"setup-config.sh"), + executor.PlatformSpecifics.Combine(packagePath, "reset.sh") + }; + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string fileName) => + { + if(!expectedFiles.Any(y => y == fileName)) + { + Assert.Fail($"Unexpected File Name: {fileName}. \n{string.Join("\n", expectedFiles)}"); + } + }); + + await executor.InitializeAsync().ConfigureAwait(false); + Assert.AreEqual(processCount, 2); + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(4)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxServerExecutorResetsServerAsExpected(PlatformID platform, Architecture architecture) + { + TimeSpan timeout = TimeSpan.FromMinutes(5); + this.mockFixture.Setup(platform, architecture, nameof(State)); + int processCount = 0; + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (processCount == 0) + { + // During every reset, nginx will delete existing content. + // "delete-content.sh" is downloaded from blob. + Assert.AreEqual(arguments, "bash reset.sh"); + } + else + { + Assert.AreEqual(arguments, "systemctl disable nginx", NginxCommand.Stop.ConvertToCommandArgs()); + } + + Assert.AreEqual(command, "sudo"); + Assert.IsNull(workingDir); + processCount++; + return this.memoryProcess; + }; + + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + await executor.ResetNginxAsync().ConfigureAwait(false); + Assert.AreEqual(processCount, 2); + + this.mockFixture.ApiClient.Verify(x => x.UpdateStateAsync( + It.Is(x => x == nameof(State)), + It.Is>(x => x.Definition.Online(null) == false), + It.IsAny(), + It.IsAny>()), + Times.Exactly(1)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void ResetServerWillSwallowExceptions(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => { throw new SchemaException(); }; + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Assert.DoesNotThrowAsync(async () => await executor.ResetNginxAsync().ConfigureAwait(false)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void NginxServerExecutorWillResetServerDuringDispose(PlatformID platform, Architecture architecture) + { + // Dispose will call reset nginx + TimeSpan timeout = TimeSpan.FromMinutes(5); + this.mockFixture.Setup(platform, architecture, nameof(State)); + int processCount = 0; + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (processCount == 0) + { + // During every reset, nginx will delete existing content. + // "delete-content.sh" is downloaded from blob. + Assert.AreEqual(arguments, "bash reset.sh"); + } + else + { + Assert.AreEqual(arguments, "systemctl disable nginx", NginxCommand.Stop.ConvertToCommandArgs()); + } + + Assert.AreEqual(command, "sudo"); + Assert.IsNull(workingDir); + processCount++; + return this.memoryProcess; + }; + + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + executor.Dispose(true); + Assert.AreEqual(processCount, 2); + this.mockFixture.ApiClient.Verify(x => x.UpdateStateAsync( + It.Is(x => x == nameof(State)), + It.Is>(x => x.Definition.Online(null) == false), + It.IsAny(), + It.IsAny>()), + Times.Exactly(1)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxServerExecutorRunsAsExpected(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters.Add(nameof(this.packageName), this.packageName); + this.mockFixture.Parameters.Add(nameof(this.timeout), this.timeout.ToString()); + this.mockFixture.Parameters.Add("pollingInterval", TimeSpan.FromSeconds(1).ToString()); + + int nginxServiceCalls = 0; + int shellScriptCalls = 0; + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + DependencyFixture fixture = new DependencyFixture(); + fixture.Setup(platform, architecture); + fixture.SetupPackage(packageName); + + string packagePath = executor.PlatformSpecifics.ToPlatformSpecificPath(fixture.PackageManager.FirstOrDefault(), platform, architecture).Path; + this.mockFixture.PackageManager.Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())).ReturnsAsync(fixture.PackageManager.FirstOrDefault()); + this.mockFixture.FileSystem.Setup(x => x.File.Exists(It.IsAny())).Returns(true); + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.AreEqual(command, "sudo"); + + if (new[] {"bash setup-content.sh 1", "bash setup-config.sh", "bash setup-config.sh auto Client 1.2.3.5", "bash reset.sh" }.Contains(arguments, StringComparer.OrdinalIgnoreCase)) + { + shellScriptCalls++; + Assert.AreEqual(workingDir, packagePath); + } + else if (new[] { "systemctl restart nginx", "systemctl disable nginx"}.Contains(arguments, StringComparer.OrdinalIgnoreCase)) + { + nginxServiceCalls++; + Assert.IsNull(workingDir); + } + else if (arguments == "nginx -V") + { + nginxServiceCalls++; + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Nginx"); + string outputPath = Path.Combine(examplesDirectory, @"NginxVersionExample.txt"); + string rawText = File.ReadAllText(outputPath); + this.memoryProcess.StandardError = new ConcurrentBuffer(new StringBuilder(rawText)); + } + else + { + Assert.Fail($"Unexpected Arguments: {arguments}"); + } + + return this.memoryProcess; + }; + + + Item onlineClientState = new Item(nameof(State), new State()); + onlineClientState.Definition.Online(true); + + Item expiredState = new Item(nameof(State), new State()); + expiredState.Definition.Timeout(DateTime.UtcNow.AddDays(-1)); + + // Set up for polling online client state. + this.mockFixture.ApiClient.SetupSequence(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK, onlineClientState)) // Polling for online state + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK, expiredState)); // Get client state + + this.mockFixture.ApiClient.Setup(x => x.CreateStateAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + + this.mockFixture.ApiClient.Setup(x => x.UpdateStateAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + + await executor.InitializeAsync().ConfigureAwait(false); + await executor.ExecuteAsync().ConfigureAwait(false); + + Assert.AreEqual(shellScriptCalls, 3); + Assert.AreEqual(nginxServiceCalls, 3); + + // nginx version and local online state + this.mockFixture.ApiClient.Verify(x => x.CreateStateAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>()), Times.Exactly(2)); + + // first loop and reset + this.mockFixture.ApiClient.Verify(x => x.UpdateStateAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny>()), Times.Exactly(2)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void NginxServerExecutorWillResetServerIfFailure(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + int processCount = 0; + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (arguments == "nginx -V") + { + // This will ensure nginx version is empty therefore ArgumentException will be thrown. + this.memoryProcess.StandardError = new ConcurrentBuffer(new StringBuilder(string.Empty)); + } + else if (arguments == "systemctl disable nginx" || arguments == "bash reset.sh") + { + } + else + { + Assert.Fail(); + } + + processCount++; + return this.memoryProcess; + }; + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => await executor.ExecuteAsync().ConfigureAwait(false)); + Assert.AreEqual(processCount, 3); + + // Expected to delete local server state & will create new offline state. + this.mockFixture.ApiClient.Verify(x => x.UpdateStateAsync( + It.Is(x => x == nameof(State)), + It.Is>(x => x.Definition.Online(null) == false), + It.IsAny(), + It.IsAny>()), + Times.Exactly(1)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task NginxExecutorParsesNginxVersion(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Nginx"); + string outputPath = Path.Combine(examplesDirectory, @"NginxVersionExample.txt"); + string rawText = File.ReadAllText(outputPath); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (arguments == "nginx -V") + { + // This will ensure nginx version is empty therefore ArgumentException will be thrown. + this.memoryProcess.StandardError = new ConcurrentBuffer(new StringBuilder(rawText)); + } + else + { + Assert.Fail(); + } + + return this.memoryProcess; + }; + + TestNginxServerExecutor executor = new TestNginxServerExecutor(this.mockFixture); + Dictionary version = await executor.GetNginxVersionAsync().ConfigureAwait(false); + + Assert.AreEqual(version["nginxVersion"], "nginx/1.18.0 (Ubuntu)"); + Assert.IsTrue(version["sslVersion"].Contains("1.1.1f", StringComparison.OrdinalIgnoreCase)); + Assert.AreEqual(version["serverNameIndicationSupport"], "TLS SNI support enabled"); + Assert.IsTrue(version["arguments"].StartsWith("--with-cc-opt='-g -O2")); + } + + private class TestNginxServerExecutor : NginxServerExecutor + { + public TestNginxServerExecutor(MockFixture mockFixture, IDictionary parameters = null) + : base(mockFixture.Dependencies, mockFixture.Parameters) + { + this.ServerApi = mockFixture.ApiClient.Object; + this.ClientApi = mockFixture.ApiClient.Object; + } + + public new void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public async Task InitializeAsync() + { + await base.InitializeAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ExecuteAsync() + { + await base.ExecuteAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ResetNginxAsync() + { + await base.ResetNginxAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task> GetNginxVersionAsync() + { + return await base.GetNginxVersionAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/Wrk2ExecutorTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/Wrk2ExecutorTest.cs new file mode 100644 index 0000000000..f79de2eb51 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/Wrk2ExecutorTest.cs @@ -0,0 +1,437 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Reflection; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Common; + using VirtualClient.Contracts; + using Microsoft.CodeAnalysis; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Moq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Telemetry; + using Architecture = System.Runtime.InteropServices.Architecture; + + [TestFixture] + [Category("Unit")] + public class Wrk2ExecutorTests + { + private string ClientStateId = nameof(ClientStateId); + private string ServerStateId = nameof(ServerStateId); + + private MockFixture mockFixture; + private DependencyFixture dependencyFixture; + private InMemoryProcess memoryProcess; + private Dictionary defaultProperties; + private string packageName = "wrk2"; + private string scriptpackageName = "wrkconfiguration"; + + [SetUp] + public void SetupTests() + { + this.mockFixture = new MockFixture(); + this.memoryProcess = new InMemoryProcess + { + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + + this.defaultProperties = new Dictionary() + { + { "PackageName", this.packageName }, + { "Scenario", "1000r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb" }, + { "CommandArguments", "--latency --threads{ThreadCount} --connections{Connection} --duration{Duration.TotalSeconds}s" }, + { "Connection", 100 }, + { "ThreadCount", 10 }, + { "MaxCoreCount", 10 }, + { "TestDuration", "00:02:30"}, + { "Timeout", "00:20:00"}, + { "FileSizeInKB", 10}, + { "Role", "Client"}, + { "Tags", "Networking,NGINX,WRK2"}, + }; + + dependencyFixture = new DependencyFixture(); + } + + [Test] + [TestCase(PlatformID.Win32NT, Architecture.X64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public void Wrk2ExecutorThrowsErrorIfPlatformIsWrong(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ClientStateId); + this.mockFixture.Parameters = this.defaultProperties; + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + Assert.IsFalse(VirtualClientComponent.IsSupported(executor)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void Wrk2ExecutorThrowsErrorIfPackageIsMissing(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ClientStateId); + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + + this.mockFixture.Parameters = this.defaultProperties; + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.packageName); + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.FirstOrDefault()); + + TestWrk2Executor executor2 = new TestWrk2Executor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor2.InitializeAsync().ConfigureAwait(false); + }); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void Wrk2ExecutorOnlySupportsWrk2(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ClientStateId); + this.mockFixture.Parameters = new Dictionary() + { + { "PackageName", "wrk" }, + }; + + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage("wrk"); + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.FirstOrDefault()); + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + DependencyException exc = Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + + Assert.AreEqual(exc.Message, "TestWrk2Executor did not find correct package in the directory. Supported Package: wrk2. Package Provided: wrk"); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task Wrk2ExecutorInitializesAsExpected(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ServerStateId); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.packageName); + dependencyFixture.SetupPackage(this.scriptpackageName); + TestWrk2Executor dummyExecutor = new TestWrk2Executor(this.mockFixture); + + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.packageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptpackageName).FirstOrDefault(), platform, architecture).Path; + string[] expectedFiles = new string[] + { + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath, "setup-reset.sh"), + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath,"setup-config.sh"), + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath,"reset.sh"), + dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath,"wrk"), + dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath,"runwrk.sh") + }; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string fileName) => + { + if (!expectedFiles.Any(y => y == fileName)) + { + Assert.Fail($"Unexpected File Name: {fileName}. \n{string.Join("\n", expectedFiles)}"); + } + }); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptpackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptpackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.packageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.packageName).FirstOrDefault()); + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(expectedFiles.Count())); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + [Ignore("Unit test is way too complex and needs to be refactored.")] + public async Task Wrk2ExecutorExecutesAsyncAsExpected(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, this.ServerStateId); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.packageName); + dependencyFixture.SetupPackage(this.scriptpackageName); + TestWrk2Executor dummyExecutor = new TestWrk2Executor(this.mockFixture); + + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.packageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptpackageName).FirstOrDefault(), platform, architecture).Path; + string[] expectedFiles = new string[] + { + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath, "setup-reset.sh"), + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath,"setup-config.sh"), + dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath,"reset.sh"), + dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath,"wrk"), + dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath,"runwrk.sh") + }; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string fileName) => + { + if (!expectedFiles.Any(y => y == fileName)) + { + Assert.Fail($"Unexpected File Name: {fileName}. \n{string.Join("\n", expectedFiles)}"); + } + }); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptpackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptpackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.packageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.packageName).FirstOrDefault()); + + Item serverState = new Item(this.ServerStateId, new State()); + serverState.Definition.Online(true); + Item clientState = new Item(this.ClientStateId, new State()); + + // create local state + this.mockFixture.ApiClient + .Setup(x => x.CreateStateAsync(It.Is(y => y == this.ClientStateId), It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)) + .Callback((string id, State state, CancellationToken _, IAsyncPolicy __) => + { + Assert.IsTrue(state.Online()); + }); + + this.mockFixture.ApiClient.Setup(s => s.GetStateAsync(It.Is(x => x == this.ServerStateId), It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => + { + Item result = new Item(this.ServerStateId, new State()); + result.Definition.Online(true); + return this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, result); + }); + + this.mockFixture.ApiClient.Setup(s => s.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => + { + // both ServerStateId and serverVersion will be covered with this set up. + Item result = new Item(this.ServerStateId, new State()); + result.Definition.Online(true); + return this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, result); + }); + + // update local state before running working + this.mockFixture.ApiClient + .Setup(x => x.UpdateStateAsync(It.Is(y => y == this.ClientStateId), It.IsAny>(), It.IsAny(), It.IsAny>())) + .Callback((string id, Item state, CancellationToken _, IAsyncPolicy __) => + { + Assert.IsTrue(state.Definition.Online()); + }); + + // delete local state before exiting + this.mockFixture.ApiClient.Setup(x => x.DeleteStateAsync(It.Is(y => y == this.ClientStateId), It.IsAny(), It.IsAny>())); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.IsTrue(command == "sudo" || command.EndsWith("wrk")); + + if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")} --version")); + } + + return this.memoryProcess; + }; + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + Assert.ThrowsAsync(async () => + { + await executor.ExecuteAsync().ConfigureAwait(false); + }, "wrk2 did not write metrics to console."); + + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(8)); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.IsTrue(command == "sudo" || command.EndsWith("wrk")); + + if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")} --version")); + } + else + { + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + + return this.memoryProcess; + }; + TestWrk2Executor executor2 = new TestWrk2Executor(this.mockFixture); + await executor2.InitializeAsync().ConfigureAwait(false); + await executor2.ExecuteAsync().ConfigureAwait(false); + + this.mockFixture.FileSystem.Verify(x => x.File.Exists(It.IsAny()), Times.Exactly(15)); + this.mockFixture.ApiClient.Verify(x => x.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())); + } + + [Test] + [TestCase("-L -R 1000 -t 10 -c 50 -d 100s --timeout 10s https://{serverip}/api_new/1kb")] + [TestCase("{serverip}_{clientip}_{reverseproxyip}_{serverip}")] + public void WrkClientExecutorReturnsCorrectArguments(string commandArg) + { + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); + ClientInstance serverInstance = new ClientInstance(name: nameof(State), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(State), ipAddress: "5.6.7.8", role: ClientRole.Client); + ClientInstance reverseProxyInstance = new ClientInstance(name: nameof(State), ipAddress: "9.0.1.2", role: ClientRole.ReverseProxy); + + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance, reverseProxyInstance }); + + this.mockFixture.Parameters = new Dictionary() + { + { "CommandArguments", commandArg } + }; + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + string expected = executor.GetCommandLineArguments(); + + string result = commandArg + .Replace("{serverip}", "1.2.3.4") + .Replace("{clientip}", "5.6.7.8") + .Replace("{reverseproxyip}", "9.0.1.2"); + + Assert.AreEqual(expected, result); + } + + [Test] + public async Task WrkClientExecutorReturnsCorrectArguments() + { + string commandArgumentInput = @"--rate 1000 --latency --threads 10 --connections 100 --duration 100s --timeout 10s https://{serverip}/api_new/5kb"; + ClientInstance serverInstance = new ClientInstance(name: nameof(State), ipAddress: "1.2.3.4", role: ClientRole.Server); + + string directory = @"some/random\dir/name/"; + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance }); + this.mockFixture.Parameters = new Dictionary() + { + {"CommandArguments", commandArgumentInput }, + { "Scenario", "bar" }, + { "ToolName", "wrk" }, + { "PackageName", "wrk" }, + { "FileSizeInKB", 5}, + { "TestDuration", TimeSpan.FromSeconds(60).ToString() } + }; + + TestWrk2Executor executor = new TestWrk2Executor(this.mockFixture); + executor.PackageDirectory = directory; + string result = executor.GetCommandLineArguments(); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string file) => + { + string result = executor.Combine(directory, "runwrk.sh"); + Assert.AreEqual(file, result); + }); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + string results = commandArgumentInput.Replace("{serverip}", "1.2.3.4"); + Assert.AreEqual(command, "sudo"); + if (arguments.Contains("--version")) + { + Assert.AreEqual(arguments, $"bash {executor.Combine(directory, "runwrk.sh")} --version"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + } + else + { + Assert.AreEqual(arguments, $"bash {executor.Combine(directory, "runwrk.sh")} \"{results}\""); + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + Assert.AreEqual(workingDir, directory); + return this.memoryProcess; + }; + + await executor.ExecuteWorkloadAsync(result, workingDir: directory).ConfigureAwait(false); + } + + public void SetUpWorkloadOutput() + { + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample2.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + + private class TestWrk2Executor : Wrk2Executor + { + public TestWrk2Executor(MockFixture mockFixture) + : base(mockFixture.Dependencies, mockFixture.Parameters) + { + this.ServerApi = mockFixture.ApiClient.Object; + this.ClientFlowRetryPolicy = Policy.NoOpAsync(); + this.ClientRetryPolicy = Policy.NoOpAsync(); + } + + public new void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public string GetCommandLineArguments() + { + return base.GetCommandLineArguments(CancellationToken.None); + } + + public async Task InitializeAsync() + { + await base.InitializeAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ExecuteAsync() + { + await base.ExecuteAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ExecuteWorkloadAsync(string commandArguments, string workingDir) + { + await base.ExecuteWorkloadAsync(commandArguments, workingDir, EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs new file mode 100644 index 0000000000..1adfe0e1a5 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkExecutorTest.cs @@ -0,0 +1,687 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Reflection; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Common; + using VirtualClient.Actions.Memtier; + using VirtualClient.Contracts; + using Microsoft.CodeAnalysis; + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Moq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Telemetry; + using Architecture = System.Runtime.InteropServices.Architecture; + + [TestFixture] + [Category("Unit")] + public class WrkExecutorTests + { + private MockFixture mockFixture; + private DependencyFixture dependencyFixture; + private InMemoryProcess memoryProcess; + private Dictionary defaultProperties; + private string wrkPackageName = "wrk"; + private string scriptPackageName = "wrkconfiguration"; + + [SetUp] + public void SetupTests() + { + this.mockFixture = new MockFixture(); + this.memoryProcess = new InMemoryProcess + { + ExitCode = 0, + OnStart = () => true, + OnHasExited = () => true + }; + this.defaultProperties = new Dictionary() + { + { "PackageName", this.wrkPackageName }, + { "Scenario", "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb" }, + { "CommandArguments", "--latency --threads{ThreadCount} --connections{Connection} --duration{Duration.TotalSeconds}s" }, + { "Connection", 100 }, + { "ThreadCount", 10 }, + { "MaxCoreCount", 10 }, + { "TestDuration", "00:02:30"}, + { "Timeout", "00:20:00"}, + { "FileSizeInKB", 10}, + { "Role", "Client"}, + { "Tags", "Networking,NGINX,WRK"}, + }; + + dependencyFixture = new DependencyFixture(); + } + + public void SetSingleServerInstance() + { + // Setup: + // One server instance running on port 9876 with affinity to 4 logical processors + this.mockFixture.ApiClient.OnGetState(nameof(ServerState)) + .ReturnsAsync(this.mockFixture.CreateHttpResponse( + HttpStatusCode.OK, + new Item(nameof(ServerState), new ServerState(new List + { + new PortDescription + { + CpuAffinity = "0,1,2,3", + Port = 9876 + } + })))); + + this.mockFixture.ApiClientManager.Setup(mgr => mgr.GetOrCreateApiClient(It.IsAny(), It.IsAny())) + .Returns((id, instance) => this.mockFixture.ApiClient.Object); + + this.mockFixture.ApiClient.OnGetHeartbeat() + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + + this.mockFixture.ApiClient.OnGetServerOnline() + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + } + + [Test] + [TestCase("-L -t 10 -c 50 -d 100s --timeout 10s https://{serverip}/api_new/1kb")] + [TestCase("{serverip}_{clientip}_{reverseproxyip}")] + public void WrkClientExecutorReturnsCorrectArguments(string commandArg) + { + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + ClientInstance reverseProxyInstance = new ClientInstance(name: nameof(ClientRole.ReverseProxy), ipAddress: "9.0.1.2", role: ClientRole.ReverseProxy); + + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance, reverseProxyInstance }); + + this.mockFixture.Parameters = new Dictionary() + { + { "CommandArguments", commandArg } + }; + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + string results = commandArg + .Replace("{serverip}", "1.2.3.4") + .Replace("{clientip}", "5.6.7.8") + .Replace("{reverseproxyip}", "9.0.1.2"); + + Assert.AreEqual(executor.GetCommandLineArguments(), results); + } + + [Test] + public async Task WrkClientExecutorRunsWorkloadWithCorrectArguments() + { + string commandArgumentInput = @"--latency --threads 5 --connections 100 --duration 60s --timeout 10s https://{serverip}/api_new/5kb"; + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + ClientInstance reverseProxyInstance = new ClientInstance(name: nameof(ClientRole.ReverseProxy), ipAddress: "9.0.1.2", role: ClientRole.ReverseProxy); + + string directory = @"some/random\dir/name/"; + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance, reverseProxyInstance }); + this.mockFixture.Parameters = new Dictionary() + { + {"CommandArguments", commandArgumentInput }, + { "Scenario", "bar" }, + { "ToolName", "wrk" }, + { "PackageName", "wrk" }, + { "FileSizeInKB", 5}, + { "TestDuration", TimeSpan.FromSeconds(60).ToString() }, + { "TargetService", "server"} + }; + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + executor.PackageDirectory = directory; + this.SetUpWorkloadOutput(); + string result = executor.GetCommandLineArguments(); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string file) => + { + string result = executor.Combine(directory, "runwrk.sh"); + Assert.AreEqual(file, result); + }); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + string results = commandArgumentInput.Replace("{serverip}", "1.2.3.4"); + Assert.AreEqual(command, "sudo"); + if (arguments.Contains("--version")) + { + Assert.AreEqual(arguments, $"bash {executor.Combine(directory, "runwrk.sh")} --version"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + } + else + { + Assert.AreEqual(arguments, $"bash {executor.Combine(directory, "runwrk.sh")} \"{results}\""); + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + Assert.AreEqual(workingDir, directory); + return this.memoryProcess; + }; + + await executor.ExecuteWorkloadAsync(result, workingDir: directory).ConfigureAwait(false); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task WrkClientExecutorSetsServerWarmedUpFlagAfterWarmupExecution(PlatformID platform, Architecture architecture) + { + // Arrange + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance }); + + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + + // Setup parameters with WarmUp=true + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.Parameters.Add("WarmUp", true); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((string packageName, CancellationToken cancellationToken) => + dependencyFixture.PackageManager.FirstOrDefault(pkg => pkg.Name == packageName)); + + // Setup API client for server heartbeat and state + this.mockFixture.ApiClient.Setup(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => + { + Item result = new Item(nameof(State), new State()); + result.Definition.Online(true); + return this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, result); + }); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + } + else + { + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + + return this.memoryProcess; + }; + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + + // Act + await executor.InitializeAsync().ConfigureAwait(false); + await executor.ExecuteAsync().ConfigureAwait(false); + + // Assert + Assert.IsTrue(executor.GetIsServerWarmedUp(), "IsServerWarmedUp flag should be set to true after warm-up execution"); + } + + [Test] + [TestCase(PlatformID.Win32NT, Architecture.X64)] + [TestCase(PlatformID.Win32NT, Architecture.Arm64)] + public void WrkClientExecutorThrowsErrorIfPlatformIsWrong(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters = this.defaultProperties; + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + Assert.IsFalse(VirtualClientComponent.IsSupported(executor)); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void WrkExecutorOnlySupportsWrkandWrk2(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Parameters = new Dictionary() + { + { "PackageName", "wrk2" }, + }; + + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage("wrk2"); + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.FirstOrDefault()); + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + DependencyException exc = Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + + Assert.AreEqual(exc.Message, "TestWrkExecutor did not find correct package in the directory. Supported Package: wrk. Package Provided: wrk2"); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public void WrkClientExecutorThrowsErrorIfPackageIsMissing(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.SetSingleServerInstance(); + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor.InitializeAsync().ConfigureAwait(false); + }); + + this.mockFixture.Parameters = this.defaultProperties; + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.FirstOrDefault()); + + TestWrkExecutor executor2 = new TestWrkExecutor(this.mockFixture); + Assert.ThrowsAsync(async () => + { + await executor2.InitializeAsync().ConfigureAwait(false); + }); + + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task WrkClientExecutorSkipsExecutionWhenWarmupAndServerIsWarmedUp(PlatformID platform, Architecture architecture) + { + // Arrange + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance }); + + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.Parameters.Add("WarmUp", true); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync((string packageName, CancellationToken cancellationToken) => + dependencyFixture.PackageManager.FirstOrDefault(pkg => pkg.Name == packageName)); + + // Used to track if ExecuteWorkloadAsync is called + bool workloadExecuted = false; + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + executor.SetIsServerWarmedUp(true); // Simulate server already warmed up + + // Override the ExecuteWorkloadAsync to track if it's called + executor.ExecuteWorkloadAsyncCallback = () => workloadExecuted = true; + + // Act + await executor.InitializeAsync().ConfigureAwait(false); + await executor.ExecuteAsync().ConfigureAwait(false); + + // Assert + Assert.IsFalse(workloadExecuted, "WorkloadAsync should not be executed when WarmUp=true and server is already warmed up"); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task WrkClientExecutorSetsUpWrkClientForFirstTime(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + TestWrkExecutor dummyExecutor = new TestWrkExecutor(this.mockFixture); + + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault(), platform, architecture).Path; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.Is(x => x == "reset.sh"))) + .Returns(false); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.Is(x => x != "reset.sh"))) + .Returns(true); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.wrkPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault()); + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64)] + [TestCase(PlatformID.Unix, Architecture.Arm64)] + public async Task WrkClientExecutorInitializesAsExpected(PlatformID platform, Architecture architecture) + { + this.mockFixture.Setup(platform, architecture, nameof(State)); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + TestWrkExecutor dummyExecutor = new TestWrkExecutor(this.mockFixture); + + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault(), platform, architecture).Path; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string file) => + { + if (file.EndsWith("wrk")) + { + string result = dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "wrk"); + Assert.AreEqual(file, result); + } + else + { + string result = dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath, Path.GetFileName(file)); + Assert.AreEqual(file, result); + } + }); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.wrkPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault()); + + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + } + + [Test] + [TestCase(PlatformID.Unix, Architecture.X64, "server")] + [TestCase(PlatformID.Unix, Architecture.Arm64, "server")] + [TestCase(PlatformID.Unix, Architecture.X64, "rp")] + [TestCase(PlatformID.Unix, Architecture.Arm64, "rp")] + public async Task WrkClientExecutorExecutesAsyncAsExpected(PlatformID platform, Architecture architecture, string targetService) + { + ClientInstance serverInstance = new ClientInstance(name: nameof(ClientRole.Server), ipAddress: "1.2.3.4", role: ClientRole.Server); + ClientInstance clientInstance = new ClientInstance(name: nameof(ClientRole.Client), ipAddress: "5.6.7.8", role: ClientRole.Client); + ClientInstance reverseProxyInstance = new ClientInstance(name: nameof(ClientRole.ReverseProxy), ipAddress: "9.0.1.2", role: ClientRole.ReverseProxy); + + this.mockFixture.Setup(platform, architecture, nameof(State)); + this.mockFixture.Layout = new EnvironmentLayout(new List() { serverInstance, clientInstance, reverseProxyInstance }); + dependencyFixture.Setup(platform, architecture); + dependencyFixture.SetupPackage(this.wrkPackageName); + dependencyFixture.SetupPackage(this.scriptPackageName); + TestWrkExecutor dummyExecutor = new TestWrkExecutor(this.mockFixture); + string wrkPackagePath = dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault().Path; + string scriptPackagePath = dummyExecutor.ToPlatformSpecificPath(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault(), platform, architecture).Path; + + this.mockFixture.Parameters = this.defaultProperties; + this.mockFixture.Parameters.Add("TargetService", targetService); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true) + .Callback((string file) => + { + if (file.EndsWith("wrk")) + { + string result = dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "wrk"); + Assert.AreEqual(file, result); + } + else if (file.EndsWith("runwrk.sh")) + { + string result = dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh"); + Assert.AreEqual(file, result); + } + else + { + string result = dummyExecutor.PlatformSpecifics.Combine(scriptPackagePath, Path.GetFileName(file)); + Assert.AreEqual(file, result); + } + }); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.scriptPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.scriptPackageName).FirstOrDefault()); + + this.mockFixture.PackageManager + .Setup(x => x.GetPackageAsync(It.Is(x => x == this.wrkPackageName), It.IsAny())) + .ReturnsAsync(dependencyFixture.PackageManager.Where(x => x.Name == this.wrkPackageName).FirstOrDefault()); + + // create local state + this.mockFixture.ApiClient + .Setup(x => x.CreateStateAsync(It.Is(y => y == nameof(State)), It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.mockFixture.CreateHttpResponse(System.Net.HttpStatusCode.OK)) + .Callback((string id, State state, CancellationToken _, IAsyncPolicy __) => + { + Assert.IsTrue(state.Online()); + }); + + this.mockFixture.ApiClient.Setup(client => client.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())) + .ReturnsAsync(() => + { + Item result = new Item(nameof(State), new State()); + result.Definition.Online(true); + return this.mockFixture.CreateHttpResponse(HttpStatusCode.OK, result); + }); + + // delete local state before exiting + this.mockFixture.ApiClient.Setup(x => x.DeleteStateAsync(It.Is(y => y == nameof(State)), It.IsAny(), It.IsAny>())); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.AreEqual(command, "sudo"); + + if (arguments.Contains("chmod")) + { + Assert.AreEqual(arguments, $"chmod +x \"{dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "wrk")}\""); + } + else if (arguments.Contains("setup-config")) + { + Assert.AreEqual(arguments, $"bash setup-config.sh"); + } + else if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")} --version")); + } + else + { + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")}")); + } + + return this.memoryProcess; + }; + //this.SetUpWorkloadOutput(); + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + await executor.InitializeAsync().ConfigureAwait(false); + Assert.ThrowsAsync(async () => + { + await executor.ExecuteAsync().ConfigureAwait(false); + }); + + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.AreEqual(command, "sudo"); + + if (arguments.Contains("chmod")) + { + Assert.AreEqual(arguments, $"chmod +x \"{dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "wrk")}\""); + } + else if (arguments.Contains("setup-config")) + { + Assert.AreEqual(arguments, $"bash setup-config.sh"); + } + else if (arguments.Contains("--version")) + { + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("wrk 4.2.0 [epoll] Copyright (C) 2012 Will Glozer")); + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")} --version")); + } + else + { + Assert.IsTrue(arguments.StartsWith($"bash {dummyExecutor.PlatformSpecifics.Combine(wrkPackagePath, "runwrk.sh")}")); + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(File.ReadAllText(outputPath))); + } + + return this.memoryProcess; + }; + TestWrkExecutor executor2 = new TestWrkExecutor(this.mockFixture); + await executor2.InitializeAsync().ConfigureAwait(false); + await executor2.ExecuteAsync().ConfigureAwait(false); + + this.mockFixture.ApiClient.Verify(x => x.GetStateAsync(It.IsAny(), It.IsAny(), It.IsAny>())); + } + + [Test] + public void GetWrkVersionReturnsCorrectVersion() + { + // Arrange + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + string expectedVersion = "4.2.0"; + string wrkOutput = $"wrk {expectedVersion} [epoll] Copyright (C) 2012 Will Glozer"; + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true); + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.AreEqual(command, "sudo"); + Assert.AreEqual(arguments, $"bash {executor.Combine(executor.PackageDirectory, WrkExecutor.WrkRunShell)} --version"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder(wrkOutput)); + return this.memoryProcess; + }; + + string actualVersion = executor.GetWrkVersion(); + Assert.AreEqual(expectedVersion, actualVersion); + } + + [Test] + public void GetWrkVersion_ThrowsException_WhenVersionCannotBeParsed() + { + // Arrange + this.mockFixture.Setup(PlatformID.Unix, Architecture.X64); + TestWrkExecutor executor = new TestWrkExecutor(this.mockFixture); + + this.mockFixture.FileSystem + .Setup(x => x.File.Exists(It.IsAny())) + .Returns(true); + + // Process returns invalid output that doesn't contain version info + this.mockFixture.ProcessManager.OnCreateProcess = (command, arguments, workingDir) => + { + Assert.AreEqual(command, "sudo"); + Assert.IsTrue(arguments.Contains("--version")); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("Invalid output without version")); + return this.memoryProcess; + }; + + // Act & Assert + WorkloadException exception = Assert.Throws(() => executor.GetWrkVersion()); + Assert.AreEqual("Failed to parse wrk version from output.", exception.Message); + } + + + public void SetUpWorkloadOutput() + { + string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + this.memoryProcess.StandardOutput = new ConcurrentBuffer(new StringBuilder("")); + } + + private class TestWrkExecutor : WrkExecutor + { + public Action ExecuteWorkloadAsyncCallback { get; set; } + + public TestWrkExecutor(MockFixture mockFixture) + : base(mockFixture.Dependencies, mockFixture.Parameters) + { + this.ServerApi = mockFixture.ApiClient.Object; + this.ReverseProxyApi = mockFixture.ApiClient.Object; + this.ClientFlowRetryPolicy = Policy.NoOpAsync(); + this.ClientRetryPolicy = Policy.NoOpAsync(); + } + + public new void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + public string GetCommandLineArguments() + { + return base.GetCommandLineArguments(CancellationToken.None); + } + + public bool GetIsServerWarmedUp() + { + return base.IsServerWarmedUp; + } + + public string GetWrkVersion() + { + return base.GetWrkVersion(EventContext.None, CancellationToken.None); + } + + public void SetIsServerWarmedUp(bool value) + { + base.IsServerWarmedUp = value; + } + + public async Task InitializeAsync() + { + await base.InitializeAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task SetupWrkClient() + { + await base.SetupWrkClient(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + + public async Task ExecuteWorkloadAsync(string commandArguments, string workingDir) + { + await base.ExecuteWorkloadAsync(commandArguments, workingDir, EventContext.None, CancellationToken.None).ConfigureAwait(false); + ExecuteWorkloadAsyncCallback?.Invoke(); + } + + public async Task ExecuteAsync() + { + await base.ExecuteAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs new file mode 100644 index 0000000000..648808b058 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Wrk/WrkMetricsParserTest.cs @@ -0,0 +1,174 @@ +using System.Collections.Generic; +using System.IO; +using System.Reflection; +using VirtualClient.Contracts; +using NUnit.Framework; +using VirtualClient; +using VirtualClient.Actions; + +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + [TestFixture] + [Category("Unit")] + public class WrkMetricsParserTest + { + private static string examplesDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Examples", "Wrk"); + + [Test] + [TestCase(true)] + [TestCase(false)] + [TestCase(null)] + public void WRKParsesResultsCorrectly01(bool emitSpectrum) + { + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample1.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + IList actualMetrics = parser.Parse(emitSpectrum); + + Assert.AreEqual(parser.GetTestConfig(), "Running 1m test @ https://10.1.0.26/api_new/1kb with 2 threads and 20 connections"); + MetricAssert.Exists(actualMetrics, "latency_p50", 29.56 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p75", 39.75 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p90", 45.97 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99", 49.74 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_9", 50.17 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_99", 50.23 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_999", 50.23 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p100", 50.23 * 1000, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "requests/sec", 16305.17); + MetricAssert.Exists(actualMetrics, "transfers/sec", 20.01, MetricUnit.Megabytes); + + if (emitSpectrum == true) + { + + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_000000", 8372.223); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_887500", 45449.215); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_997266", 50069.503); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_999805", 50200.575); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_999902", 50233.343); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p1_000000", 50233.343); + } + + } + + [Test] + public void WRKParsesResultsCorrectly02() + { + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample2.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + IList actualMetrics = parser.Parse(true); + + // Error + MetricAssert.Exists(actualMetrics, "Non-2xx or 3xx responses", 58902); + + // Raw data + MetricAssert.Exists(actualMetrics, "requests/sec", 1963.39); + MetricAssert.Exists(actualMetrics, "transfers/sec", 0.61033203125, MetricUnit.Megabytes); + + //Latency Distribution + MetricAssert.Exists(actualMetrics, "latency_p50", 1.43 , MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p75", 1.98 , MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p90", 2.68 , MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99", 3.96, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_9", 6.93, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_99", 8.99, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99_999", 9.77, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p100", 9.77, MetricUnit.Milliseconds); + + //Uncorrected Latency Distribution + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p50", 483 * 0.001, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p75", 1.12, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p90", 1.71, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p99", 2.87, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p99_9", 5.76, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p99_99", 8.02, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p99_999", 8.41, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_p100", 8.41, MetricUnit.Milliseconds); + + // latency spectrum + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_000000", 0.175); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_775000", 2.067); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_937500", 2.981); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_998437", 6.331); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_999219", 7.511); + MetricAssert.Exists(actualMetrics, "latency_spectrum_p0_999969", 9.503); + + // Uncorrected latency spectrum + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_000000", 0.135); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_100000", 0.242); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_200000", 0.287); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_981250", 2.531); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_996094", 3.793); + MetricAssert.Exists(actualMetrics, "uncorrected_latency_spectrum_p0_999939", 8.223); + + Assert.AreEqual(parser.GetTestConfig(), "Running 30s test @ http://10.1.0.13/index.html with 2 threads and 100 connections"); + } + + [Test] + public void WRKParsesResultsCorrectly03() + { + string outputPath = Path.Combine(examplesDirectory, @"wrkStandardExample3.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + IList actualMetrics = parser.Parse(true); + + MetricAssert.Exists(actualMetrics, "Non-2xx or 3xx responses", 184596465); + Assert.AreEqual(parser.GetTestConfig(), "Running 2m test @ http://10.9.0.7/api_new/10kb with 32 threads and 5000 connections"); + // Raw data + MetricAssert.Exists(actualMetrics, "requests/sec", 1229838.61); + MetricAssert.Exists(actualMetrics, "transfers/sec", 382.35, MetricUnit.Megabytes); + + //Latency Distribution + MetricAssert.Exists(actualMetrics, "latency_p50", 3.85, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p75", 5.05, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p90", 7.27, MetricUnit.Milliseconds); + MetricAssert.Exists(actualMetrics, "latency_p99", 11.16, MetricUnit.Milliseconds); + } + + [Test] + public void WRKParsesErrorCorrectly01() + { + string outputPath = Path.Combine(examplesDirectory, @"wrkErrorExample1.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + Assert.AreEqual(parser.GetTestConfig(), "Running 2m test @ http://10.1.0.15/api_new/10kb with 1 threads and 10000 connections"); + + WorkloadException exc = Assert.Throws(() => + { + parser.Parse(); + }); + + Assert.AreEqual(exc.Message, "Socket errors: connect 0, read 1645610, write 16, timeout 0"); + } + + [Test] + public void WRKParsesErrorCorrectly02() + { + string outputPath = Path.Combine(examplesDirectory, @"wrkErrorExample2.txt"); + string rawText = File.ReadAllText(outputPath); + WrkMetricParser testParser = new WrkMetricParser(rawText); + + WrkMetricParser parser = new WrkMetricParser(rawText); + Assert.AreEqual(parser.GetTestConfig(), "Running 30s test @ http://10.1.0.13/index.html with 5 threads and 100 connections"); + + Assert.DoesNotThrow(() => { parser.Parse(); }); + + IList actualMetrics = parser.Parse(); + MetricAssert.Exists(actualMetrics, "Non-2xx or 3xx responses", 59956); + + MetricAssert.Exists(actualMetrics, "requests/sec", 1998.42); + MetricAssert.Exists(actualMetrics, "transfers/sec", 636.13/1024.0, MetricUnit.Megabytes); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchBaseExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchBaseExecutor.cs deleted file mode 100644 index 1628f0192e..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchBaseExecutor.cs +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.IO.Abstractions; - using System.Runtime.InteropServices; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.CodeAnalysis; - using Microsoft.Extensions.DependencyInjection; - using VirtualClient.Common; - using VirtualClient.Common.Extensions; - using VirtualClient.Common.Platform; - using VirtualClient.Common.Telemetry; - using VirtualClient.Contracts; - using VirtualClient.Contracts.Metadata; - using static System.Net.Mime.MediaTypeNames; - - /// - /// The AspNetBench workload executor. - /// - [SupportedPlatforms("linux-arm64,linux-x64,win-arm64,win-x64")] - public abstract class AspNetBenchBaseExecutor : VirtualClientMultiRoleComponent - { - private IFileSystem fileSystem; - private IPackageManager packageManager; - private IStateManager stateManager; - private ISystemManagement systemManagement; - - private string dotnetExePath; - private string aspnetBenchDirectory; - private string aspnetBenchDllPath; - private string bombardierFilePath; - private string wrkFilePath; - private string serverArgument; - private string clientArgument; - - /// - /// Constructor for - /// - /// Provides required dependencies to the component. - /// Parameters defined in the profile or supplied on the command line. - public AspNetBenchBaseExecutor(IServiceCollection dependencies, IDictionary parameters) - : base(dependencies, parameters) - { - this.systemManagement = this.Dependencies.GetService(); - this.packageManager = this.systemManagement.PackageManager; - this.stateManager = this.systemManagement.StateManager; - this.fileSystem = this.systemManagement.FileSystem; - } - - /// - /// The name of the package where the AspNetBench package is downloaded. - /// - public string TargetFramework - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.TargetFramework)).ToLower(); - } - } - - /// - /// The port for ASPNET to run. - /// - public string Port - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.Port), "9876"); - } - } - - /// - /// The name of the package where the bombardier package is downloaded. - /// - public string BombardierPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.BombardierPackageName), "bombardier"); - } - } - - /// - /// The name of the package where the wrk package is downloaded. - /// - public string WrkPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.WrkPackageName), "wrk"); - } - } - - /// - /// The name of the package where the DotNetSDK package is downloaded. - /// - public string DotNetSdkPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.DotNetSdkPackageName), "dotnetsdk"); - } - } - - /// - /// ASPNETCORE_threadCount - /// - public string AspNetCoreThreadCount - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.AspNetCoreThreadCount), 1); - } - } - - /// - /// DOTNET_SYSTEM_NET_SOCKETS_THREAD_COUNT - /// - public string DotNetSystemNetSocketsThreadCount - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.DotNetSystemNetSocketsThreadCount), 1); - } - } - - /// - /// wrk commandline - /// - public string WrkCommandLine - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchBaseExecutor.WrkCommandLine), string.Empty); - } - } - - /// - /// Initializes the environment for execution of the AspNetBench workload. - /// - protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - // This workload needs three packages: aspnetbenchmarks, dotnetsdk, bombardier - DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, CancellationToken.None) - .ConfigureAwait(false); - - if (workloadPackage != null) - { - // the directory we are looking for is at the src/Benchmarks - this.aspnetBenchDirectory = this.Combine(workloadPackage.Path, "src", "Benchmarks"); - } - - try - { - // Check for Bombardier Package, if not available try wrk package - DependencyPath bombardierPackage = await this.GetPlatformSpecificPackageAsync(this.BombardierPackageName, cancellationToken); - - if (bombardierPackage != null) - { - this.bombardierFilePath = this.Combine(bombardierPackage.Path, this.Platform == PlatformID.Unix ? "bombardier" : "bombardier.exe"); - await this.systemManagement.MakeFileExecutableAsync(this.bombardierFilePath, this.Platform, cancellationToken); - } - } - catch (DependencyException) - { - DependencyPath wrkPackage = await this.packageManager.GetPackageAsync(this.WrkPackageName, cancellationToken); - - if (wrkPackage != null) - { - this.wrkFilePath = this.Combine(wrkPackage.Path, "wrk"); - await this.systemManagement.MakeFileExecutableAsync(this.wrkFilePath, this.Platform, cancellationToken); - } - } - } - - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - /// - protected async Task BuildAspNetBenchAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - DependencyPath dotnetSdkPackage = await this.packageManager.GetPackageAsync(this.DotNetSdkPackageName, CancellationToken.None) - .ConfigureAwait(false); - - if (dotnetSdkPackage == null) - { - throw new DependencyException( - $"The expected DotNet SDK package does not exist on the system or is not registered.", - ErrorReason.WorkloadDependencyMissing); - } - - this.dotnetExePath = this.Combine(dotnetSdkPackage.Path, this.Platform == PlatformID.Unix ? "dotnet" : "dotnet.exe"); - // ~/vc/packages/dotnet/dotnet build -c Release -p:BenchmarksTargetFramework=net9.0 - // Build the aspnetbenchmark project - string buildArgument = $"build -c Release -p:BenchmarksTargetFramework={this.TargetFramework}"; - await this.ExecuteCommandAsync(this.dotnetExePath, buildArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken) - .ConfigureAwait(false); - - // "C:\Users\vcvmadmin\Benchmarks\src\Benchmarks\bin\Release\net9.0\Benchmarks.dll" - this.aspnetBenchDllPath = this.Combine( - this.aspnetBenchDirectory, - "bin", - "Release", - this.TargetFramework, - "Benchmarks.dll"); - } - - /// - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// - protected void CaptureMetrics(IProcessProxy process, EventContext telemetryContext) - { - try - { - this.MetadataContract.AddForScenario( - "AspNetBench", - $"{this.clientArgument},{this.serverArgument}", - toolVersion: null); - - this.MetadataContract.Apply(telemetryContext); - - WrkMetricParser parser = new WrkMetricParser(process.StandardOutput.ToString()); - - this.Logger.LogMetrics( - toolName: "AspNetBench", - scenarioName: $"ASP.NET_{this.TargetFramework}_Performance", - process.StartTime, - process.ExitTime, - parser.Parse(), - metricCategorization: "json", - scenarioArguments: $"Client: {this.clientArgument} | Server: {this.serverArgument}", - this.Tags, - telemetryContext); - } - catch (Exception exc) - { - throw new WorkloadResultsException($"Failed to parse bombardier output.", exc, ErrorReason.InvalidResults); - } - } - - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected Task StartAspNetServerAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - // Example: - // dotnet \Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:5000 --server Kestrel --kestrelTransport Sockets --protocol http - // --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" --header "Connection: keep-alive" - - string options = $"--nonInteractive true --scenarios json --urls http://*:{this.Port} --server Kestrel --kestrelTransport Sockets --protocol http"; - string headers = @"--header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive"""; - this.serverArgument = $"{this.aspnetBenchDllPath} {options} {headers}"; - - return this.ExecuteCommandAsync(this.dotnetExePath, this.serverArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken); - } - - /// - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected async Task RunBombardierAsync(string ipAddress, EventContext telemetryContext, CancellationToken cancellationToken) - { - using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) - { - // https://pkg.go.dev/github.com/codesenberg/bombardier - // ./bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:5000/json --print r --format json - this.clientArgument = $"--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ipAddress}:{this.Port}/json --print r --format json"; - - using (IProcessProxy process = await this.ExecuteCommandAsync(this.bombardierFilePath, this.clientArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken, runElevated: true) - .ConfigureAwait(false)) - { - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "AspNetBench", logToFile: true); - - process.ThrowIfWorkloadFailed(); - this.CaptureMetrics(process, telemetryContext); - } - } - } - } - - /// - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected async Task RunWrkAsync(string ipAddress, EventContext telemetryContext, CancellationToken cancellationToken) - { - using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) - { - // https://pkg.go.dev/github.com/codesenberg/bombardier - // ./wrk -t 256 -c 256 -d 15s --timeout 10s http://10.1.0.23:9876/json --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" - this.clientArgument = this.WrkCommandLine; - this.clientArgument = this.clientArgument.Replace("{ipAddress}", ipAddress); - this.clientArgument = this.clientArgument.Replace("{port}", this.Port); - - using (IProcessProxy process = await this.ExecuteCommandAsync(this.wrkFilePath, this.clientArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken, runElevated: true) - .ConfigureAwait(false)) - { - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "wrk", logToFile: true); - - process.ThrowIfWorkloadFailed(); - this.CaptureMetrics(process, telemetryContext); - } - } - } - } - } -} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchClientExecutor.cs deleted file mode 100644 index f8af30830a..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchClientExecutor.cs +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.Linq; - using System.Net; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Extensions.DependencyInjection; - using Microsoft.Extensions.Logging; - using Polly; - using VirtualClient; - using VirtualClient.Actions.Memtier; - using VirtualClient.Actions.NetworkPerformance; - using VirtualClient.Common; - using VirtualClient.Common.Contracts; - using VirtualClient.Common.Extensions; - using VirtualClient.Common.Telemetry; - using VirtualClient.Contracts; - using VirtualClient.Contracts.Metadata; - - /// - /// AspNetBench Benchmark Client Executor. - /// - public class AspNetBenchClientExecutor : AspNetBenchBaseExecutor - { - /// - /// Initializes a new instance of the class. - /// - /// Provides all of the required dependencies to the Virtual Client component. - /// An enumeration of key-value pairs that can control the execution of the component./param> - public AspNetBenchClientExecutor(IServiceCollection dependencies, IDictionary parameters = null) - : base(dependencies, parameters) - { - this.PollingTimeout = TimeSpan.FromMinutes(40); - } - - /// - /// Executes client side. - /// - protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - await this.WaitForRoleAsync(ClientRole.Server, telemetryContext, cancellationToken).ConfigureAwait(false); - string serverIPAddress = this.GetLayoutClientInstances(ClientRole.Server).First().IPAddress; - await this.RunWrkAsync(serverIPAddress, telemetryContext, cancellationToken).ConfigureAwait(false); - } - - /// - /// Initializes the environment and dependencies for client of AspNetBench Benchmark workload. - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - await base.InitializeAsync(telemetryContext, cancellationToken).ConfigureAwait(false); - - this.RegisterToTerminateRole(ClientRole.Server); - } - } -} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchExecutor.cs deleted file mode 100644 index b67cc23cfc..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchExecutor.cs +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.IO.Abstractions; - using System.Runtime.InteropServices; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.CodeAnalysis; - using Microsoft.Extensions.DependencyInjection; - using VirtualClient.Common; - using VirtualClient.Common.Extensions; - using VirtualClient.Common.Telemetry; - using VirtualClient.Contracts; - using VirtualClient.Contracts.Metadata; - - /// - /// The AspNetBench workload executor. - /// - public class AspNetBenchExecutor : VirtualClientComponent - { - private IFileSystem fileSystem; - private IPackageManager packageManager; - private IStateManager stateManager; - private ISystemManagement systemManagement; - - private string dotnetExePath; - private string aspnetBenchDirectory; - private string aspnetBenchDllPath; - private string bombardierFilePath; - private string serverArgument; - private string clientArgument; - - private Action killServer; - - /// - /// Constructor for - /// - /// Provides required dependencies to the component. - /// Parameters defined in the profile or supplied on the command line. - public AspNetBenchExecutor(IServiceCollection dependencies, IDictionary parameters) - : base(dependencies, parameters) - { - this.systemManagement = this.Dependencies.GetService(); - this.packageManager = this.systemManagement.PackageManager; - this.stateManager = this.systemManagement.StateManager; - this.fileSystem = this.systemManagement.FileSystem; - } - - /// - /// The name of the package where the AspNetBench package is downloaded. - /// - public string TargetFramework - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchExecutor.TargetFramework)).ToLower(); - } - } - - /// - /// The port for ASPNET to run. - /// - public string Port - { - get - { - // Lower case to prevent build path issue. - return this.Parameters.GetValue(nameof(AspNetBenchExecutor.Port), "9876"); - } - } - - /// - /// The name of the package where the bombardier package is downloaded. - /// - public string BombardierPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchExecutor.BombardierPackageName), "bombardier"); - } - } - - /// - /// The name of the package where the DotNetSDK package is downloaded. - /// - public string DotNetSdkPackageName - { - get - { - return this.Parameters.GetValue(nameof(AspNetBenchExecutor.DotNetSdkPackageName), "dotnetsdk"); - } - } - - /// - /// Executes the AspNetBench workload. - /// - protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - Task serverTask = this.StartAspNetServerAsync(cancellationToken); - await this.RunBombardierAsync(telemetryContext, cancellationToken); - - this.killServer.Invoke(); - } - - /// - /// Initializes the environment for execution of the AspNetBench workload. - /// - protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - // This workload needs three packages: aspnetbenchmarks, dotnetsdk, bombardier - DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, CancellationToken.None, throwIfNotfound: true); - - // the directory we are looking for is at the src/Benchmarks - this.aspnetBenchDirectory = this.Combine(workloadPackage.Path, "src", "Benchmarks"); - - DependencyPath bombardierPackage = await this.GetPlatformSpecificPackageAsync(this.BombardierPackageName, cancellationToken); - this.bombardierFilePath = this.Combine(bombardierPackage.Path, this.Platform == PlatformID.Unix ? "bombardier" : "bombardier.exe"); - - await this.systemManagement.MakeFileExecutableAsync(this.bombardierFilePath, this.Platform, cancellationToken); - - DependencyPath dotnetSdkPackage = await this.packageManager.GetPackageAsync(this.DotNetSdkPackageName, CancellationToken.None); - - if (dotnetSdkPackage == null) - { - throw new DependencyException( - $"The expected DotNet SDK package does not exist on the system or is not registered.", - ErrorReason.WorkloadDependencyMissing); - } - - this.dotnetExePath = this.Combine(dotnetSdkPackage.Path, this.Platform == PlatformID.Unix ? "dotnet" : "dotnet.exe"); - - // ~/vc/packages/dotnet/dotnet build -c Release -p:BenchmarksTargetFramework=net9.0 - // Build the aspnetbenchmark project - string buildArgument = $"build -c Release -p:BenchmarksTargetFramework={this.TargetFramework}"; - await this.ExecuteCommandAsync(this.dotnetExePath, buildArgument, this.aspnetBenchDirectory, cancellationToken); - - // "C:\Users\vcvmadmin\Benchmarks\src\Benchmarks\bin\Release\net9.0\Benchmarks.dll" - this.aspnetBenchDllPath = this.Combine( - this.aspnetBenchDirectory, - "bin", - "Release", - this.TargetFramework, - "Benchmarks.dll"); - } - - private void CaptureMetrics(IProcessProxy process, EventContext telemetryContext) - { - try - { - this.MetadataContract.AddForScenario( - "AspNetBench", - $"{this.clientArgument},{this.serverArgument}", - toolVersion: null); - - this.MetadataContract.Apply(telemetryContext); - - BombardierMetricsParser parser = new BombardierMetricsParser(process.StandardOutput.ToString()); - - this.Logger.LogMetrics( - toolName: "AspNetBench", - scenarioName: $"ASP.NET_{this.TargetFramework}_Performance", - process.StartTime, - process.ExitTime, - parser.Parse(), - metricCategorization: "json", - scenarioArguments: $"Client: {this.clientArgument} | Server: {this.serverArgument}", - this.Tags, - telemetryContext); - } - catch (Exception exc) - { - throw new WorkloadResultsException($"Failed to parse bombardier output.", exc, ErrorReason.InvalidResults); - } - } - - private Task StartAspNetServerAsync(CancellationToken cancellationToken) - { - // Example: - // dotnet \Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:5000 --server Kestrel --kestrelTransport Sockets --protocol http - // --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" --header "Connection: keep-alive" - - string options = $"--nonInteractive true --scenarios json --urls http://localhost:{this.Port} --server Kestrel --kestrelTransport Sockets --protocol http"; - string headers = @"--header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive"""; - this.serverArgument = $"{this.aspnetBenchDllPath} {options} {headers}"; - - return this.ExecuteCommandAsync(this.dotnetExePath, this.serverArgument, this.aspnetBenchDirectory, cancellationToken, isServer: true); - } - - private async Task RunBombardierAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) - { - // https://pkg.go.dev/github.com/codesenberg/bombardier - // ./bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:5000/json --print r --format json - this.clientArgument = $"--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://localhost:{this.Port}/json --print r --format json"; - - using (IProcessProxy process = await this.ExecuteCommandAsync(this.bombardierFilePath, this.clientArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken, runElevated: true) - .ConfigureAwait(false)) - { - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext, "AspNetBench", logToFile: true); - - process.ThrowIfWorkloadFailed(); - this.CaptureMetrics(process, telemetryContext); - } - } - } - } - - private async Task ExecuteCommandAsync(string pathToExe, string commandLineArguments, string workingDirectory, CancellationToken cancellationToken, bool isServer = false) - { - if (!cancellationToken.IsCancellationRequested) - { - this.Logger.LogTraceMessage($"Executing process '{pathToExe}' '{commandLineArguments}' at directory '{workingDirectory}'."); - - EventContext telemetryContext = EventContext.Persisted() - .AddContext("command", pathToExe) - .AddContext("commandArguments", commandLineArguments); - - using (IProcessProxy process = this.systemManagement.ProcessManager.CreateElevatedProcess(this.Platform, pathToExe, commandLineArguments, workingDirectory)) - { - if (isServer) - { - this.killServer = () => process.SafeKill(this.Logger); - } - - this.CleanupTasks.Add(() => process.SafeKill(this.Logger)); - await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); - - if (!cancellationToken.IsCancellationRequested) - { - await this.LogProcessDetailsAsync(process, telemetryContext); - - if (!isServer) - { - // We will kill the server at the end, exit code is -1, and we don't want it to log as failure. - process.ThrowIfWorkloadFailed(); - } - } - } - } - } - } -} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchServerExecutor.cs deleted file mode 100644 index db60f3b6b6..0000000000 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetBenchServerExecutor.cs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace VirtualClient.Actions -{ - using System; - using System.Collections.Generic; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Extensions.DependencyInjection; - using VirtualClient; - using VirtualClient.Actions.Memtier; - using VirtualClient.Common.Contracts; - using VirtualClient.Common.Telemetry; - - /// - /// Redis Benchmark Client Executor. - /// - public class AspNetBenchServerExecutor : AspNetBenchBaseExecutor - { - /// - /// Initializes a new instance of the class. - /// - /// Provides all of the required dependencies to the Virtual Client component. - /// An enumeration of key-value pairs that can control the execution of the component./param> - public AspNetBenchServerExecutor(IServiceCollection dependencies, IDictionary parameters = null) - : base(dependencies, parameters) - { - } - - /// - /// - /// - /// Provides context information that will be captured with telemetry events. - /// A token that can be used to cancel the operation. - /// - protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) - { - await this.BuildAspNetBenchAsync(telemetryContext, cancellationToken).ConfigureAwait(false); - - await this.StartAspNetServerAsync(telemetryContext, cancellationToken).ConfigureAwait(false); - await this.WaitAsync(cancellationToken) - .ConfigureAwait(false); - } - - private async Task GetServerStateAsync(IApiClient serverApiClient, CancellationToken cancellationToken) - { - Item state = await serverApiClient.GetStateAsync( - nameof(ServerState), - cancellationToken); - - if (state == null) - { - throw new WorkloadException( - $"Expected server state information missing. The server did not return state indicating the details for the Memcached server(s) running.", - ErrorReason.WorkloadUnexpectedAnomaly); - } - - return state.Definition; - } - } -} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs new file mode 100644 index 0000000000..bd48d2541e --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetOrchardServerExecutor.cs @@ -0,0 +1,398 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO.Abstractions; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// AspNet Orchard Server Executor. + /// + public class AspNetOrchardServerExecutor : VirtualClientMultiRoleComponent + { + private Task serverProcess; + private bool disposed; + private IFileSystem fileSystem; + private IPackageManager packageManager; + private IStateManager stateManager; + private ISystemManagement systemManagement; + + private string dotnetExePath; + private string aspnetOrchardDirectory; + private string aspnetOrchardPublishPath; + + /// + /// Initializes a new instance of the class. + /// + /// Provides all of the required dependencies to the Virtual Client component. + /// An enumeration of key-value pairs that can control the execution of the component./param> + public AspNetOrchardServerExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + this.systemManagement = this.Dependencies.GetService(); + this.packageManager = this.systemManagement.PackageManager; + this.stateManager = this.systemManagement.StateManager; + this.fileSystem = this.systemManagement.FileSystem; + this.ServerRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(3, (retries) => TimeSpan.FromSeconds(retries)); + } + + /// + /// The name of the package where the AspNetBench package is downloaded. + /// + public string TargetFramework + { + get + { + // Lower case to prevent build path issue. + return this.Parameters.GetValue(nameof(AspNetOrchardServerExecutor.TargetFramework)).ToLower(); + } + } + + /// + /// The port to run for Orchard Server. + /// + public string ServerPort + { + get + { + return this.Parameters.GetValue(nameof(AspNetOrchardServerExecutor.ServerPort), "5014"); + } + } + + /// + /// API Client that is used to communicate with server-hosted instance of the Virtual Client Server. + /// + public IApiClient ServerApi { get; set; } + + /// + /// The name of the package where the wrk package is downloaded. + /// + public string WrkPackageName + { + get + { + return this.Parameters.GetValue(nameof(AspNetOrchardServerExecutor.WrkPackageName), "wrk"); + } + } + + /// + /// The name of the package where the DotNetSDK package is downloaded. + /// + public string DotNetSdkPackageName + { + get + { + return this.Parameters.GetValue(nameof(AspNetOrchardServerExecutor.DotNetSdkPackageName), "dotnetsdk"); + } + } + + /// + /// Whether to bind the workload process to specific CPU cores using + /// numactl (Linux) or processor affinity bitmask (Windows). + /// Default: false. + /// + public bool BindToCores + { + get + { + return this.Parameters.GetValue(nameof(this.BindToCores), defaultValue: false); + } + } + + /// + /// The CPU core affinity specification. Supports ranges ("0-3"), + /// comma-separated ("0,2,4,6"), or mixed ("0-3,8-11"). + /// Required when BindToCores is true. + /// + public string CoreAffinity + { + get + { + this.Parameters.TryGetValue(nameof(this.CoreAffinity), out IConvertible value); + return value?.ToString(); + } + } + + /// + /// A retry policy to apply to the server when starting to handle transient issues that + /// would otherwise prevent it from starting successfully. + /// + protected IAsyncPolicy ServerRetryPolicy { get; set; } + + /// + /// Disposes of resources used by the executor including shutting down any + /// instances of Redis server running. + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + if (!this.disposed) + { + // We MUST stop the server instance from running before VC exits or it will continue running until explicitly stopped. + this.KillServerInstancesAsync(null, CancellationToken.None) + .GetAwaiter().GetResult(); + this.disposed = true; + } + } + } + + /// + /// Initializes the environment for execution of the AspNetBench workload. + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + // This workload needs two packages: orchardcms and dotnetsdk + DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, cancellationToken) + .ConfigureAwait(false); + if (workloadPackage == null) + { + throw new DependencyException( + $"The expected workload package '{this.PackageName}' does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } + else + { + this.aspnetOrchardDirectory = this.Combine(workloadPackage.Path, "src", "OrchardCore.Cms.Web"); + } + + DependencyPath dotnetSdkPackage = await this.packageManager.GetPackageAsync(this.DotNetSdkPackageName, cancellationToken) + .ConfigureAwait(false); + if (dotnetSdkPackage == null) + { + throw new DependencyException( + $"The expected DotNet SDK package does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } + + this.dotnetExePath = this.Combine(dotnetSdkPackage.Path, this.Platform == PlatformID.Unix ? "dotnet" : "dotnet.exe"); + + this.InitializeApiClients(); + } + + /// + /// Initializes API client. + /// + protected void InitializeApiClients() + { + IApiClientManager clientManager = this.Dependencies.GetService(); + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + } + + /// + /// + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + /// + protected async Task BuildAspNetOrchardAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + string buildArgument = $"publish -c Release --sc -f {this.TargetFramework} {this.Combine(this.aspnetOrchardDirectory, "OrchardCore.Cms.Web.csproj")}"; + await this.ExecuteCommandAsync(this.dotnetExePath, buildArgument, this.aspnetOrchardDirectory, telemetryContext, cancellationToken) + .ConfigureAwait(false); + + // bin/Release/net9.0/linux-x64/publish + this.aspnetOrchardPublishPath = this.Combine( + this.aspnetOrchardDirectory, + "bin", + "Release", + this.TargetFramework, + this.PlatformArchitectureName, + "publish"); + } + + /// + /// Validates required parameters. + /// + protected override void Validate() + { + base.Validate(); + if (this.BindToCores) + { + this.ThrowIfParameterNotDefined(nameof(this.CoreAffinity)); + } + } + + /// + /// + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + protected override Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return this.Logger.LogMessageAsync($"{this.TypeName}.ExecuteServer", telemetryContext, async () => + { + try + { + this.SetServerOnline(false); + + await this.ServerApi.PollForHeartbeatAsync(TimeSpan.FromMinutes(5), cancellationToken); + + await this.DeleteStateAsync(telemetryContext, cancellationToken); + await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + await this.BuildAspNetOrchardAsync(telemetryContext, cancellationToken); + this.StartServerInstances(telemetryContext, cancellationToken); + + await this.SaveStateAsync(telemetryContext, cancellationToken); + this.SetServerOnline(true); + + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) + { + await Task.WhenAny(this.serverProcess); + // A cancellation is request, then we allow each of the server instances + // to gracefully exit. If a cancellation was not requested, it means that one + // or more of the server instances exited and we will want to allow the component + // to start over restarting the servers. + if (cancellationToken.IsCancellationRequested) + { + await Task.WhenAll(this.serverProcess); + } + } + } + catch + { + this.SetServerOnline(false); + await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + throw; + } + }); + } + + private Task DeleteStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + return this.Logger.LogMessageAsync($"{this.TypeName}.DeleteState", relatedContext, async () => + { + using (HttpResponseMessage response = await this.ServerApi.DeleteStateAsync(nameof(State), cancellationToken)) + { + relatedContext.AddResponseContext(response); + if (response.StatusCode != HttpStatusCode.NoContent) + { + response.ThrowOnError(ErrorReason.HttpNonSuccessResponse); + } + } + }); + } + + private Task KillServerInstancesAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + // These commands will kill ALL the existing OrchardCore Processes in the system and free up the Server port. + this.Logger.LogTraceMessage($"{this.TypeName}.KillServerInstances"); + this.ExecuteCommandAsync("pkill", "OrchardCore", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); + this.ExecuteCommandAsync("fuser", $"-n tcp -k {this.ServerPort}", this.aspnetOrchardDirectory, telemetryContext, cancellationToken); + + return this.WaitAsync(TimeSpan.FromSeconds(3), cancellationToken); + } + + private void StartServerInstances(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + + this.Logger.LogMessage($"{this.TypeName}.StartServerInstances", relatedContext, () => + { + try + { + string command = "nohup"; + string workingDirectory = this.aspnetOrchardDirectory; + string commandArguments = $"{this.Combine(this.aspnetOrchardPublishPath, "OrchardCore.Cms.Web")} --urls http://*:{this.ServerPort}"; + + relatedContext.AddContext("command", command); + relatedContext.AddContext("commandArguments", commandArguments); + relatedContext.AddContext("workingDir", workingDirectory); + + this.serverProcess = this.StartServerInstanceAsync(command, commandArguments, workingDirectory, relatedContext, cancellationToken); + } + catch (OperationCanceledException) + { + // Expected whenever certain operations (e.g. Task.Delay) are cancelled. + } + }); + } + + private Task StartServerInstanceAsync(string command, string commandArguments, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken) + { + return (this.ServerRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => + { + try + { + string effectiveCoreAffinity = this.BindToCores ? this.CoreAffinity : null; + + var (process, affinityConfig) = WorkloadAffinitySupport.CreateProcessWithOptionalAffinity( + this.systemManagement.ProcessManager, + this.Platform, + command, + commandArguments, + workingDirectory, + effectiveCoreAffinity); + + WorkloadAffinitySupport.AddAffinityContext(telemetryContext, this.BindToCores, this.CoreAffinity, affinityConfig); + + this.CleanupTasks.Add(() => process.SafeKill(this.Logger)); + + using (process) + { + await WorkloadAffinitySupport.StartAndWaitWithAffinityAsync( + process, this.Platform, affinityConfig, cancellationToken) + .ConfigureAwait(false); + + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "Orchard"); + process.ThrowIfWorkloadFailed(successCodes: new int[] { 0 }); + } + } + } + catch (OperationCanceledException) + { + // Expected whenever certain operations (e.g. Task.Delay) are cancelled. + } + catch (Exception exc) + { + this.Logger.LogMessage( + $"{this.TypeName}.StartServerInstanceError", + LogLevel.Error, + telemetryContext.Clone().AddError(exc)); + + throw; + } + }); + } + + private Task SaveStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + return this.Logger.LogMessageAsync($"{this.TypeName}.SaveState", relatedContext, async () => + { + Item serverState = new Item(nameof(State), new State()); + serverState.Definition.Online(true); + using (HttpResponseMessage response = await this.ServerApi.UpdateStateAsync(serverState.Id, serverState, cancellationToken)) + { + relatedContext.AddResponseContext(response); + response.ThrowOnError(ErrorReason.HttpNonSuccessResponse); + } + }); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs new file mode 100644 index 0000000000..04a2b5cb38 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/AspNetServerExecutor.cs @@ -0,0 +1,425 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO.Abstractions; + using System.Linq; + using System.Net; + using System.Net.Http; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// AspNet Server Executor. + /// + [SupportedPlatforms("linux-arm64,linux-x64,win-arm64,win-x64")] + public class AspNetServerExecutor : VirtualClientMultiRoleComponent + { + private Task serverProcess; + private bool disposed; + private IFileSystem fileSystem; + private IPackageManager packageManager; + private IStateManager stateManager; + private ISystemManagement systemManagement; + + private string dotnetExePath; + private string aspnetBenchDirectory; + private string aspnetBenchDllPath; + + /// + /// Initializes a new instance of the class. + /// + /// Provides all of the required dependencies to the Virtual Client component. + /// An enumeration of key-value pairs that can control the execution of the component./param> + public AspNetServerExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + this.systemManagement = this.Dependencies.GetService(); + this.packageManager = this.systemManagement.PackageManager; + this.stateManager = this.systemManagement.StateManager; + this.fileSystem = this.systemManagement.FileSystem; + this.ServerRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(3, (retries) => TimeSpan.FromSeconds(retries)); + } + + /// + /// The name of the package where the AspNetBench package is downloaded. + /// + public string TargetFramework + { + get + { + // Lower case to prevent build path issue. + return this.Parameters.GetValue(nameof(AspNetServerExecutor.TargetFramework)).ToLower(); + } + } + + /// + /// The port for ASPNET to run. + /// + public string ServerPort + { + get + { + return this.Parameters.GetValue(nameof(AspNetServerExecutor.ServerPort), "9876"); + } + } + + /// + /// API Client that is used to communicate with server-hosted instance of the Virtual Client Server. + /// + public IApiClient ServerApi { get; set; } + + /// + /// The name of the package where the DotNetSDK package is downloaded. + /// + public string DotNetSdkPackageName + { + get + { + return this.Parameters.GetValue(nameof(AspNetServerExecutor.DotNetSdkPackageName), "dotnetsdk"); + } + } + + /// + /// ASPNETCORE_threadCount + /// + public string AspNetCoreThreadCount + { + get + { + return this.Parameters.GetValue(nameof(AspNetServerExecutor.AspNetCoreThreadCount), "1"); + } + } + + /// + /// DOTNET_SYSTEM_NET_SOCKETS_THREAD_COUNT + /// + public string DotNetSystemNetSocketsThreadCount + { + get + { + return this.Parameters.GetValue(nameof(AspNetServerExecutor.DotNetSystemNetSocketsThreadCount), "1"); + } + } + + /// + /// Whether to bind the workload process to specific CPU cores using + /// numactl (Linux) or processor affinity bitmask (Windows). + /// Default: false. + /// + public bool BindToCores + { + get + { + return this.Parameters.GetValue(nameof(this.BindToCores), defaultValue: false); + } + } + + /// + /// The CPU core affinity specification. Supports ranges ("0-3"), + /// comma-separated ("0,2,4,6"), or mixed ("0-3,8-11"). + /// Required when BindToCores is true. + /// + public string CoreAffinity + { + get + { + this.Parameters.TryGetValue(nameof(this.CoreAffinity), out IConvertible value); + return value?.ToString(); + } + } + + /// + /// A retry policy to apply to the server when starting to handle transient issues that + /// would otherwise prevent it from starting successfully. + /// + protected IAsyncPolicy ServerRetryPolicy { get; set; } + + /// + /// Disposes of resources used by the executor including shutting down any + /// instances of ASP.NET server running. + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + if (!this.disposed) + { + // We MUST stop the server instance from running before VC exits or it will continue running until explicitly stopped. + this.KillServerInstancesAsync(null, CancellationToken.None) + .GetAwaiter().GetResult(); + this.disposed = true; + } + } + } + + /// + /// Initializes the environment for execution of the AspNetBench workload. + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + // This workload needs three packages: aspnetbenchmarks, dotnetsdk, bombardier + DependencyPath workloadPackage = await this.packageManager.GetPackageAsync(this.PackageName, cancellationToken) + .ConfigureAwait(false); + + if (workloadPackage != null) + { + // the directory we are looking for is at the src/Benchmarks + this.aspnetBenchDirectory = this.Combine(workloadPackage.Path, "src", "Benchmarks"); + } + else + { + throw new DependencyException( + $"The expected workload package '{this.PackageName}' does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } + + DependencyPath dotnetSdkPackage = await this.packageManager.GetPackageAsync(this.DotNetSdkPackageName, cancellationToken) + .ConfigureAwait(false); + + if (dotnetSdkPackage == null) + { + throw new DependencyException( + $"The expected DotNet SDK package does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } + + this.dotnetExePath = this.Combine(dotnetSdkPackage.Path, this.Platform == PlatformID.Unix ? "dotnet" : "dotnet.exe"); + + this.InitializeApiClients(); + } + + /// + /// Initializes API client. + /// + protected void InitializeApiClients() + { + IApiClientManager clientManager = this.Dependencies.GetService(); + bool isSingleVM = !this.IsMultiRoleLayout(); + + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + } + + /// + /// Validates required parameters. + /// + protected override void Validate() + { + base.Validate(); + if (this.BindToCores) + { + this.ThrowIfParameterNotDefined(nameof(this.CoreAffinity)); + } + } + + /// + /// + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + protected override Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return this.Logger.LogMessageAsync($"{this.TypeName}.ExecuteServer", telemetryContext, async () => + { + try + { + this.SetServerOnline(false); + + await this.ServerApi.PollForHeartbeatAsync(TimeSpan.FromMinutes(5), cancellationToken); + + await this.DeleteStateAsync(telemetryContext, cancellationToken); + await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + await this.BuildAspNetBenchAsync(telemetryContext, cancellationToken); + this.StartServerInstances(telemetryContext, cancellationToken); + + await this.SaveStateAsync(telemetryContext, cancellationToken); + this.SetServerOnline(true); + + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) + { + await Task.WhenAny(this.serverProcess); + + // A cancellation is request, then we allow each of the server instances + // to gracefully exit. If a cancellation was not requested, it means that one + // or more of the server instances exited and we will want to allow the component + // to start over restarting the servers. + if (cancellationToken.IsCancellationRequested) + { + await Task.WhenAll(this.serverProcess); + } + } + } + catch + { + this.SetServerOnline(false); + await this.KillServerInstancesAsync(telemetryContext, cancellationToken); + throw; + } + }); + } + + /// + /// Builds the ASP.NET Benchmark application + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + protected async Task BuildAspNetBenchAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + // ~/vc/packages/dotnet/dotnet build -c Release -p:BenchmarksTargetFramework=net8.0 + // Build the aspnetbenchmark project + string buildArgument = $"build -c Release -p:BenchmarksTargetFramework={this.TargetFramework}"; + await this.ExecuteCommandAsync(this.dotnetExePath, buildArgument, this.aspnetBenchDirectory, telemetryContext, cancellationToken) + .ConfigureAwait(false); + + // "C:\Users\vcvmadmin\Benchmarks\src\Benchmarks\bin\Release\net8.0\Benchmarks.dll" + this.aspnetBenchDllPath = this.Combine( + this.aspnetBenchDirectory, + "bin", + "Release", + this.TargetFramework, + "Benchmarks.dll"); + } + + private Task DeleteStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + return this.Logger.LogMessageAsync($"{this.TypeName}.DeleteState", relatedContext, async () => + { + using (HttpResponseMessage response = await this.ServerApi.DeleteStateAsync(nameof(State), cancellationToken)) + { + relatedContext.AddResponseContext(response); + if (response.StatusCode != HttpStatusCode.NoContent) + { + response.ThrowOnError(ErrorReason.HttpNonSuccessResponse); + } + } + }); + } + + private Task KillServerInstancesAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + this.Logger.LogTraceMessage($"{this.TypeName}.KillServerInstances"); + + // Kill ALL existing dotnet processes that might be running the ASP.NET benchmarks + this.ExecuteCommandAsync("pkill", "dotnet", this.aspnetBenchDirectory, telemetryContext, cancellationToken); + + // Free up the Server port + this.ExecuteCommandAsync("fuser", $"-n tcp -k {this.ServerPort}", this.aspnetBenchDirectory, telemetryContext, cancellationToken); + + return this.WaitAsync(TimeSpan.FromSeconds(3), cancellationToken); + } + + private void StartServerInstances(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + + this.Logger.LogMessage($"{this.TypeName}.StartServerInstances", relatedContext, () => + { + try + { + // Example: + // dotnet \Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:5000 + // --server Kestrel --kestrelTransport Sockets --protocol http + // --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" + // --header "Connection: keep-alive" + + string options = $"--nonInteractive true --scenarios json --urls http://*:{this.ServerPort} --server Kestrel --kestrelTransport Sockets --protocol http"; + string headers = @"--header ""Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"" --header ""Connection: keep-alive"""; + string commandArguments = $"{this.aspnetBenchDllPath} {options} {headers}"; + string workingDirectory = this.aspnetBenchDirectory; + + relatedContext.AddContext("command", this.dotnetExePath); + relatedContext.AddContext("commandArguments", commandArguments); + relatedContext.AddContext("workingDir", workingDirectory); + + this.serverProcess = this.StartServerInstanceAsync(this.dotnetExePath, commandArguments, workingDirectory, relatedContext, cancellationToken); + } + catch (OperationCanceledException) + { + // Expected whenever certain operations (e.g. Task.Delay) are cancelled. + } + }); + } + + private Task StartServerInstanceAsync(string command, string commandArguments, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken) + { + return (this.ServerRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => + { + try + { + string effectiveCoreAffinity = this.BindToCores ? this.CoreAffinity : null; + + var (process, affinityConfig) = WorkloadAffinitySupport.CreateProcessWithOptionalAffinity( + this.systemManagement.ProcessManager, + this.Platform, + command, + commandArguments, + workingDirectory, + effectiveCoreAffinity); + + WorkloadAffinitySupport.AddAffinityContext(telemetryContext, this.BindToCores, this.CoreAffinity, affinityConfig); + + this.CleanupTasks.Add(() => process.SafeKill(this.Logger)); + + using (process) + { + await WorkloadAffinitySupport.StartAndWaitWithAffinityAsync( + process, this.Platform, affinityConfig, cancellationToken) + .ConfigureAwait(false); + + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "AspNetBenchmarks"); + process.ThrowIfWorkloadFailed(successCodes: new int[] { 0 }); + } + } + } + catch (OperationCanceledException) + { + // Expected whenever certain operations (e.g. Task.Delay) are cancelled. + } + catch (Exception exc) + { + this.Logger.LogMessage( + $"{this.TypeName}.StartServerInstanceError", + LogLevel.Error, + telemetryContext.Clone().AddError(exc)); + + throw; + } + }); + } + + private Task SaveStateAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + EventContext relatedContext = telemetryContext.Clone(); + return this.Logger.LogMessageAsync($"{this.TypeName}.SaveState", relatedContext, async () => + { + Item serverState = new Item(nameof(State), new State()); + serverState.Definition.Online(true); + using (HttpResponseMessage response = await this.ServerApi.UpdateStateAsync(serverState.Id, serverState, cancellationToken)) + { + relatedContext.AddResponseContext(response); + response.ThrowOnError(ErrorReason.HttpNonSuccessResponse); + } + }); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs new file mode 100644 index 0000000000..0f97cfc961 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Bombardier/BombardierExecutor.cs @@ -0,0 +1,506 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.IO.Abstractions; + using System.Linq; + using System.Net.Http; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + using VirtualClient.Contracts.Metadata; + + /// + /// The Bombardier Client executor. + /// + [SupportedPlatforms("linux-arm64,linux-x64,win-x64,win-arm64")] + public class BombardierExecutor : VirtualClientMultiRoleComponent + { + /// + /// Constructor + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public BombardierExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + // This retry is for the whole execution flow right from waiting for server to be online + this.ClientFlowRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(2, (retries) => TimeSpan.FromSeconds(retries * 2)); + + // This retry is for the individual workload execution + this.ClientRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(2, (retries) => TimeSpan.FromSeconds(retries)); + + this.SystemManagement = this.Dependencies.GetService(); + } + + /// + /// API Client that is used to communicate with server-hosted instance of the Virtual Client Server. + /// + public IApiClient ServerApi { get; set; } + + /// + /// API Client that is used to communicate with ReverseProxy instance of the Virtual Client Server. + /// + public IApiClient ReverseProxyApi { get; set; } + + /// + /// Provides components and services necessary for interacting with the local system and environment. + /// + public ISystemManagement SystemManagement { get; } + + /// + /// Option for testing webserver (default), reverse-proxy (rp), api-gateway (apigw) + /// + public string TargetService + { + get + { + switch (this.Parameters.GetValue(nameof(this.TargetService), string.Empty).ToLower()) + { + case "reverse-proxy": + case "rp": + return "rp"; + case "apiwg": + case "apigw": + case "api-gateway": + return "apigw"; + case "server": + return "server"; + default: + IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); + if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + { + return "server"; + } + else + { + return "rp"; + } + } + } + } + + /// + /// The command line argument defined in the profile. + /// + public string CommandArguments + { + get + { + return this.Parameters.GetValue(nameof(this.CommandArguments)); + } + } + + /// + /// Polling Timeout + /// + public TimeSpan Timeout + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.Timeout), TimeSpan.FromMinutes(5)); + } + } + + /// + /// Parameter defines true/false whether the action is meant to warm up the server. + /// We do not capture metrics on warm up operations. + /// + public bool WarmUp + { + get + { + return this.Parameters.GetValue(nameof(this.WarmUp), false); + } + } + + /// + /// Whether to bind the workload process to specific CPU cores using + /// numactl (Linux) or processor affinity bitmask (Windows). + /// Default: false. + /// + public bool BindToCores + { + get + { + return this.Parameters.GetValue(nameof(this.BindToCores), defaultValue: false); + } + } + + /// + /// The CPU core affinity specification. Supports ranges ("0-3"), + /// comma-separated ("0,2,4,6"), or mixed ("0-3,8-11"). + /// Required when BindToCores is true. + /// + public string CoreAffinity + { + get + { + this.Parameters.TryGetValue(nameof(this.CoreAffinity), out IConvertible value); + return value?.ToString(); + } + } + + /// + /// The path to the Bombardier package. + /// + public string PackageDirectory { get; set; } + + /// + /// True/false whether the server instance has been warmed up. + /// + protected bool IsServerWarmedUp { get; set; } + + /// + /// The retry policy to apply to the client-side execution workflow. + /// + protected IAsyncPolicy ClientFlowRetryPolicy { get; set; } + + /// + /// The retry policy to apply to each Bombardier workload instance when trying to startup + /// against a target server. + /// + protected IAsyncPolicy ClientRetryPolicy { get; set; } + + /// + /// Validates required parameters. + /// + protected override void Validate() + { + base.Validate(); + if (this.BindToCores) + { + this.ThrowIfParameterNotDefined(nameof(this.CoreAffinity)); + } + } + + /// + /// Initializes the executor dependencies, package locations, server api, etc... + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + DependencyPath workloadPackage = await this.SystemManagement.PackageManager.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); + + if (workloadPackage == null) + { + throw new DependencyException( + $"The expected workload package '{this.PackageName}' does not exist on the system or is not registered.", + ErrorReason.WorkloadDependencyMissing); + } + + this.PackageDirectory = workloadPackage.Path; + + // Make bombardier executable on Unix systems + if (this.Platform == PlatformID.Unix) + { + string bombardierPath = this.Combine(this.PackageDirectory, "bombardier"); + await this.SystemManagement.MakeFileExecutableAsync(bombardierPath, this.Platform, cancellationToken).ConfigureAwait(false); + } + + this.InitializeApiClients(); + } + + /// + /// Executes component logic + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + if (!this.WarmUp || !this.IsServerWarmedUp) + { + Task clientWorkloadTask; + + clientWorkloadTask = this.ClientFlowRetryPolicy.ExecuteAsync(async () => + { + if (!cancellationToken.IsCancellationRequested) + { + // Confirm server is online. + this.Logger.LogTraceMessage("Synchronization: Poll server API for heartbeat..."); + await this.ServerApi.PollForHeartbeatAsync(this.PollingTimeout, cancellationToken); + this.Logger.LogTraceMessage("Synchronization: Poll server for online signal..."); + + await this.ServerApi.PollForExpectedStateAsync(nameof(State), (state => state.Online()), this.Timeout, cancellationToken).ConfigureAwait(false); + + this.Logger.LogTraceMessage("Synchronization: Server online signal confirmed..."); + + // verify ReverseProxy is online + if ((this.ReverseProxyApi != null) && ((this.TargetService == "rp") || (this.TargetService == "apigw"))) + { + this.Logger.LogTraceMessage("Synchronization: Poll ReverseProxy for online signal..."); + await this.ReverseProxyApi.PollForExpectedStateAsync(nameof(State), (state => state.Online()), TimeSpan.FromMinutes(10), cancellationToken).ConfigureAwait(false); + this.Logger.LogTraceMessage("Synchronization: ReverseProxy online signal confirmed..."); + Item reverseProxyState = await this.ReverseProxyApi.GetStateAsync(nameof(State), cancellationToken).ConfigureAwait(false); + telemetryContext.AddContext(nameof(reverseProxyState), reverseProxyState); + + HttpResponseMessage reverseProxyHttpResponseMessage = await this.ReverseProxyApi.GetStateAsync("version", cancellationToken).ConfigureAwait(continueOnCapturedContext: false); + telemetryContext.AddResponseContext(reverseProxyHttpResponseMessage); + + if (reverseProxyHttpResponseMessage.IsSuccessStatusCode) + { + Item reverseProxyVersion = await reverseProxyHttpResponseMessage.FromContentAsync>(); + telemetryContext.AddContext(nameof(reverseProxyVersion), reverseProxyVersion.Definition.Properties); + } + } + + this.Logger.LogTraceMessage("Synchronization: Start client workload..."); + + // Execute the client workload. + // =========================================================================== + string commandArguments = this.GetCommandLineArguments(cancellationToken); + await this.ExecuteWorkloadAsync(commandArguments, this.PackageDirectory, telemetryContext, cancellationToken); + } + }); + + await Task.WhenAll(clientWorkloadTask); + + if (this.WarmUp) + { + this.IsServerWarmedUp = true; + } + } + } + + /// + /// Initializes API client. + /// + protected void InitializeApiClients() + { + IApiClientManager clientManager = this.Dependencies.GetService(); + + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ServerApi); + + IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); + if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + { + this.ReverseProxyApi = null; + } + else + { + ClientInstance reverseProxyInstance = reverseProxyInstanceEnumerable.FirstOrDefault(); + this.ReverseProxyApi = clientManager.GetOrCreateApiClient(reverseProxyInstance.Name, reverseProxyInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ReverseProxyApi); + } + } + + /// + /// Gets Command Line Argument to start workload. + /// + protected string GetCommandLineArguments(CancellationToken cancellationToken) + { + string result = this.CommandArguments; + + Dictionary roleAndRegexKvp = new Dictionary() + { + { ClientRole.Server, new Regex(@"\{ServerIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) }, + { ClientRole.ReverseProxy, new Regex(@"\{ReverseProxyIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) }, + { ClientRole.Client, new Regex(@"\{ClientIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) } + }; + + foreach (KeyValuePair kvp in roleAndRegexKvp) + { + MatchCollection matches = kvp.Value.Matches(this.CommandArguments); + + if (matches?.Any() == true) + { + foreach (Match match in matches) + { + ClientInstance roleIP = this.GetLayoutClientInstances(kvp.Key).FirstOrDefault(); + result = Regex.Replace(result, match.Value, roleIP.IPAddress); + } + } + } + + return result; + } + + /// + /// Execute Bombardier Executor + /// + /// Command argument to execute on workload + /// Working Directory + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected async Task ExecuteWorkloadAsync(string commandArguments, string workingDir, EventContext telemetryContext, CancellationToken cancellationToken) + { + commandArguments.ThrowIfNullOrEmpty(nameof(commandArguments)); + + string bombardierPath = this.Platform == PlatformID.Unix + ? this.Combine(this.PackageDirectory, "bombardier") + : this.Combine(this.PackageDirectory, "bombardier.exe"); + + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(bombardierPath); + + EventContext relatedContext = telemetryContext.Clone() + .AddContext(nameof(bombardierPath), bombardierPath) + .AddContext(nameof(commandArguments), commandArguments); + + try + { + await (this.ClientRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => + { + try + { + DateTime startTime = DateTime.UtcNow; + + string effectiveCoreAffinity = this.BindToCores ? this.CoreAffinity : null; + + var (process, affinityConfig) = WorkloadAffinitySupport.CreateProcessWithOptionalAffinity( + this.SystemManagement.ProcessManager, + this.Platform, + bombardierPath, + commandArguments, + workingDir, + effectiveCoreAffinity); + + WorkloadAffinitySupport.AddAffinityContext(relatedContext, this.BindToCores, this.CoreAffinity, affinityConfig); + + using (process) + { + await WorkloadAffinitySupport.StartAndWaitWithAffinityAsync( + process, this.Platform, affinityConfig, cancellationToken) + .ConfigureAwait(false); + + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "Bombardier", logToFile: true); + process.ThrowIfWorkloadFailed(); + + if (process.StandardOutput.Length == 0) + { + throw new WorkloadException($"{this.PackageName} did not write metrics to console.", ErrorReason.CriticalWorkloadFailure); + } + + // We don't capture metrics on warm up operations. + if (!this.WarmUp) + { + this.CaptureMetrics(process, commandArguments, relatedContext, cancellationToken); + } + } + } + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.WorkloadStartError", LogLevel.Warning, telemetryContext.Clone().AddError(exc)); + throw; + } + }); + } + catch (OperationCanceledException) + { + this.Logger.LogMessage($"{this.TypeName}.OperationCanceledException", LogLevel.Warning, telemetryContext.Clone()); + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.ExecuteWorkloadError", LogLevel.Error, telemetryContext.Clone().AddError(exc)); + throw; + } + } + + /// + /// Get Bombardier Version + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// Bombardier Version + protected string GetBombardierVersion(EventContext telemetryContext, CancellationToken cancellationToken) + { + string bombardierPath = this.Platform == PlatformID.Unix + ? this.Combine(this.PackageDirectory, "bombardier") + : this.Combine(this.PackageDirectory, "bombardier.exe"); + + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(bombardierPath); + + string commandArguments = "--version"; + string versionPattern = @"bombardier\s+version\s+(\d+\.\d+\.\d+)"; + Regex versionRegex = new Regex(versionPattern, RegexOptions.Compiled | RegexOptions.IgnoreCase); + string bombardierVersion = null; + + try + { + using (IProcessProxy process = this.ExecuteCommandAsync(bombardierPath, commandArguments, workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).Result) + { + if (!cancellationToken.IsCancellationRequested) + { + this.LogProcessDetailsAsync(process, telemetryContext, "BombardierVersion", logToFile: true).Wait(); + string output = process.StandardOutput.ToString(); + Match match = versionRegex.Match(output); + + if (match.Success) + { + bombardierVersion = match.Groups[1].Value; + telemetryContext.AddContext("BombardierVersion", bombardierVersion); + this.Logger.LogMessage($"{this.TypeName}.BombardierVersionCaptured", LogLevel.Information, telemetryContext); + } + else + { + throw new WorkloadException("Failed to parse bombardier version from output.", ErrorReason.CriticalWorkloadFailure); + } + } + } + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.BombardierVersionCaptureError", LogLevel.Error, telemetryContext.Clone().AddError(exc)); + throw; + } + + return bombardierVersion; + } + + private void CaptureMetrics(IProcessProxy workloadProcess, string commandArguments, EventContext context, CancellationToken cancellationToken) + { + if (!cancellationToken.IsCancellationRequested) + { + if (workloadProcess.ExitCode == 0) + { + EventContext telemetryContext = context.Clone(); + telemetryContext.AddContext(nameof(this.MetricScenario), this.MetricScenario); + telemetryContext.AddContext(nameof(this.Scenario), this.Scenario); + + BombardierMetricsParser resultsParser = new BombardierMetricsParser(workloadProcess.StandardOutput.ToString()); + IList metrics = resultsParser.Parse(); + + string bombardierVersion = this.GetBombardierVersion(telemetryContext, cancellationToken); + + this.MetadataContract.AddForScenario( + toolName: this.PackageName, + toolArguments: commandArguments, + toolVersion: bombardierVersion, + packageName: this.PackageName, + packageVersion: null, + additionalMetadata: null); + this.MetadataContract.Apply(telemetryContext); + + this.Logger.LogMetrics( + toolName: this.PackageName, + scenarioName: this.MetricScenario ?? this.Scenario, + workloadProcess.StartTime, + workloadProcess.ExitTime, + metrics: metrics, + metricCategorization: null, + scenarioArguments: commandArguments, + tags: this.Tags, + eventContext: telemetryContext); + } + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxCommand.cs b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxCommand.cs new file mode 100644 index 0000000000..c09743d927 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxCommand.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + /// + /// Common Nginx Commands + /// Explore more here: https://nginx.org/en/docs/switches.html + /// + public enum NginxCommand + { + /// + /// "service nginx start" + /// + Start, + + /// + /// "service nginx stop" + /// shut down quickly + /// + Stop, + + /// + /// "nginx -T"; + /// -V: print nginx version, compiler version, and configure parameters. + /// -T: Test configuration file. And dump it. + /// Use standard output to read + /// + GetVersion, + + /// + /// "nginx -T"; + /// -T: Test configuration file. And dump it. + /// Use standard output to read + /// + GetConfig + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs new file mode 100644 index 0000000000..5981ce65f4 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxExtensions.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using VirtualClient.Common.Extensions; + using VirtualClient.Contracts; + + /// + /// Extension method for Nginx Workload. + /// + public static class NginxExtensions + { + /// + /// Checks if workload has expired from the state's "Timeout" property + /// + /// + /// + public static bool IsExpired(this State state) + { + return state.Timeout() < DateTime.UtcNow; + } + + /// + /// Gets or sets the 'Timeout' property value from the state. + /// + public static DateTime Timeout(this State state, DateTime? value = null) + { + state.ThrowIfNull("state"); + if (value != null) + { + state.Properties["Timeout"] = value.Value; + } + + return state.Properties.GetValue("Timeout"); + } + + /// + /// Returns nginx commands to start the process. + /// + /// + /// + /// + public static string ConvertToCommandArgs(this NginxCommand command) + { + switch (command) + { + case NginxCommand.Start: + return "systemctl restart nginx"; + + case NginxCommand.Stop: + return "systemctl disable nginx"; + + case NginxCommand.GetVersion: + return "nginx -V"; + + case NginxCommand.GetConfig: + return "nginx -T"; + + default: + throw new WorkloadException($"Unable to convert {nameof(NginxCommand)} enum into string value. Value: {Enum.GetName(command)} - {command}"); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs new file mode 100644 index 0000000000..5c88bc3032 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Nginx/NginxServerExecutor.cs @@ -0,0 +1,371 @@ +// Copyright (c) Microsoft Corporation. +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.IO.Abstractions; + using System.Linq; + using System.Net; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.CodeAnalysis; + using Microsoft.Extensions.DependencyInjection; + using VirtualClient.Common; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.Platform; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// The NGINX Server Executor + /// + [UnixCompatible] + public class NginxServerExecutor : VirtualClientComponent + { + private TimeSpan pollingInterval = TimeSpan.FromSeconds(120); + + /// + /// Constructor + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public NginxServerExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + this.SystemManagement = this.Dependencies.GetService(); + this.pollingInterval = parameters.GetTimeSpanValue(nameof(this.pollingInterval), TimeSpan.FromSeconds(60)); + } + + /// + /// Number workers used. Number cores nginx can use. Set to null or 0 to use all cores. + /// + public int Workers + { + get + { + return this.Parameters.GetValue(nameof(this.Workers), 0); + } + } + + /// + /// Polling Timeout + /// + public TimeSpan Timeout + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.Timeout), TimeSpan.FromMinutes(30)); + } + } + + /// + /// File size to transport between Client and Server + /// + public int FileSizeInKB + { + get + { + return this.Parameters.GetValue(nameof(this.FileSizeInKB), 1); + } + } + + /// + /// The role of current instance + /// + public string Role + { + get + { + return this.Parameters.GetValue(nameof(this.Role)); + } + } + + /// + /// Provides components and services necessary for interacting with the local system and environment. + /// + protected ISystemManagement SystemManagement { get; } + + /// + /// API Client that is used to communicate with self-hosted instance of the Virtual Client. + /// + protected IApiClient ServerApi { get; set; } + + /// + /// API Client that is used to communicate with client-hosted instance of the Virtual Client Client. + /// + protected IApiClient ClientApi { get; set; } + + /// + /// The path to the Nginx package. + /// + protected string PackageDirectory { get; set; } + + /// + /// Initializes the API dependencies for running Nginx Server + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + this.Logger.LogTraceMessage($"Role: {this.Roles}; Layout: {this.GetLayoutClientInstances}"); + + PlatformSpecifics.ThrowIfNotSupported(this.CpuArchitecture); + if (this.Platform != PlatformID.Unix) + { + this.Logger.LogNotSupported(this.PackageName, this.Platform, this.CpuArchitecture, EventContext.Persisted()); + throw new NotSupportedException($"The OS/system platform '{this.Platform}' is not supported."); + } + + IApiClientManager clientManager = this.Dependencies.GetService(); + + ClientInstance serverInstance = this.GetLayoutClientInstances(this.Role).First(); + IPAddress.TryParse(serverInstance.IPAddress, out IPAddress serverIPAddress); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + + ClientInstance clientInstance = this.GetLayoutClientInstances(ClientRole.Client).First(); + IPAddress.TryParse(clientInstance.IPAddress, out IPAddress clientIPAddress); + this.ClientApi = clientManager.GetOrCreateApiClient(clientInstance.Name, clientInstance); + + DependencyPath workloadPackage = await this.SystemManagement.PackageManager.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); + if (workloadPackage == null) + { + throw new DependencyException($"{this.TypeName} did not find package ({this.PackageName}) in the packages directory.", ErrorReason.WorkloadDependencyMissing); + } + + // Step 1: Install Nginx + // Step 2: Install NginxConfiguration Package + // Step 3: Verify files exist + workloadPackage = this.PlatformSpecifics.ToPlatformSpecificPath(workloadPackage, this.Platform, this.CpuArchitecture); + this.PackageDirectory = workloadPackage.Path; + + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.PackageDirectory, "setup-reset.sh")); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.PackageDirectory, "setup-config.sh")); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.PackageDirectory, "setup-content.sh")); + + string resetFilePath = this.PlatformSpecifics.Combine(this.PackageDirectory, "reset.sh"); + + // Step 4: Create reset script to leave server as VC found it. + if (!this.SystemManagement.FileSystem.File.Exists(resetFilePath)) + { + IProcessProxy process1 = await this.ExecuteCommandAsync(command: "bash", commandArguments: "setup-reset.sh", workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true); + await this.LogProcessDetailsAsync(process1, telemetryContext, this.PackageDirectory, logToFile: true); + process1.ThrowIfWorkloadFailed(); + telemetryContext.AddContext("resetContent", process1.StandardOutput); + + using (FileSystemStream fileStream = this.SystemManagement.FileSystem.FileStream.New(resetFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite)) + { + byte[] bytedata = Encoding.Default.GetBytes(process1.StandardOutput.ToString()); + fileStream.Write(bytedata, 0, bytedata.Length); + await fileStream.FlushAsync().ConfigureAwait(false); + fileStream.Close(); + fileStream.Dispose(); + this.Logger.LogTraceMessage($"File Created...{resetFilePath}"); + } + } + + // Step 5: set up nginx config + IProcessProxy process2 = await this.ExecuteCommandAsync(command: "bash", commandArguments: $"setup-content.sh {this.FileSizeInKB}", workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process2, telemetryContext, this.PackageDirectory, logToFile: true); + process2.ThrowIfWorkloadFailed(); + + // Step 6: set up nginx config + ClientInstance backendInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + IPAddress.TryParse(backendInstance.IPAddress, out IPAddress backendIPAddress); + IProcessProxy process3 = await this.ExecuteCommandAsync(command: "bash", commandArguments: $"setup-config.sh {((this.Workers != 0) ? this.Workers : "auto")} {this.Role} {backendIPAddress}", workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process3, telemetryContext, this.PackageDirectory, logToFile: true); + process3.ThrowIfWorkloadFailed(); + + await this.ServerApi.DeleteStateAsync("version", cancellationToken).ConfigureAwait(false); + await this.ServerApi.DeleteStateAsync(nameof(State), cancellationToken).ConfigureAwait(false); + this.SetServerOnline(true); + this.Logger.LogTraceMessage($"{this.TypeName} Initialize Complete."); + } + + /// + /// Executes component logic + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + telemetryContext + .AddContext("currentDirectory", Environment.CurrentDirectory) + .AddContext("toolName", "nginx") + .AddContext("timeout", this.Timeout); + + if (!cancellationToken.IsCancellationRequested) + { + try + { + // step 1: get nginx version + Dictionary nginxVersion = await this.GetNginxVersionAsync(telemetryContext, cancellationToken).ConfigureAwait(false); + State nginxState = new State(); + nginxVersion.Any(x => nginxState.Properties.TryAdd(x.Key, x.Value)); + await this.ServerApi.CreateStateAsync("version", nginxState, cancellationToken).ConfigureAwait(false); + telemetryContext.AddContext(nameof(nginxVersion), nginxVersion); + + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) + { + await this.ExecuteNginxCommandAsync(NginxCommand.Start, workingDirectory: null, telemetryContext, cancellationToken).ConfigureAwait(false); + + Item serverState = new Item(nameof(State), new State()); + serverState.Definition.Timeout(DateTime.UtcNow.Add(this.Timeout)); + serverState.Definition.Online(true); + await this.ServerApi.CreateStateAsync(nameof(State), serverState.Definition, cancellationToken).ConfigureAwait(false); + + while (!(serverState.Definition.Timeout() < DateTime.UtcNow)) + { + EventContext relatedContext = telemetryContext + .Clone() + .AddContext(nameof(serverState), serverState); + + // step 2: Verify client is online. + await this.ClientApi.PollForExpectedStateAsync(nameof(State), (state => state.Online() == true), this.Timeout, cancellationToken, this.pollingInterval).ConfigureAwait(false); + Item clientState = await this.ClientApi.GetStateAsync(nameof(State), cancellationToken).ConfigureAwait(false); + relatedContext.AddContext(nameof(clientState), clientState); + + await Task.Delay(this.pollingInterval, cancellationToken); + DateTime timeout = clientState.Definition.Properties.GetValue("Timeout", serverState.Definition.Timeout()); + + serverState.Definition.Timeout(timeout); + await this.ServerApi.UpdateStateAsync(nameof(State), serverState, cancellationToken).ConfigureAwait(false); + relatedContext.AddContext(nameof(serverState), serverState); + } + } + } + catch (Exception exc) + { + this.Logger.LogErrorMessage(exc, telemetryContext); + throw; + } + finally + { + // Virtual Client is attempting to leave the server as it found it. + await this.ResetNginxAsync(telemetryContext, cancellationToken).ConfigureAwait(false); + } + } + } + + /// + /// Disposes of resources used by the executor including resetting server. + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + // We MUST stop the server instances from running before VC exits + Console.WriteLine("Disposed"); + Task.Run((Func)(async () => + { + await this.ResetNginxAsync(EventContext.None, CancellationToken.None).ConfigureAwait(false); + })).Wait(); + } + } + + /// + /// Reset Server for Nginx + /// + /// Provides context information to include with telemetry events. + /// A token that can be used to cancel the operation. + protected async Task ResetNginxAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + try + { + // step 1: Reset config created by virtual client + IProcessProxy process1 = await this.ExecuteCommandAsync(command: "bash", commandArguments: "reset.sh", workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process1, telemetryContext, this.PackageName, logToFile: true); + + // step 2: stop nginx server + await this.ExecuteNginxCommandAsync(NginxCommand.Stop, workingDirectory: null, telemetryContext, cancellationToken).ConfigureAwait(false); + + // step 3: recreate server state + State serverState = new State(); + serverState.Online(false); + serverState.Properties["ResetTime"] = DateTime.UtcNow.ToString(); + await this.ServerApi.UpdateStateAsync(nameof(State), new Item(nameof(State), serverState), cancellationToken).ConfigureAwait(false); + } + catch + { + this.Logger.LogTraceMessage("Failed to reset server."); + // Best effort + } + } + + /// + /// Executes a command to get Nginx Version installed from the system. + /// + /// Provides context information to include with telemetry events. + /// A token that can be used to cancel the process execution. + /// NginxVersion + protected async Task> GetNginxVersionAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + // Nginx consistently uses stderr for all output (including -V) + IProcessProxy process = await this.ExecuteNginxCommandAsync(NginxCommand.GetVersion, null, telemetryContext, cancellationToken).ConfigureAwait(false); + string standardErr = process.StandardError.ToString(); + + return this.TransformNginxVersionToDictionary(standardErr); + } + + private Dictionary TransformNginxVersionToDictionary(string processOutput) + { + processOutput.ThrowIfNullOrEmpty(nameof(processOutput)); + string[] standardOutputSections = processOutput.Split(Environment.NewLine, StringSplitOptions.TrimEntries); + + string nginxVersion = + standardOutputSections + .Where(x => x.Contains("nginx version", StringComparison.OrdinalIgnoreCase)).FirstOrDefault() + .Split(":").Last().Trim(); + + nginxVersion.ThrowIfNullOrEmpty(nameof(nginxVersion), $"{nameof(processOutput)} does not contain nginx version. {nameof(processOutput)}: {processOutput}"); + + string sslVersion = + standardOutputSections + .Where(x => x.Contains("OpenSSL", StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + + string serverNameIndicationSupport = + standardOutputSections + .Where(x => x.Contains("TLS", StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + + string arguments = + standardOutputSections + .Where(x => x.Contains("configure arguments", StringComparison.OrdinalIgnoreCase)).FirstOrDefault() + ?.Split(":").Last().Trim(); + + nginxVersion.ThrowIfNullOrEmpty(nameof(nginxVersion), $"{nameof(processOutput)} does not contain nginx version. {nameof(processOutput)}: {processOutput}"); + return new Dictionary + { + { nameof(nginxVersion), nginxVersion }, + { nameof(sslVersion), sslVersion }, + { nameof(serverNameIndicationSupport), serverNameIndicationSupport }, + { nameof(arguments), arguments } + }; + } + + private Task ExecuteNginxCommandAsync(NginxCommand nginxCommand, string workingDirectory, EventContext telemetryContext, CancellationToken cancellationToken) + { + nginxCommand.ThrowIfNull("nginxCommand"); + telemetryContext.ThrowIfNull("telemetryContext"); + + telemetryContext.AddContext(nameof(nginxCommand), nginxCommand); + string commandArgs = nginxCommand.ConvertToCommandArgs(); + telemetryContext.AddContext(nameof(commandArgs), $"{commandArgs}"); + + if (this.Platform != PlatformID.Unix) + { + throw new NotSupportedException($"Nginx command is not supported on '{this.Platform}' platform/architecture systems."); + } + + return this.Logger.LogMessageAsync($"{nameof(this.TypeName)}.ExecuteNginxCommand", telemetryContext, async () => + { + IProcessProxy process = await this.ExecuteCommandAsync(command: "sudo", commandArguments: commandArgs, workingDirectory: workingDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process, telemetryContext, logToFile: true); + process.ThrowIfWorkloadFailed(); + return process; + }); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/WorkloadAffinitySupport.cs b/src/VirtualClient/VirtualClient.Actions/WorkloadAffinitySupport.cs new file mode 100644 index 0000000000..78cd394340 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/WorkloadAffinitySupport.cs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Common; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// Shared helper for creating and running processes with optional CPU core affinity binding. + /// Follows the pattern established by RedisServerExecutor. + /// + internal static class WorkloadAffinitySupport + { + /// + /// Creates a process with optional CPU affinity binding and elevated privileges. + /// On Linux, wraps the command with numactl via CreateElevatedProcessWithAffinity. + /// On Windows, creates a normal elevated process (affinity applied post-start). + /// + /// The process manager used to create the process. + /// The OS platform. + /// The command to run. + /// The command line arguments. + /// The working directory. + /// The core specification (e.g., "0-3", "0,2,4,6"). Null for no affinity. + /// A tuple of (process, affinityConfig). affinityConfig is null when no binding requested. + public static (IProcessProxy Process, ProcessAffinityConfiguration AffinityConfig) CreateProcessWithOptionalAffinity( + ProcessManager processManager, + PlatformID platform, + string command, + string arguments, + string workingDirectory, + string coreAffinity) + { + if (string.IsNullOrWhiteSpace(coreAffinity)) + { + // No affinity — standard elevated process + IProcessProxy process = processManager.CreateElevatedProcess(platform, command, arguments, workingDirectory); + return (process, null); + } + + ProcessAffinityConfiguration affinityConfig = ProcessAffinityConfiguration.Create(platform, coreAffinity); + + if (platform == PlatformID.Unix) + { + // Linux: numactl wrapping handled by CreateElevatedProcessWithAffinity + IProcessProxy process = processManager.CreateElevatedProcessWithAffinity( + platform, command, arguments, workingDirectory, affinityConfig); + return (process, affinityConfig); + } + else + { + // Windows: create elevated, affinity applied post-start via ApplyAffinity + IProcessProxy process = processManager.CreateElevatedProcess(platform, command, arguments, workingDirectory); + return (process, affinityConfig); + } + } + + /// + /// Starts a process and applies Windows CPU affinity if applicable. + /// On Linux, the process already has numactl wrapping so just starts normally. + /// On Windows, starts the process then applies the affinity bitmask. + /// + /// The process to start. + /// The OS platform. + /// The affinity config, or null if no binding. + /// A token to cancel the operation. + public static async Task StartAndWaitWithAffinityAsync( + IProcessProxy process, + PlatformID platform, + ProcessAffinityConfiguration affinityConfig, + CancellationToken cancellationToken) + { + if (affinityConfig != null && platform == PlatformID.Win32NT) + { + // Windows: start first, then apply affinity bitmask + process.Start(); + process.ApplyAffinity((WindowsProcessAffinityConfiguration)affinityConfig); + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + } + else + { + // Linux (with or without numactl) or no affinity + await process.StartAndWaitAsync(cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Adds affinity-related context to telemetry for diagnostics. + /// + public static void AddAffinityContext(EventContext telemetryContext, bool bindToCores, string coreAffinity, ProcessAffinityConfiguration affinityConfig) + { + telemetryContext.AddContext("bindToCores", bindToCores); + if (bindToCores) + { + telemetryContext.AddContext("coreAffinity", coreAffinity); + if (affinityConfig != null) + { + telemetryContext.AddContext("affinityDetails", affinityConfig.ToString()); + } + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Wrk/Wrk2Executor.cs b/src/VirtualClient/VirtualClient.Actions/Wrk/Wrk2Executor.cs new file mode 100644 index 0000000000..86b93096b5 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Wrk/Wrk2Executor.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using VirtualClient.Common; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// The Wrk Client executor. + /// + [SupportedPlatforms("linux-arm64,linux-x64")] + public class Wrk2Executor : WrkExecutor + { + /// + /// Constructor + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public Wrk2Executor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + } + + /// + /// Initializes the executor dependencies, package locations, server api, etc... + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + DependencyPath workloadPackage = await this.SystemManagement.PackageManager.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); + DependencyPath scriptPackage = await this.SystemManagement.PackageManager.GetPackageAsync(Wrk2Executor.WrkConfiguration, cancellationToken).ConfigureAwait(false); + + if (workloadPackage == null || this.PackageName != "wrk2") + { + throw new DependencyException($"{this.TypeName} did not find correct package in the directory. Supported Package: wrk2. Package Provided: {this.PackageName}", ErrorReason.WorkloadDependencyMissing); + } + + if (scriptPackage == null) + { + throw new DependencyException($"{this.TypeName} did not find package ({WrkExecutor.WrkConfiguration}) in the packages directory.", ErrorReason.WorkloadDependencyMissing); + } + + this.PackageDirectory = workloadPackage.Path; + this.ScriptDirectory = this.PlatformSpecifics.ToPlatformSpecificPath(scriptPackage, this.Platform, this.CpuArchitecture).Path; + + this.InitializeApiClients(); + await this.SetupWrkClient(telemetryContext, cancellationToken).ConfigureAwait(false); + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs new file mode 100644 index 0000000000..6668606244 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Wrk/WrkExecutor.cs @@ -0,0 +1,563 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.IO.Abstractions; + using System.Linq; + using System.Net.Http; + using System.Text; + using System.Text.RegularExpressions; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.CodeAnalysis; + using Microsoft.Extensions.DependencyInjection; + using Microsoft.Extensions.Logging; + using Polly; + using VirtualClient.Common; + using VirtualClient.Common.ProcessAffinity; + using VirtualClient.Common.Contracts; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + using VirtualClient.Contracts.Metadata; + + /// + /// The Wrk Client executor. + /// + [SupportedPlatforms("linux-arm64,linux-x64")] + public class WrkExecutor : VirtualClientMultiRoleComponent + { + internal const string WrkConfiguration = "wrkconfiguration"; + internal const string WrkRunShell = "runwrk.sh"; + + /// + /// Constructor + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public WrkExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + // This retry is for the whole execution flow right from waiting for server to be online + this.ClientFlowRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(2, (retries) => TimeSpan.FromSeconds(retries * 2)); + + // This retry is for the individual workload execution + this.ClientRetryPolicy = Policy.Handle(exc => !(exc is OperationCanceledException)) + .WaitAndRetryAsync(2, (retries) => TimeSpan.FromSeconds(retries)); + + this.SystemManagement = this.Dependencies.GetService(); + } + + /// + /// API Client that is used to communicate with server-hosted instance of the Virtual Client Server. + /// + public IApiClient ServerApi { get; set; } + + /// + /// API Client that is used to communicate with ReverseProxy instance of the Virtual Client Server. + /// + public IApiClient ReverseProxyApi { get; set; } + + /// + /// Provides components and services necessary for interacting with the local system and environment. + /// + public ISystemManagement SystemManagement { get; } + + /// + /// Option for testing webserver (default), reverse-proxy (rp), api-gateway (apigw) + /// + public string TargetService + { + get + { + switch (this.Parameters.GetValue(nameof(this.TargetService), string.Empty).ToLower()) + { + case "reverse-proxy": + case "rp": + return "rp"; + case "apiwg": + case "apigw": + case "api-gateway": + return "apigw"; + case "server": + return "server"; + default: + IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); + if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + { + return "server"; + } + else + { + return "rp"; + } + } + } + } + + /// + /// The command line argument defined in the profile. + /// + public string CommandArguments + { + get + { + return this.Parameters.GetValue(nameof(this.CommandArguments)); + } + } + + /// + /// Emit Latency Spectrum as additional metrics + /// + public bool EmitLatencySpectrum + { + get + { + return this.Parameters.GetValue(nameof(this.EmitLatencySpectrum), false); + } + } + + /// + /// Polling Timeout + /// + public TimeSpan Timeout + { + get + { + return this.Parameters.GetTimeSpanValue(nameof(this.Timeout), TimeSpan.FromMinutes(5)); + } + } + + /// + /// Parameter defines true/false whether the action is meant to warm up the server. + /// We do not capture metrics on warm up operations. + /// + public bool WarmUp + { + get + { + return this.Parameters.GetValue(nameof(this.WarmUp), false); + } + } + + /// + /// Whether to bind the workload process to specific CPU cores using + /// numactl (Linux) or processor affinity bitmask (Windows). + /// Default: false. + /// + public bool BindToCores + { + get + { + return this.Parameters.GetValue(nameof(this.BindToCores), defaultValue: false); + } + } + + /// + /// The CPU core affinity specification. Supports ranges ("0-3"), + /// comma-separated ("0,2,4,6"), or mixed ("0-3,8-11"). + /// Required when BindToCores is true. + /// + public string CoreAffinity + { + get + { + this.Parameters.TryGetValue(nameof(this.CoreAffinity), out IConvertible value); + return value?.ToString(); + } + } + + /// + /// The path to the Wrk package. + /// + public string PackageDirectory { get; set; } + + /// + /// The path to wrk scripts. + /// + public string ScriptDirectory { get; set; } + + /// + /// True/false whether the server instance has been warmed up. + /// + protected bool IsServerWarmedUp { get; set; } + + /// + /// The retry policy to apply to the client-side execution workflow. + /// + protected IAsyncPolicy ClientFlowRetryPolicy { get; set; } + + /// + /// The retry policy to apply to each Memtier workload instance when trying to startup + /// against a target server. + /// + protected IAsyncPolicy ClientRetryPolicy { get; set; } + + /// + /// Initializes the executor dependencies, package locations, server api, etc... + /// + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + DependencyPath workloadPackage = await this.SystemManagement.PackageManager.GetPackageAsync(this.PackageName, cancellationToken).ConfigureAwait(false); + DependencyPath scriptPackage = await this.SystemManagement.PackageManager.GetPackageAsync(WrkExecutor.WrkConfiguration, cancellationToken).ConfigureAwait(false); + + if (this.PackageName != "wrk" || workloadPackage == null) + { + throw new DependencyException($"{this.TypeName} did not find correct package in the directory. Supported Package: wrk. Package Provided: {this.PackageName}", ErrorReason.WorkloadDependencyMissing); + } + + if (scriptPackage == null) + { + throw new DependencyException($"{this.TypeName} did not find package ({WrkExecutor.WrkConfiguration}) in the packages directory.", ErrorReason.WorkloadDependencyMissing); + } + + this.PackageDirectory = workloadPackage.Path; + this.ScriptDirectory = this.PlatformSpecifics.ToPlatformSpecificPath(scriptPackage, this.Platform, this.CpuArchitecture).Path; + + this.InitializeApiClients(); + await this.SetupWrkClient(telemetryContext, cancellationToken).ConfigureAwait(false); + } + + /// + /// Validates required parameters. + /// + protected override void Validate() + { + base.Validate(); + if (this.BindToCores) + { + this.ThrowIfParameterNotDefined(nameof(this.CoreAffinity)); + } + } + + /// + /// Executes component logic + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + if (!this.WarmUp || !this.IsServerWarmedUp) + { + Task clientWorkloadTask; + + clientWorkloadTask = this.ClientFlowRetryPolicy.ExecuteAsync(async () => + { + if (!cancellationToken.IsCancellationRequested) + { + // Confirm server is online. + // =========================================================================== + this.Logger.LogTraceMessage("Synchronization: Poll server API for heartbeat..."); + await this.ServerApi.PollForHeartbeatAsync(this.PollingTimeout, cancellationToken); + this.Logger.LogTraceMessage("Synchronization: Poll server for online signal..."); + + await this.ServerApi.PollForExpectedStateAsync(nameof(State), (state => state.Online()), this.Timeout, cancellationToken).ConfigureAwait(false); + + this.Logger.LogTraceMessage("Synchronization: Server online signal confirmed..."); + + // verify ReverseProxy is online + if ((this.ReverseProxyApi != null) && ((this.TargetService == "rp") || (this.TargetService == "apigw"))) + { + this.Logger.LogTraceMessage("Synchronization: Poll ReverseProxy for online signal..."); + await this.ReverseProxyApi.PollForExpectedStateAsync(nameof(State), (state => state.Online()), TimeSpan.FromMinutes(10), cancellationToken).ConfigureAwait(false); + this.Logger.LogTraceMessage("Synchronization: ReverseProxy online signal confirmed..."); + Item reverseProxyState = await this.ReverseProxyApi.GetStateAsync(nameof(State), cancellationToken).ConfigureAwait(false); + telemetryContext.AddContext(nameof(reverseProxyState), reverseProxyState); + + HttpResponseMessage reverseProxyHttpResponseMessage = await this.ReverseProxyApi.GetStateAsync("version", cancellationToken).ConfigureAwait(continueOnCapturedContext: false); + telemetryContext.AddResponseContext(reverseProxyHttpResponseMessage); + + if (reverseProxyHttpResponseMessage.IsSuccessStatusCode) + { + Item reverseProxyVersion = await reverseProxyHttpResponseMessage.FromContentAsync>(); + telemetryContext.AddContext(nameof(reverseProxyVersion), reverseProxyVersion.Definition.Properties); + } + } + + this.Logger.LogTraceMessage("Synchronization: Start client workload..."); + + // 4) Execute the client workload. + // =========================================================================== + string commandArguments = this.GetCommandLineArguments(cancellationToken); + await this.ExecuteWorkloadAsync(commandArguments, this.PackageDirectory, telemetryContext, cancellationToken); + } + }); + + await Task.WhenAll(clientWorkloadTask); + + if (this.WarmUp) + { + this.IsServerWarmedUp = true; + } + } + } + + /// + /// Setup Wrk Executor + /// + protected async Task SetupWrkClient(EventContext telemetryContext, CancellationToken cancellationToken) + { + this.PackageDirectory.ThrowIfNullOrWhiteSpace(nameof(this.PackageDirectory)); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.PackageDirectory, "wrk")); + await this.SystemManagement.MakeFileExecutableAsync(this.PlatformSpecifics.Combine(this.PackageDirectory, "wrk"), this.Platform, cancellationToken).ConfigureAwait(false); + + string shellScriptPath = this.PlatformSpecifics.GetScriptPath("wrk", WrkExecutor.WrkRunShell); + this.SystemManagement.FileSystem.File.Copy(shellScriptPath, this.Combine(this.PackageDirectory, WrkExecutor.WrkRunShell), true); + + this.ScriptDirectory.ThrowIfNullOrWhiteSpace(nameof(this.ScriptDirectory)); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.ScriptDirectory, "setup-reset.sh")); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(this.PlatformSpecifics.Combine(this.ScriptDirectory, "setup-config.sh")); + + string resetFilePath = this.PlatformSpecifics.Combine(this.ScriptDirectory, "reset.sh"); + if (!this.SystemManagement.FileSystem.File.Exists(resetFilePath)) + { + IProcessProxy process1 = await this.ExecuteCommandAsync(command: "bash", commandArguments: "setup-reset.sh", workingDirectory: this.ScriptDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process1, telemetryContext, WrkExecutor.WrkConfiguration, logToFile: true); + process1.ThrowIfWorkloadFailed(); + + using (FileSystemStream fileStream = this.SystemManagement.FileSystem.FileStream.New(resetFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite)) + { + byte[] bytedata = Encoding.Default.GetBytes(process1.StandardOutput.ToString()); + fileStream.Write(bytedata, 0, bytedata.Length); + await fileStream.FlushAsync().ConfigureAwait(false); + this.Logger.LogTraceMessage($"File Created...{resetFilePath}"); + } + } + + // set up Config + IProcessProxy process2 = await this.ExecuteCommandAsync(command: "bash", commandArguments: "setup-config.sh", workingDirectory: this.ScriptDirectory, telemetryContext, cancellationToken, runElevated: true).ConfigureAwait(false); + await this.LogProcessDetailsAsync(process2, telemetryContext, WrkExecutor.WrkConfiguration, logToFile: true); + process2.ThrowIfWorkloadFailed(); + } + + /// + /// Initializes API client. + /// + protected void InitializeApiClients() + { + IApiClientManager clientManager = this.Dependencies.GetService(); + + ClientInstance serverInstance = this.GetLayoutClientInstances(ClientRole.Server).First(); + this.ServerApi = clientManager.GetOrCreateApiClient(serverInstance.Name, serverInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ServerApi); + + IEnumerable reverseProxyInstanceEnumerable = this.GetLayoutClientInstances(ClientRole.ReverseProxy, false); + if ((reverseProxyInstanceEnumerable == null) || (!reverseProxyInstanceEnumerable.Any())) + { + this.ReverseProxyApi = null; + } + else + { + ClientInstance reverseProxyInstance = reverseProxyInstanceEnumerable.FirstOrDefault(); + this.ReverseProxyApi = clientManager.GetOrCreateApiClient(reverseProxyInstance.Name, reverseProxyInstance); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", this.ReverseProxyApi); + } + } + + /// + /// Gets Command Line Argument to start workload. + /// + protected string GetCommandLineArguments(CancellationToken cancellationToken) + { + string result = this.CommandArguments; + + Dictionary roleAndRegexKvp = new Dictionary() + { + { ClientRole.Server, new Regex(@"\{ServerIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) }, + { ClientRole.ReverseProxy, new Regex(@"\{ReverseProxyIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) }, + { ClientRole.Client, new Regex(@"\{ClientIp\}", RegexOptions.Compiled | RegexOptions.IgnoreCase) } + }; + + foreach (KeyValuePair kvp in roleAndRegexKvp) + { + MatchCollection matches = kvp.Value.Matches(this.CommandArguments); + + if (matches?.Any() == true) + { + foreach (Match match in matches) + { + ClientInstance roleIP = this.GetLayoutClientInstances(kvp.Key).FirstOrDefault(); + result = Regex.Replace(result, match.Value, roleIP.IPAddress); + } + } + } + + return result; + } + + /// + /// Execute Wrk Executor + /// + /// Command argument ti execute on workload + /// Working Directory + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected async Task ExecuteWorkloadAsync(string commandArguments, string workingDir, EventContext telemetryContext, CancellationToken cancellationToken) + { + commandArguments.ThrowIfNullOrEmpty(nameof(commandArguments)); + string scriptPath = this.Combine(this.PackageDirectory, WrkExecutor.WrkRunShell); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(scriptPath); + // ToDo: Replace bash script with processManager.createProcess("bash", "ulimit -n 65535 && ./wrk {arg}") + string command = $"bash {scriptPath}"; + + EventContext relatedContext = telemetryContext.Clone() + .AddContext(nameof(command), command) + .AddContext(nameof(commandArguments), commandArguments); + + try + { + await (this.ClientRetryPolicy ?? Policy.NoOpAsync()).ExecuteAsync(async () => + { + try + { + DateTime startTime = DateTime.UtcNow; + + string effectiveCoreAffinity = this.BindToCores ? this.CoreAffinity : null; + + var (process, affinityConfig) = WorkloadAffinitySupport.CreateProcessWithOptionalAffinity( + this.SystemManagement.ProcessManager, + this.Platform, + command, + $"\"{commandArguments}\"", + workingDir, + effectiveCoreAffinity); + + WorkloadAffinitySupport.AddAffinityContext(relatedContext, this.BindToCores, this.CoreAffinity, affinityConfig); + + using (process) + { + await WorkloadAffinitySupport.StartAndWaitWithAffinityAsync( + process, this.Platform, affinityConfig, cancellationToken) + .ConfigureAwait(false); + + if (!cancellationToken.IsCancellationRequested) + { + await this.LogProcessDetailsAsync(process, telemetryContext, "Wrk", logToFile: true); + process.ThrowIfWorkloadFailed(); + + if (process.StandardOutput.Length == 0) + { + throw new WorkloadException($"{this.PackageName} did not write metrics to console.", ErrorReason.CriticalWorkloadFailure); + } + + // We don't capture metrics on warm up operations. + if (!this.WarmUp) + { + this.CaptureMetrics(process, commandArguments, this.EmitLatencySpectrum, relatedContext, cancellationToken); + } + } + } + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.WorkloadStartError", LogLevel.Warning, telemetryContext.Clone().AddError(exc)); + throw; + } + }); + } + catch (OperationCanceledException) + { + this.Logger.LogMessage($"{this.TypeName}.OperationCanceledException", LogLevel.Warning, telemetryContext.Clone()); + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.ExecuteWorkloadError", LogLevel.Error, telemetryContext.Clone().AddError(exc)); + throw; + } + } + + /// + /// Get Wrk Version + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + /// Wrk Version + protected string GetWrkVersion(EventContext telemetryContext, CancellationToken cancellationToken) + { + string scriptPath = this.Combine(this.PackageDirectory, WrkExecutor.WrkRunShell); + this.SystemManagement.FileSystem.File.ThrowIfFileDoesNotExist(scriptPath); + + string command = $"bash {scriptPath}"; + string commandArguments = "--version"; + string versionPattern = @"wrk\s(\d+\.\d+\.\d+)"; + Regex versionRegex = new Regex(versionPattern, RegexOptions.Compiled); + string wrkVersion = null; + + try + { + using (IProcessProxy process = this.ExecuteCommandAsync(command, commandArguments, workingDirectory: this.PackageDirectory, telemetryContext, cancellationToken, runElevated: true).Result) + { + if (!cancellationToken.IsCancellationRequested) + { + this.LogProcessDetailsAsync(process, telemetryContext, "WrkVersion", logToFile: true).Wait(); + string output = process.StandardOutput.ToString(); + Match match = versionRegex.Match(output); + + if (match.Success) + { + wrkVersion = match.Groups[1].Value; + telemetryContext.AddContext("WrkVersion", wrkVersion); + this.Logger.LogMessage($"{this.TypeName}.WrkVersionCaptured", LogLevel.Information, telemetryContext); + } + else + { + throw new WorkloadException("Failed to parse wrk version from output.", ErrorReason.CriticalWorkloadFailure); + } + } + } + } + catch (Exception exc) + { + this.Logger.LogMessage($"{this.TypeName}.WrkVersionCaptureError", LogLevel.Error, telemetryContext.Clone().AddError(exc)); + throw; + } + + return wrkVersion; + } + + private void CaptureMetrics(IProcessProxy workloadProcess, string commandArguments, bool emitLatencySpectrum, EventContext context, CancellationToken cancellationToken) + { + if (!cancellationToken.IsCancellationRequested) + { + if (workloadProcess.ExitCode == 0) + { + EventContext telemetryContext = context.Clone(); + telemetryContext.AddContext(nameof(this.MetricScenario), this.MetricScenario); + telemetryContext.AddContext(nameof(this.Scenario), this.Scenario); + + WrkMetricParser resultsParser = new WrkMetricParser(workloadProcess.StandardOutput.ToString()); + IList metrics = resultsParser.Parse(emitLatencySpectrum); + + string wrkVersion = this.GetWrkVersion(telemetryContext, cancellationToken); + + this.MetadataContract.AddForScenario( + toolName: this.PackageName, + toolArguments: commandArguments, + toolVersion: wrkVersion, + packageName: this.PackageName, + packageVersion: null, + additionalMetadata: null); + telemetryContext.AddContext("TestConfig", resultsParser.GetTestConfig()); + this.MetadataContract.Apply(telemetryContext); + + this.Logger.LogMetrics( + toolName: this.PackageName, + scenarioName: this.MetricScenario ?? this.Scenario, + workloadProcess.StartTime, + workloadProcess.ExitTime, + metrics: metrics, + metricCategorization: null, + scenarioArguments: commandArguments, + tags: this.Tags, + eventContext: telemetryContext); + } + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/Wrk/runwrk.sh b/src/VirtualClient/VirtualClient.Actions/Wrk/runwrk.sh new file mode 100644 index 0000000000..01f4e14828 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/Wrk/runwrk.sh @@ -0,0 +1,2 @@ +ulimit -n 65535 +./wrk $1 diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-ORCHARD-WRK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-ORCHARD-WRK.json new file mode 100644 index 0000000000..84d24b06dc --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-ORCHARD-WRK.json @@ -0,0 +1,161 @@ +{ + "Description": ".NET benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedOperatingSystems": "CBL-Mariner,Ubuntu" + }, + "Parameters": { + "DotNetVersion": "9.0.203", + "TargetFramework": "net9.0", + "OrchardCoreAdminValue": "Compete@CRC1", + "ServerPort": 5014, + "TestDuration": "00:00:20", + "EmitLatencySpectrum": false, + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s http://{serverip}:{ServerPort}/about --header \"Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\" --header \"Connection:keep-alive\"" + }, + "Actions": [ + { + "Type": "AspNetOrchardServerExecutor", + "Parameters": { + "Role": "Server", + "Scenario": "OrchardCore", + "PackageName": "orchardcore", + "DotNetSdkPackageName": "dotnetsdk", + "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "$.Parameters.ServerPort" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Warmup", + "MetricScenario": "Warmup_Orchard-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "ServerPort": "$.Parameters.ServerPort", + "ThreadCount": "64", + "Connection": 128, + "TestDuration": "00:02:00", + "WarmUp": "true", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Tags": "Orchard,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Benchmark_Measurement", + "MetricScenario": "Orchard-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "ServerPort": "$.Parameters.ServerPort", + "ThreadCount": "64", + "Connection": 128, + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Tags": "Orchard,WRK" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "build-essential,libevent-dev,pkg-config,zlib1g-dev,libssl-dev,autoconf,automake,make,libpcre3-dev,gcc,unzip,openssl", + "Packages-Yum": "zlib-devel,pcre-devel,libevent-devel,openssl-devel,git,gcc-c++,make,autoconf,automake", + "Packages-Dnf": "zlib-devel,libevent-devel,openssl-devel,git,gcc,gcc-c++,make,autoconf,icu,automake,unzip,binutils,libstdc++-devel,kernel-headers,glibc-devel,perl,perl-Module-CoreList" + } + }, + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "wget,build-essential,tcl-dev,numactl", + "Packages-Yum": "wget,numactl,tcl-devel", + "Packages-Dnf": "wget,numactl,tcl-devel,iptables" + } + }, + { + "Type": "ChocolateyInstallation", + "Parameters": { + "Scenario": "InstallChocolatey", + "PackageName": "chocolatey" + } + }, + { + "Type": "ChocolateyPackageInstallation", + "Parameters": { + "Scenario": "InstallGit", + "PackageName": "chocolatey", + "Packages": "git" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneOrchardCoreRepo", + "RepoUri": "https://github.com/orchardcms/orchardcore.git", + "Commit": "4e7c47c", + "PackageName": "orchardcore", + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "DotNetInstallation", + "Parameters": { + "Scenario": "InstallDotNetSdk", + "DotNetVersion": "$.Parameters.DotNetVersion", + "PackageName": "dotnetsdk", + "Role": "Server" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + }, + { + "Type": "SetEnvironmentVariable", + "Parameters": { + "Scenario": "SetEnvironmentVariableForAspNet", + "EnvironmentVariables": "OrchardCore__OrchardCore_AutoSetup__Tenants__0__ShellName=Default;OrchardCore__OrchardCore_AutoSetup__Tenants__0__SiteName=Benchmark;OrchardCore__OrchardCore_AutoSetup__Tenants__0__SiteTimeZone=Europe/Amsterdam;OrchardCore__OrchardCore_AutoSetup__Tenants__0__AdminUsername=admin;OrchardCore__OrchardCore_AutoSetup__Tenants__0__AdminEmail=info@orchardproject.net;OrchardCore__OrchardCore_AutoSetup__Tenants__0__AdminPassword={OrchardCoreAdminValue};OrchardCore__OrchardCore_AutoSetup__Tenants__0__DatabaseProvider=Sqlite;OrchardCore__OrchardCore_AutoSetup__Tenants__0__RecipeName=Blog;DOTNET_GCDynamicAdaptationMode=0;DOTNET_HillClimbing_Disable=1", + "OrchardCoreAdminValue": "$.Parameters.OrchardCoreAdminValue", + "Role": "Server" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-TEJSON-WRK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-TEJSON-WRK.json new file mode 100644 index 0000000000..ea2bad74f1 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-TEJSON-WRK.json @@ -0,0 +1,150 @@ +{ + "Description": ".NET benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:10:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" + }, + "Parameters": { + "DotNetVersion": "9.0.101", + "TargetFramework": "net9.0", + "ServerPort": 9876, + "TestDuration": "00:00:15", + "Timeout": "00:10:00", + "EmitLatencySpectrum": false, + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s http://{serverip}:{ServerPort}/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\"" + }, + "Actions": [ + { + "Type": "AspNetServerExecutor", + "Parameters": { + "Role": "Server", + "Scenario": "ExecuteJsonSerializationBenchmark", + "PackageName": "aspnetbenchmarks", + "DotNetSdkPackageName": "dotnetsdk", + "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "$.Parameters.ServerPort" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Warmup", + "MetricScenario": "Warmup_ASP.NET-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "ServerPort": "$.Parameters.ServerPort", + "ThreadCount": "64", + "Connection": 4096, + "TestDuration": "00:00:45", + "WarmUp": "true", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "ASP.NET,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Benchmark Measurement", + "MetricScenario": "ASP.NET-WRK_t{ThreadCount}_c{Connection}_d{TestDuration.TotalSeconds}s", + "CommandArguments": "$.Parameters.CommandArguments", + "ServerPort": "$.Parameters.ServerPort", + "ThreadCount": "64", + "Connection": 4096, + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "ASP.NET,WRK" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip", + "Packages-Yum": "zlib-devel,pcre-devel,libevent-devel,openssl-devel,git,gcc-c++,make,autoconf,automake", + "Packages-Dnf": "git,make,gcc,zlib-devel,openssl-devel,icu,glibc-devel,kernel-headers,binutils,perl,unzip" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "DisableFirewall", + "Command": "sudo iptables -P INPUT ACCEPT" + } + }, + { + "Type": "ChocolateyInstallation", + "Parameters": { + "Scenario": "InstallChocolatey", + "PackageName": "chocolatey" + } + }, + { + "Type": "ChocolateyPackageInstallation", + "Parameters": { + "Scenario": "InstallGit", + "PackageName": "chocolatey", + "Packages": "git" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneAspNetBenchmarksRepo", + "RepoUri": "https://github.com/aspnet/Benchmarks.git", + "PackageName": "aspnetbenchmarks", + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "DotNetInstallation", + "Parameters": { + "Scenario": "InstallDotNetSdk", + "DotNetVersion": "$.Parameters.DotNetVersion", + "PackageName": "dotnetsdk" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-AFFINITY.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-AFFINITY.json new file mode 100644 index 0000000000..38569443f3 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-AFFINITY.json @@ -0,0 +1,94 @@ +{ + "Description": "ASP.NET JSON Serialization Benchmark with CPU Core Affinity Pinning", + "MinimumExecutionInterval": "00:01:00", + "Metadata": { + "RecommendedMinimumExecutionTime": "00:05:00", + "SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows" + }, + "Parameters": { + "DotNetVersion": "8.0.204", + "TargetFramework": "net8.0", + "ServerPort": 9876, + "ServerCoreAffinity": "0-7", + "ClientCoreAffinity": "8-15" + }, + "Actions": [ + { + "Type": "AspNetServerExecutor", + "Parameters": { + "Scenario": "JsonSerializationWithAffinity", + "Role": "Server", + "PackageName": "aspnetbenchmarks", + "DotNetSdkPackageName": "dotnetsdk", + "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "$.Parameters.ServerPort", + "BindToCores": true, + "CoreAffinity": "$.Parameters.ServerCoreAffinity" + } + }, + { + "Type": "BombardierExecutor", + "Parameters": { + "Scenario": "JsonSerializationWithAffinity", + "Role": "Client", + "PackageName": "bombardier", + "CommandArguments": "--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ServerIp}:$.Parameters.ServerPort/json --print r --format json", + "WarmUp": false, + "Timeout": "00:05:00", + "BindToCores": true, + "CoreAffinity": "$.Parameters.ClientCoreAffinity" + } + } + ], + "Dependencies": [ + { + "Type": "ChocolateyInstallation", + "Parameters": { + "Scenario": "InstallChocolatey", + "PackageName": "chocolatey" + } + }, + { + "Type": "ChocolateyPackageInstallation", + "Parameters": { + "Scenario": "InstallGit", + "PackageName": "chocolatey", + "Packages": "git" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneAspNetBenchmarksRepo", + "RepoUri": "https://github.com/aspnet/Benchmarks.git", + "Commit": "cf5b6ee", + "PackageName": "aspnetbenchmarks" + } + }, + { + "Type": "DotNetInstallation", + "Parameters": { + "Scenario": "InstallDotNetSdk", + "DotNetVersion": "$.Parameters.DotNetVersion", + "PackageName": "dotnetsdk" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallBombardierPackage", + "BlobContainer": "packages", + "BlobName": "bombardier.1.2.5.zip", + "PackageName": "bombardier", + "Extract": true + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json index 44f2ef4970..ab5d00b09e 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json @@ -14,40 +14,38 @@ }, "Actions": [ { - "Type": "AspNetBenchServerExecutor", + "Type": "AspNetServerExecutor", "Parameters": { "Role": "Server", "Scenario": "ExecuteJsonSerializationBenchmark", "PackageName": "aspnetbenchmarks", - "BombardierPackageName": "bombardier", "DotNetSdkPackageName": "dotnetsdk", "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "9876", "AspNetCoreThreadCount": "$.Parameters.AspNetCoreThreadCount", "DotNetSystemNetSocketsThreadCount": "$.Parameters.DotNetSystemNetSocketsThreadCount" } }, { - "Type": "AspNetBenchClientExecutor", + "Type": "WrkExecutor", "Parameters": { "Role": "Client", "Scenario": "ExecuteJsonSerializationBenchmarkWarmUp", - "PackageName": "aspnetbenchmarks", - "BombardierPackageName": "bombardier", - "DotNetSdkPackageName": "dotnetsdk", - "TargetFramework": "$.Parameters.TargetFramework", - "WrkCommandLine": "-t 256 -c 256 -d 45s --timeout 10s http://{ipAddress}:{port}/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q = 0.9,application/xml;q=0.8,*/*;q=0.7\"" + "PackageName": "wrk", + "CommandArguments": "-t 256 -c 256 -d 45s --timeout 10s http://{ServerIp}:9876/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\"", + "WarmUp": true, + "Timeout": "00:05:00" } }, { - "Type": "AspNetBenchClientExecutor", + "Type": "WrkExecutor", "Parameters": { "Role": "Client", "Scenario": "ExecuteJsonSerializationBenchmark", - "PackageName": "aspnetbenchmarks", - "BombardierPackageName": "bombardier", - "DotNetSdkPackageName": "dotnetsdk", - "TargetFramework": "$.Parameters.TargetFramework", - "WrkCommandLine": "-t 256 -c 256 -d 15s --timeout 10s http://{ipAddress}:{port}/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q = 0.9,application/xml;q=0.8,*/*;q=0.7\"" + "PackageName": "wrk", + "CommandArguments": "-t 256 -c 256 -d 15s --timeout 10s http://{ServerIp}:9876/json --header \"Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7\"", + "WarmUp": false, + "Timeout": "00:05:00" } } @@ -85,6 +83,17 @@ "Role": "Client" } }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, { "Type": "GitRepoClone", "Parameters": { diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json index f4baf54582..d8e1606d0b 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json @@ -7,17 +7,30 @@ }, "Parameters": { "DotNetVersion": "8.0.204", - "TargetFramework": "net8.0" + "TargetFramework": "net8.0", + "ServerPort": 9876 }, "Actions": [ { - "Type": "AspNetBenchExecutor", + "Type": "AspNetServerExecutor", "Parameters": { + "Role": "Server", "Scenario": "ExecuteJsonSerializationBenchmark", "PackageName": "aspnetbenchmarks", - "BombardierPackageName": "bombardier", "DotNetSdkPackageName": "dotnetsdk", - "TargetFramework": "$.Parameters.TargetFramework" + "TargetFramework": "$.Parameters.TargetFramework", + "ServerPort": "$.Parameters.ServerPort" + } + }, + { + "Type": "BombardierExecutor", + "Parameters": { + "Role": "Client", + "Scenario": "ExecuteJsonSerializationBenchmark", + "PackageName": "bombardier", + "CommandArguments": "--duration 15s --connections 256 --timeout 10s --fasthttp --insecure -l http://{ServerIp}:9876/json --print r --format json", + "WarmUp": false, + "Timeout": "00:05:00" } } ], @@ -63,6 +76,12 @@ "PackageName": "bombardier", "Extract": true } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } } ] } \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK-RP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK-RP.json new file mode 100644 index 0000000000..2f93d6ae6b --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK-RP.json @@ -0,0 +1,404 @@ +{ + "Description": "Networking benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "01:00:00", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu", + "Note": "Reverse Proxy config requires three nodes for the roles Client, Reverse Proxy, and Server." + }, + "Parameters": { + "TestDuration": "00:02:30", + "Timeout": "00:30:00", + "FileSizeInKB": 1, + "EmitLatencySpectrum": false, + "Workers": null, + "TargetService": "reverse-proxy", + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s https://{reverseproxyip}/{FileSizeInKB}kb" + }, + "Actions": [ + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "Server", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "ReverseProxy", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip nginx" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "StopNginxAutostart", + "Command": "sudo systemctl disable nginx" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "ReverseProxy" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK.json new file mode 100644 index 0000000000..5e97fc1fa6 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK.json @@ -0,0 +1,382 @@ +{ + "Description": "Networking benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "01:00:00", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu" + }, + "Parameters": { + "TestDuration": "00:02:30", + "Timeout": "00:30:00", + "FileSizeInKB": 1, + "EmitLatencySpectrum": false, + "Workers": null, + "TargetService": "server", + "CommandArguments": "--latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s https://{serverip}/api_new/{FileSizeInKB}kb" + }, + "Actions": [ + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "Server", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/2", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/4", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_100_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 100, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_1K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 1000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_5K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 5000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + }, + { + "Type": "WrkExecutor", + "Parameters": { + "PackageName": "wrk", + "Scenario": "Latency_10K_Connections_Thread/8", + "MetricScenario": "{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 10000, + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip nginx" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "StopNginxAutostart", + "Command": "sudo systemctl disable nginx" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk", + "RepoUri": "https://github.com/wg/wrk", + "PackageName": "wrk", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk}", + "Role": "Client" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2-RP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2-RP.json new file mode 100644 index 0000000000..5bcc46e73e --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2-RP.json @@ -0,0 +1,421 @@ +{ + "Description": "Networking benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "01:00:00", + "SupportedPlatforms": "linux-x64", + "SupportedOperatingSystems": "Ubuntu 22", + "Note": "Reverse Proxy config requires three nodes for the roles Client, Reverse Proxy, and Server." + }, + "Parameters": { + "TestDuration": "00:02:30", + "Timeout": "00:20:00", + "FileSizeInKB": 1, + "EmitLatencySpectrum": false, + "Workers": null, + "TargetService": "reverse-proxy", + "Rate": 1000, + "CommandArguments": "--rate {Rate} --latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s https://{reverseproxyip}/{FileSizeInKB}kb" + }, + "Actions": [ + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "Server", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "ReverseProxy", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip nginx" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "StopNginxAutostart", + "Command": "sudo systemctl disable nginx" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "ReverseProxy" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk2", + "RepoUri": "https://github.com/giltene/wrk2", + "PackageName": "wrk2", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk2", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk2}", + "Role": "Client" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2.json new file mode 100644 index 0000000000..b74759e262 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-WEB-NGINX-WRK2.json @@ -0,0 +1,399 @@ +{ + "Description": "Networking benchmarking Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "01:00:00", + "SupportedPlatforms": "linux-x64", + "SupportedOperatingSystems": "Ubuntu 22" + }, + "Parameters": { + "TestDuration": "00:02:30", + "Timeout": "00:20:00", + "FileSizeInKB": 1, + "EmitLatencySpectrum": false, + "Workers": null, + "TargetService": "server", + "Rate": 1000, + "CommandArguments": "--rate {Rate} --latency --threads {ThreadCount} --connections {Connection} --duration {TestDuration.TotalSeconds}s --timeout 10s https://{serverip}/api_new/{FileSizeInKB}kb" + }, + "Actions": [ + { + "Type": "NginxServerExecutor", + "Parameters": { + "Role": "Server", + "PackageName": "nginxconfiguration", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "Timeout": "$.Parameters.Timeout", + "Workers": "$.Parameters.Workers" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{LogicalCoreCount}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/2", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/2)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/4", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/4)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_100_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 100, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_1K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 1000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_5K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 5000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + }, + { + "Type": "Wrk2Executor", + "Parameters": { + "PackageName": "wrk2", + "Scenario": "Latency_10K_Connections_Thread/8", + "MetricScenario": "{Rate}r_{ThreadCount}th_{Connection}c_{FileSizeInKB}kb", + "CommandArguments": "$.Parameters.CommandArguments", + "ThreadCount": "{calculate({LogicalCoreCount}/8)}", + "Connection": 10000, + "Rate": "$.Parameters.Rate", + "FileSizeInKB": "$.Parameters.FileSizeInKB", + "TestDuration": "$.Parameters.TestDuration", + "EmitLatencySpectrum": "$.Parameters.EmitLatencySpectrum", + "TargetService": "$.Parameters.TargetService", + "Role": "Client", + "Timeout": "$.Parameters.Timeout", + "Tags": "Networking,NGINX,WRK2" + } + } + ], + "Dependencies": [ + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "openssl make gcc zlib1g-dev libssl-dev unzip nginx" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "StopNginxAutostart", + "Command": "sudo systemctl disable nginx" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallNginxConfiguration", + "BlobContainer": "packages", + "BlobName": "nginxconfiguration.1.0.2.zip", + "PackageName": "nginxconfiguration", + "Extract": true, + "Role": "Server" + } + }, + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallWrkConfiguration", + "BlobContainer": "packages", + "BlobName": "wrkconfiguration.1.0.0.zip", + "PackageName": "wrkconfiguration", + "Extract": true, + "Role": "Client" + } + }, + { + "Type": "GitRepoClone", + "Parameters": { + "Scenario": "CloneWrk2", + "RepoUri": "https://github.com/giltene/wrk2", + "PackageName": "wrk2", + "Role": "Client" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "CompileWrk2", + "Command": "make", + "WorkingDirectory": "{PackagePath:wrk2}", + "Role": "Client" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] +} diff --git a/website/docs/workloads/aspnetbench/aspnetbench-profiles.md b/website/docs/workloads/aspnetbench/aspnetbench-profiles.md index d44fcc0010..4b3e16c659 100644 --- a/website/docs/workloads/aspnetbench/aspnetbench-profiles.md +++ b/website/docs/workloads/aspnetbench/aspnetbench-profiles.md @@ -4,46 +4,176 @@ The following profiles run customer-representative or benchmarking scenarios usi * [Workload Details](./aspnetbench.md) ## PERF-ASPNETBENCH.json -Runs the AspNetBench benchmark workload to assess the performance of an ASP.NET Server. +Runs the AspNetBench benchmark workload using Bombardier client to assess the performance of an ASP.NET Server. This profile uses a **multi-role architecture** with separate server and client executors running on the same machine. + +**Executors Used:** +- `AspNetServerExecutor` (Server role) - Builds and runs ASP.NET Benchmarks +- `BombardierExecutor` (Client role) - Sends HTTP load to the server * [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH.json) +* **Dependencies** + The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. + * Internet connection. + * .NET SDK + * AspNetBenchmarks repository + * Bombardier package + + Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: + * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) + + | Parameter | Purpose | Default Value | + |---------------------------|-------------------------------------------------------------------|---------------| + | DotNetVersion | Optional. The version of the [.NET SDK to download and install](https://dotnet.microsoft.com/en-us/download/visual-studio-sdks). | 8.0.204 | + | TargetFramework | Optional. The [.NET target framework](https://learn.microsoft.com/en-us/dotnet/standard/frameworks) to run (e.g. net8.0, net9.0). | net8.0 | + | ServerPort | Optional. The port number for the ASP.NET server. | 9876 | + +* **Profile Runtimes** + See the 'Metadata' section of the profile for estimated runtimes. These timings represent the length of time required to run a single round of profile + The following section provides a few basic examples of how to use the workload profile. + + ``` bash + # Execute the workload profile (single machine with multi-role pattern) + VirtualClient.exe --profile=PERF-ASPNETBENCH.json --system=Demo --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" + + # Override the profile default parameters to use a different .NET SDK version + VirtualClient.exe --profile=PERF-ASPNETBENCH.json --system=Demo --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" --parameters="DotNetVersion=9.0.101,TargetFramework=net9.0" + + # Run on distributed systems (separate server and client machines) + # On Server machine: + VirtualClient.exe --profile=PERF-ASPNETBENCH.json --system=Demo --timeout=1440 --clientId=Server --layoutPath=layout.json --packageStore="{BlobConnectionString|SAS Uri}" + + # On Client machine: + VirtualClient.exe --profile=PERF-ASPNETBENCH.json --system=Demo --timeout=1440 --clientId=Client --layoutPath=layout.json --packageStore="{BlobConnectionString|SAS Uri}" + ``` + +## PERF-ASPNETBENCH-MULTI.json +Runs the AspNetBench benchmark workload using Wrk client in a multi-role configuration. This profile demonstrates the flexibility of using different HTTP clients with the same ASP.NET server. + +**Executors Used:** +- `AspNetServerExecutor` (Server role) - Builds and runs ASP.NET Benchmarks +- `WrkExecutor` (Client role) - Sends HTTP load using Wrk (with warm-up and benchmark phases) + +* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-MULTI.json) + +* **Supported Platform/Architectures** + * linux-x64 + * linux-arm64 + +* **Profile Parameters** + + | Parameter | Purpose | Default Value | + |-------------------------------------|-------------------------------------------------------------------|---------------| + | DotNetVersion | Optional. The version of the .NET SDK. | 8.0.204 | + | TargetFramework | Optional. The .NET target framework. | net8.0 | + | AspNetCoreThreadCount | Optional. ASPNETCORE_threadCount environment variable. | 1 | + | DotNetSystemNetSocketsThreadCount | Optional. DOTNET_SYSTEM_NET_SOCKETS_THREAD_COUNT environment variable. | 1 | + +* **Usage Examples** + + ``` bash + # Execute on distributed systems (recommended for accurate results) + # On Server machine: + VirtualClient.exe --profile=PERF-ASPNETBENCH-MULTI.json --system=Demo --timeout=1440 --clientId=Server --layoutPath=layout.json --packageStore="{BlobConnectionString|SAS Uri}" + + # On Client machine: + VirtualClient.exe --profile=PERF-ASPNETBENCH-MULTI.json --system=Demo --timeout=1440 --clientId=Client --layoutPath=layout.json --packageStore="{BlobConnectionString|SAS Uri}" + ``` + +## PERF-ASPNET-TEJSON-WRK.json +Runs ASP.NET JSON serialization benchmarks using Wrk with configurable thread and connection counts. + +**Executors Used:** +- `AspNetServerExecutor` (Server role) +- `WrkExecutor` (Client role) - With warm-up phase + +* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNET-TEJSON-WRK.json) + * **Supported Platform/Architectures** * linux-x64 * linux-arm64 * win-x64 * win-arm64 -* **Supports Disconnected Scenarios** - * No. Internet connection required. +* **Profile Parameters** -* **Dependencies** + | Parameter | Purpose | Default Value | + |---------------------------|-------------------------------------------------------------------|---------------| + | DotNetVersion | Optional. The version of the .NET SDK. | 9.0.101 | + | TargetFramework | Optional. The .NET target framework. | net9.0 | + | ServerPort | Optional. The port number for the ASP.NET server. | 9876 | + | TestDuration | Optional. Duration of each test run. | 00:00:15 | + | Timeout | Optional. Timeout for operations. | 00:10:00 | + | EmitLatencySpectrum | Optional. Whether to emit detailed latency metrics. | false | + +* **Usage Examples** + + ``` bash + # Execute on distributed systems + # On Server machine: + VirtualClient.exe --profile=PERF-ASPNET-TEJSON-WRK.json --system=Demo --timeout=1440 --clientId=Server --layoutPath=layout.json --packageStore="{BlobConnectionString|SAS Uri}" + + # On Client machine: + VirtualClient.exe --profile=PERF-ASPNET-TEJSON-WRK.json --system=Demo --timeout=1440 --clientId=Client --layoutPath=layout.json --packageStore="{BlobConnectionString|SAS Uri}" + ``` + +## PERF-ASPNETBENCH-AFFINITY.json +Runs ASP.NET JSON serialization benchmark with CPU core affinity pinning. Server and client processes +are isolated to separate core sets, enabling reproducible single-VM benchmarking. + +**Executors Used:** +- `AspNetServerExecutor` (Server role) - Builds and runs ASP.NET Benchmarks pinned to ServerCoreAffinity +- `BombardierExecutor` (Client role) - Sends HTTP load pinned to ClientCoreAffinity + +* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-ASPNETBENCH-AFFINITY.json) + +* **Supported Platform/Architectures** + * linux-x64 + * linux-arm64 + * win-x64 + * win-arm64 + +* **Supports Disconnected Scenarios** + * Yes. The workload packages can be pre-installed on the system. + +* **Dependencies** The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. - * Internet connection. + * .NET SDK + * AspNetBenchmarks repository + * Bombardier package Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) -* **Profile Parameters** +* **Profile Parameters** The following parameters can be optionally supplied on the command line to modify the behaviors of the workload. | Parameter | Purpose | Default Value | |---------------------------|-------------------------------------------------------------------|---------------| - | DotNetVersion | Optional. The version of the [.NET SDK to download and install](https://dotnet.microsoft.com/en-us/download/visual-studio-sdks). | 7.0.100 | - | TargetFramework | Optional. The [.NET target framework](https://learn.microsoft.com/en-us/dotnet/standard/frameworks) to run (e.g. net6.0, net7.0). | net7.0 | + | DotNetVersion | Optional. The version of the [.NET SDK to download and install](https://dotnet.microsoft.com/en-us/download/visual-studio-sdks). | 8.0.204 | + | TargetFramework | Optional. The [.NET target framework](https://learn.microsoft.com/en-us/dotnet/standard/frameworks) to run (e.g. net8.0, net9.0). | net8.0 | + | ServerPort | Optional. The port number for the ASP.NET server. | 9876 | + | ServerCoreAffinity | Optional. CPU cores to bind the server process to (e.g., "0-7"). | 0-7 | + | ClientCoreAffinity | Optional. CPU cores to bind the client process to (e.g., "8-15"). | 8-15 | -* **Profile Runtimes** - See the 'Metadata' section of the profile for estimated runtimes. These timings represent the length of time required to run a single round of profile - actions. These timings can be used to determine minimum required runtimes for the Virtual Client in order to get results. These are often estimates based on the - number of system cores. +* **Profile Runtimes** + See the 'Metadata' section of the profile for estimated runtimes. These timings represent the length of time required to run a single round of profile + actions. These timings can be used to determine minimum required runtimes for the Virtual Client in order to get results. -* **Usage Examples** +* **Usage Examples** The following section provides a few basic examples of how to use the workload profile. ``` bash - # Execute the workload profile - VirtualClient.exe --profile=PERF-ASPNETBENCH.json --system=Demo --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" + # Single VM with default core assignments (0-7 server, 8-15 client) + VirtualClient.exe --profile=PERF-ASPNETBENCH-AFFINITY.json --system=Demo --timeout=60 --packageStore="{BlobConnectionString|SAS Uri}" - # Override the profile default parameters to use a different .NET SDK version - VirtualClient.exe --profile=PERF-ASPNETBENCH.json --system=Demo --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" --parameters="DotNetVersion=7.0.307" - ``` \ No newline at end of file + # Override core assignments for a 32-core machine + VirtualClient.exe --profile=PERF-ASPNETBENCH-AFFINITY.json --system=Demo --timeout=60 --packageStore="{BlobConnectionString|SAS Uri}" --parameters="ServerCoreAffinity=0-15,,,ClientCoreAffinity=16-31" + + # Run on distributed systems + # On Server machine: + VirtualClient.exe --profile=PERF-ASPNETBENCH-AFFINITY.json --system=Demo --timeout=60 --clientId=Server --layoutPath=layout.json --packageStore="{BlobConnectionString|SAS Uri}" + + # On Client machine: + VirtualClient.exe --profile=PERF-ASPNETBENCH-AFFINITY.json --system=Demo --timeout=60 --clientId=Client --layoutPath=layout.json --packageStore="{BlobConnectionString|SAS Uri}" + ``` diff --git a/website/docs/workloads/aspnetbench/aspnetbench.md b/website/docs/workloads/aspnetbench/aspnetbench.md index 9f89941e05..c6b987f66d 100644 --- a/website/docs/workloads/aspnetbench/aspnetbench.md +++ b/website/docs/workloads/aspnetbench/aspnetbench.md @@ -1,11 +1,11 @@ # AspNetBenchmark AspNetBenchmark is a benchmark developed by MSFT ASPNET team, based on open source benchmark TechEmpower. -This workload has server and client part, on the same test machine. The server part is started as a ASPNET service. The client calls server using open source bombardier binaries. -Bombardier binaries could be downloaded from Github release, or directly compile from source using "go build ." +This workload supports both single-machine and multi-role (client-server) configurations. The server part runs as an ASP.NET service using Kestrel. The client uses either Bombardier or Wrk to send HTTP requests to the server. * [AspNetBenchmarks Github](https://github.com/aspnet/benchmarks) * [Bombardier Github](https://github.com/codesenberg/bombardier) * [Bombardier Release](https://github.com/codesenberg/bombardier/releases/tag/v1.2.5) +* [Wrk Github](https://github.com/wg/wrk) ## Workload Metrics The following metrics are examples of those captured by the Virtual Client when running the AspNetBenchmark workload. @@ -33,24 +33,84 @@ The following metrics are examples of those captured during the operations of th | RequestPerSecond P95 | 41662.542962 | Reqs/sec | ASP.NET Web Request per second (P95) | | RequestPerSecond P99 | 48600.556224 | Reqs/sec | ASP.NET Web Request per second (P99) | +## CPU Core Affinity +Core affinity binds a process to run exclusively on specified CPU cores. This eliminates OS scheduler interference +where the server and client processes compete for the same cores on single-VM deployments. On a 16+ core machine, +pinning the server to one half and the client to the other produces results comparable to a two-machine setup. + +### Linux +Uses `numactl -C ` to wrap the process command. numactl must be installed on the system. + +### Windows +Uses the Windows processor affinity bitmask API applied to the process after it starts. + +### Core Specification Format + +| Format | Example | Meaning | +|--------|---------|---------| +| Range | `0-7` | Cores 0 through 7 | +| List | `0,2,4,6` | Specific cores | +| Mixed | `0-3,8-11` | Cores 0-3 and 8-11 | + +### Choosing Core Assignments +- Ensure server and client core sets do **not overlap**. +- Use `lscpu` (Linux) or Task Manager (Windows) to check available cores and NUMA topology. +- For NUMA-aware pinning, keep each workload within a single NUMA node. + ## Packaging and Setup -The following section covers how to create the custom Virtual Client dependency packages required to execute the workload and toolset(s). This section -is meant to provide guidance for users that would like to create their own packages with the software for use with the Virtual Client. For example, users -may want to bring in new versions of the software. See the documentation on '[Dependency Packages](https://microsoft.github.io/VirtualClient/docs/developing/0040-vc-packages/)' -for more information on the concepts. +The following section covers how to create the custom Virtual Client dependency packages required to execute the workload and toolset(s). -1. VC installs dotnet SDK -2. VC clones AspNetBenchmarks github repo -3. dotnet build src/benchmarks project in AspNetBenchmarks repo -4. Use dotnet to start server +### Architecture -``` -dotnet \Benchmarks.dll --nonInteractive true --scenarios json --urls http://localhost:5000 --server Kestrel --kestrelTransport Sockets --protocol http --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" --header "Connection: keep-alive" +The workload now uses a **decoupled architecture** with separate executors: + +**Server Executor:** +- `AspNetServerExecutor` - Builds and runs the ASP.NET Benchmarks application + +**Client Executors:** +- `BombardierExecutor` - HTTP load testing client (cross-platform) +- `WrkExecutor` - HTTP load testing client (Linux only) + +### Single Machine Setup + +For single-machine testing, both server and client run on the same system using multi-role pattern: + +1. VC installs .NET SDK +2. VC clones AspNetBenchmarks GitHub repo +3. VC builds the Benchmarks project using .NET SDK +4. `AspNetServerExecutor` starts the Kestrel server (Server role) +5. `BombardierExecutor` or `WrkExecutor` sends requests (Client role) + +**Server Command:** +```bash +dotnet Benchmarks.dll --nonInteractive true --scenarios json --urls http://*:9876 \ + --server Kestrel --kestrelTransport Sockets --protocol http \ + --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" \ + --header "Connection: keep-alive" ``` -5. Use bombardier to start client +**Client Command (Bombardier):** +```bash +bombardier --duration 15s --connections 256 --timeout 10s --fasthttp --insecure \ + -l http://localhost:9876/json --print r --format json ``` -bombardier-windows-amd64.exe -d 15s -c 256 -t 2s --fasthttp --insecure -l http://localhost:5000/json --print r --format json + +**Client Command (Wrk):** +```bash +wrk -t 256 -c 256 -d 15s --timeout 10s http://localhost:9876/json \ + --header "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7" ``` +### Multi-Machine Setup + +For distributed testing, server and client run on separate machines: + +1. **Server Machine**: Runs `AspNetServerExecutor` with `Role: Server` +2. **Client Machine(s)**: Run `BombardierExecutor` or `WrkExecutor` with `Role: Client` +3. Client automatically discovers server IP via Virtual Client API +**Benefits:** +- Eliminates resource contention between server and client +- Better represents real-world scenarios +- Can scale to multiple client machines +- Mix and match different client tools