From 6ad7b731dccc3faf83630caec861f893fffa0d16 Mon Sep 17 00:00:00 2001 From: Andy Bodnar Date: Sun, 11 Jan 2026 14:29:21 -0700 Subject: [PATCH 1/3] Add integration tests for MonoAOTLLVMToolchain Closes #2536 --- .../MonoAotLlvmTests.cs | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 tests/BenchmarkDotNet.IntegrationTests/MonoAotLlvmTests.cs diff --git a/tests/BenchmarkDotNet.IntegrationTests/MonoAotLlvmTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MonoAotLlvmTests.cs new file mode 100644 index 0000000000..a96d99de6a --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/MonoAotLlvmTests.cs @@ -0,0 +1,172 @@ +using System; +using System.Linq; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.IntegrationTests.Diagnosers; +using BenchmarkDotNet.IntegrationTests.Xunit; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Tests.XUnit; +using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Toolchains.MonoAotLLVM; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.IntegrationTests +{ + /// + /// In order to run MonoAotLlvmTests locally, the following prerequisites are required: + /// * Have MonoAOT workload installed + /// * Have the Mono AOT compiler available at the path specified by MONOAOTLLVM_COMPILER_PATH environment variable + /// + public class MonoAotLlvmTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) + { + private const string AotCompilerPathEnvVar = "MONOAOTLLVM_COMPILER_PATH"; + + private static string GetAotCompilerPath() + => Environment.GetEnvironmentVariable(AotCompilerPathEnvVar); + + private static bool IsAotCompilerAvailable() + => !string.IsNullOrEmpty(GetAotCompilerPath()) && System.IO.File.Exists(GetAotCompilerPath()); + + private ManualConfig GetConfig(MonoAotCompilerMode mode = MonoAotCompilerMode.llvm) + { + var aotCompilerPath = GetAotCompilerPath(); + var logger = new OutputLogger(Output); + var netCoreAppSettings = new NetCoreAppSettings( + "net8.0", + null, + "MonoAOTLLVM", + aotCompilerPath: aotCompilerPath, + aotCompilerMode: mode); + + return ManualConfig.CreateEmpty() + .AddLogger(logger) + .AddJob(Job.Dry + .WithRuntime(new MonoAotLLVMRuntime( + new System.IO.FileInfo(aotCompilerPath), + mode, + "net8.0")) + .WithToolchain(MonoAotLLVMToolChain.From(netCoreAppSettings))) + .WithBuildTimeout(TimeSpan.FromMinutes(5)) + .WithOption(ConfigOptions.GenerateMSBuildBinLog, true); + } + + private static bool GetShouldRunTest() + { + // MonoAOTLLVM is only supported on non-Windows platforms with a 64-bit architecture + if (!RuntimeInformation.Is64BitPlatform()) + return false; + + if (OsDetector.IsWindows()) + return false; + + if (!IsAotCompilerAvailable()) + return false; + + return true; + } + + [FactEnvSpecific("MonoAOTLLVM is only supported on Unix", EnvRequirement.NonWindows)] + public void MonoAotLlvmIsSupported() + { + if (!GetShouldRunTest()) + { + Output.WriteLine($"Skipping test: AOT compiler not available at {AotCompilerPathEnvVar}"); + return; + } + + try + { + CanExecute(GetConfig()); + } + catch (MisconfiguredEnvironmentException e) + { + if (ContinuousIntegration.IsLocalRun()) + Output.WriteLine(e.SkipMessage); + else + throw; + } + } + + [FactEnvSpecific("MonoAOTLLVM is only supported on Unix", EnvRequirement.NonWindows)] + public void MonoAotLlvmSupportsInProcessDiagnosers() + { + if (!GetShouldRunTest()) + { + Output.WriteLine($"Skipping test: AOT compiler not available at {AotCompilerPathEnvVar}"); + return; + } + + try + { + var diagnoser = new MockInProcessDiagnoser1(BenchmarkDotNet.Diagnosers.RunMode.NoOverhead); + var config = GetConfig().AddDiagnoser(diagnoser); + + try + { + CanExecute(config); + } + catch (MisconfiguredEnvironmentException e) + { + if (ContinuousIntegration.IsLocalRun()) + { + Output.WriteLine(e.SkipMessage); + return; + } + throw; + } + + Assert.Equal([diagnoser.ExpectedResult], diagnoser.Results.Values); + Assert.Equal([diagnoser.ExpectedResult], BaseMockInProcessDiagnoser.s_completedResults.Select(t => t.result)); + } + finally + { + BaseMockInProcessDiagnoser.s_completedResults.Clear(); + } + } + + [FactEnvSpecific("MonoAOTLLVM is only supported on Unix", EnvRequirement.NonWindows)] + public void MonoAotLlvmMiniModeIsSupported() + { + if (!GetShouldRunTest()) + { + Output.WriteLine($"Skipping test: AOT compiler not available at {AotCompilerPathEnvVar}"); + return; + } + + try + { + CanExecute(GetConfig(MonoAotCompilerMode.mini)); + } + catch (MisconfiguredEnvironmentException e) + { + if (ContinuousIntegration.IsLocalRun()) + Output.WriteLine(e.SkipMessage); + else + throw; + } + } + + public class MonoAotLlvmBenchmark + { + [Benchmark] + public void Check() + { + // Verify we're running on Mono AOT + if (!RuntimeInformation.IsMono) + { + throw new Exception("This is not Mono runtime"); + } + + if (!RuntimeInformation.IsAot) + { + throw new Exception("This is not running in AOT mode"); + } + } + } + } +} From 36e566f88a0aa2c63ec32f51393397122fa54a4c Mon Sep 17 00:00:00 2001 From: Andy Bodnar Date: Sun, 11 Jan 2026 19:19:05 -0700 Subject: [PATCH 2/3] Add Windows support and installation docs for MonoAotLlvm tests Remove the Windows-only restriction since MonoAOTLLVM can work on Windows when the compiler is built from dotnet/runtime. Added setup instructions for both Windows and Unix platforms. --- .../MonoAotLlvmTests.cs | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MonoAotLlvmTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MonoAotLlvmTests.cs index a96d99de6a..46882e3d15 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MonoAotLlvmTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MonoAotLlvmTests.cs @@ -2,7 +2,6 @@ using System.Linq; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.IntegrationTests.Diagnosers; using BenchmarkDotNet.IntegrationTests.Xunit; @@ -18,9 +17,19 @@ namespace BenchmarkDotNet.IntegrationTests { /// - /// In order to run MonoAotLlvmTests locally, the following prerequisites are required: - /// * Have MonoAOT workload installed - /// * Have the Mono AOT compiler available at the path specified by MONOAOTLLVM_COMPILER_PATH environment variable + /// Running these tests locally requires building the mono runtime from the dotnet/runtime repository, + /// since the AOT compiler isn't distributed as a standalone package. + /// + /// To set up: + /// 1. Clone https://github.com/dotnet/runtime + /// 2. Build the mono runtime with libs: + /// Windows: build.cmd -subset mono+libs -c Release + /// Unix: ./build.sh -subset mono+libs -c Release + /// 3. Set the MONOAOTLLVM_COMPILER_PATH environment variable to the compiler binary: + /// Windows: artifacts\bin\mono\windows.x64.Release\mono-sgen.exe + /// Unix: artifacts/bin/mono/[os].x64.Release/mono-sgen + /// + /// The runtime pack ends up at artifacts/bin/microsoft.netcore.app.runtime.[os]-x64/Release/ /// public class MonoAotLlvmTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) { @@ -57,20 +66,17 @@ private ManualConfig GetConfig(MonoAotCompilerMode mode = MonoAotCompilerMode.ll private static bool GetShouldRunTest() { - // MonoAOTLLVM is only supported on non-Windows platforms with a 64-bit architecture + // MonoAOTLLVM requires a 64-bit platform if (!RuntimeInformation.Is64BitPlatform()) return false; - if (OsDetector.IsWindows()) - return false; - if (!IsAotCompilerAvailable()) return false; return true; } - [FactEnvSpecific("MonoAOTLLVM is only supported on Unix", EnvRequirement.NonWindows)] + [Fact] public void MonoAotLlvmIsSupported() { if (!GetShouldRunTest()) @@ -92,7 +98,7 @@ public void MonoAotLlvmIsSupported() } } - [FactEnvSpecific("MonoAOTLLVM is only supported on Unix", EnvRequirement.NonWindows)] + [Fact] public void MonoAotLlvmSupportsInProcessDiagnosers() { if (!GetShouldRunTest()) @@ -129,7 +135,7 @@ public void MonoAotLlvmSupportsInProcessDiagnosers() } } - [FactEnvSpecific("MonoAOTLLVM is only supported on Unix", EnvRequirement.NonWindows)] + [Fact] public void MonoAotLlvmMiniModeIsSupported() { if (!GetShouldRunTest()) From e3b217d4af7d2a0f9948317ec037a32f548b6c58 Mon Sep 17 00:00:00 2001 From: Andy Bodnar Date: Fri, 16 Jan 2026 22:17:58 -0700 Subject: [PATCH 3/3] Use FactEnvSpecific to properly skip tests when toolchain missing Per @timcassell's feedback - the tests were showing as "passed" in 2ms because they were silently skipping when MONOAOTLLVM_COMPILER_PATH wasn't set. This made CI look green even though nothing was actually tested. Changes: - Added EnvRequirement.MonoAotLlvmToolchain to the test framework - EnvRequirementChecker now validates the compiler path exists - Tests now use [FactEnvSpecific] so xunit marks them as "skipped" rather than "passed" when the toolchain isn't installed - Cleaned up redundant skip logic from the test methods - Updated setup instructions in the class docs Now CI will clearly show these tests as skipped, making it obvious they need the runtime build to actually run. --- .../MonoAotLlvmTests.cs | 111 ++++-------------- .../XUnit/EnvRequirement.cs | 5 +- .../XUnit/EnvRequirementChecker.cs | 12 +- 3 files changed, 36 insertions(+), 92 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MonoAotLlvmTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MonoAotLlvmTests.cs index 46882e3d15..df5988eb5d 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MonoAotLlvmTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MonoAotLlvmTests.cs @@ -4,7 +4,6 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.IntegrationTests.Diagnosers; -using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Tests.Loggers; @@ -17,19 +16,23 @@ namespace BenchmarkDotNet.IntegrationTests { /// - /// Running these tests locally requires building the mono runtime from the dotnet/runtime repository, - /// since the AOT compiler isn't distributed as a standalone package. + /// Running these tests requires building the Mono AOT runtime from dotnet/runtime. + /// The toolchain isn't available as a standalone workload yet. + /// + /// Setup instructions: /// - /// To set up: /// 1. Clone https://github.com/dotnet/runtime - /// 2. Build the mono runtime with libs: - /// Windows: build.cmd -subset mono+libs -c Release - /// Unix: ./build.sh -subset mono+libs -c Release - /// 3. Set the MONOAOTLLVM_COMPILER_PATH environment variable to the compiler binary: - /// Windows: artifacts\bin\mono\windows.x64.Release\mono-sgen.exe - /// Unix: artifacts/bin/mono/[os].x64.Release/mono-sgen /// - /// The runtime pack ends up at artifacts/bin/microsoft.netcore.app.runtime.[os]-x64/Release/ + /// 2. Build the mono runtime with libraries: + /// Windows: .\build.cmd -subset mono+libs -c Release + /// Linux: ./build.sh -subset mono+libs -c Release + /// macOS: ./build.sh -subset mono+libs -c Release + /// + /// 3. Set MONOAOTLLVM_COMPILER_PATH to the built compiler: + /// Windows: set MONOAOTLLVM_COMPILER_PATH=C:\path\to\runtime\artifacts\bin\mono\windows.x64.Release\mono-sgen.exe + /// Unix: export MONOAOTLLVM_COMPILER_PATH=/path/to/runtime/artifacts/bin/mono/linux.x64.Release/mono-sgen + /// + /// The runtime pack is at: artifacts/bin/microsoft.netcore.app.runtime.[os]-x64/Release/ /// public class MonoAotLlvmTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) { @@ -38,9 +41,6 @@ public class MonoAotLlvmTests(ITestOutputHelper output) : BenchmarkTestExecutor( private static string GetAotCompilerPath() => Environment.GetEnvironmentVariable(AotCompilerPathEnvVar); - private static bool IsAotCompilerAvailable() - => !string.IsNullOrEmpty(GetAotCompilerPath()) && System.IO.File.Exists(GetAotCompilerPath()); - private ManualConfig GetConfig(MonoAotCompilerMode mode = MonoAotCompilerMode.llvm) { var aotCompilerPath = GetAotCompilerPath(); @@ -64,67 +64,21 @@ private ManualConfig GetConfig(MonoAotCompilerMode mode = MonoAotCompilerMode.ll .WithOption(ConfigOptions.GenerateMSBuildBinLog, true); } - private static bool GetShouldRunTest() - { - // MonoAOTLLVM requires a 64-bit platform - if (!RuntimeInformation.Is64BitPlatform()) - return false; - - if (!IsAotCompilerAvailable()) - return false; - - return true; - } - - [Fact] + [FactEnvSpecific("Requires Mono AOT LLVM toolchain from dotnet/runtime build", EnvRequirement.MonoAotLlvmToolchain)] public void MonoAotLlvmIsSupported() { - if (!GetShouldRunTest()) - { - Output.WriteLine($"Skipping test: AOT compiler not available at {AotCompilerPathEnvVar}"); - return; - } - - try - { - CanExecute(GetConfig()); - } - catch (MisconfiguredEnvironmentException e) - { - if (ContinuousIntegration.IsLocalRun()) - Output.WriteLine(e.SkipMessage); - else - throw; - } + CanExecute(GetConfig()); } - [Fact] + [FactEnvSpecific("Requires Mono AOT LLVM toolchain from dotnet/runtime build", EnvRequirement.MonoAotLlvmToolchain)] public void MonoAotLlvmSupportsInProcessDiagnosers() { - if (!GetShouldRunTest()) - { - Output.WriteLine($"Skipping test: AOT compiler not available at {AotCompilerPathEnvVar}"); - return; - } - try { var diagnoser = new MockInProcessDiagnoser1(BenchmarkDotNet.Diagnosers.RunMode.NoOverhead); var config = GetConfig().AddDiagnoser(diagnoser); - try - { - CanExecute(config); - } - catch (MisconfiguredEnvironmentException e) - { - if (ContinuousIntegration.IsLocalRun()) - { - Output.WriteLine(e.SkipMessage); - return; - } - throw; - } + CanExecute(config); Assert.Equal([diagnoser.ExpectedResult], diagnoser.Results.Values); Assert.Equal([diagnoser.ExpectedResult], BaseMockInProcessDiagnoser.s_completedResults.Select(t => t.result)); @@ -135,26 +89,10 @@ public void MonoAotLlvmSupportsInProcessDiagnosers() } } - [Fact] + [FactEnvSpecific("Requires Mono AOT LLVM toolchain from dotnet/runtime build", EnvRequirement.MonoAotLlvmToolchain)] public void MonoAotLlvmMiniModeIsSupported() { - if (!GetShouldRunTest()) - { - Output.WriteLine($"Skipping test: AOT compiler not available at {AotCompilerPathEnvVar}"); - return; - } - - try - { - CanExecute(GetConfig(MonoAotCompilerMode.mini)); - } - catch (MisconfiguredEnvironmentException e) - { - if (ContinuousIntegration.IsLocalRun()) - Output.WriteLine(e.SkipMessage); - else - throw; - } + CanExecute(GetConfig(MonoAotCompilerMode.mini)); } public class MonoAotLlvmBenchmark @@ -162,16 +100,11 @@ public class MonoAotLlvmBenchmark [Benchmark] public void Check() { - // Verify we're running on Mono AOT if (!RuntimeInformation.IsMono) - { - throw new Exception("This is not Mono runtime"); - } + throw new Exception("Expected Mono runtime"); if (!RuntimeInformation.IsAot) - { - throw new Exception("This is not running in AOT mode"); - } + throw new Exception("Expected AOT mode"); } } } diff --git a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs index e9e546bd1e..b458584c22 100644 --- a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs +++ b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirement.cs @@ -9,5 +9,6 @@ public enum EnvRequirement FullFrameworkOnly, NonFullFramework, DotNetCoreOnly, - NeedsPrivilegedProcess -} \ No newline at end of file + NeedsPrivilegedProcess, + MonoAotLlvmToolchain +} diff --git a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs index f00a7bc0da..a464f095c0 100644 --- a/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs +++ b/tests/BenchmarkDotNet.Tests/XUnit/EnvRequirementChecker.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet.Environments; using System; +using System.IO; using System.Linq; using System.Runtime.InteropServices; using System.Security.Principal; @@ -9,6 +10,8 @@ namespace BenchmarkDotNet.Tests.XUnit; public static class EnvRequirementChecker { + private const string MonoAotCompilerPathEnvVar = "MONOAOTLLVM_COMPILER_PATH"; + public static string? GetSkip(params EnvRequirement[] requirements) => requirements.Select(GetSkip).FirstOrDefault(skip => skip != null); internal static string? GetSkip(EnvRequirement requirement) => requirement switch @@ -21,6 +24,7 @@ public static class EnvRequirementChecker EnvRequirement.NonFullFramework => !BdnRuntimeInformation.IsFullFramework ? null : "Non-Full .NET Framework test", EnvRequirement.DotNetCoreOnly => BdnRuntimeInformation.IsNetCore ? null : ".NET/.NET Core-only test", EnvRequirement.NeedsPrivilegedProcess => IsPrivilegedProcess() ? null : "Needs authorization to perform security-relevant functions", + EnvRequirement.MonoAotLlvmToolchain => IsMonoAotLlvmAvailable() ? null : $"Mono AOT LLVM toolchain not found. Set {MonoAotCompilerPathEnvVar} to the mono-sgen binary path", _ => throw new ArgumentOutOfRangeException(nameof(requirement), requirement, "Unknown value") }; @@ -36,4 +40,10 @@ private static bool IsPrivilegedProcess() private static bool IsArm() => BdnRuntimeInformation.GetCurrentPlatform() is Platform.Arm64 or Platform.Arm or Platform.Armv6; -} \ No newline at end of file + + private static bool IsMonoAotLlvmAvailable() + { + var compilerPath = Environment.GetEnvironmentVariable(MonoAotCompilerPathEnvVar); + return !string.IsNullOrEmpty(compilerPath) && File.Exists(compilerPath); + } +}