dotnet · filzrev · Jan 5, 2026
diff --git a/BenchmarkDotNet.slnx b/BenchmarkDotNet.slnx
@@ -6,6 +6,7 @@
   <Folder Name="/samples/">
     <File Path="samples/Directory.Build.props" />
     <Project Path="samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj" />
+    <Project Path="samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet.Samples.Sandbox.csproj" />
     <Project Path="samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj" DefaultStartup="true" />
   </Folder>
   <Folder Name="/src/">

diff --git a/samples/BenchmarkDotNet.Samples.Sandbox.slnf b/samples/BenchmarkDotNet.Samples.Sandbox.slnf
@@ -0,0 +1,11 @@
+{
+  "solution": {
+    "path": "..\\BenchmarkDotNet.slnx",
+    "projects": [
+      "samples\\BenchmarkDotNet.Samples.Sandbox\\BenchmarkDotNet.Samples.Sandbox.csproj",
+      "src\\BenchmarkDotNet.Analyzers\\BenchmarkDotNet.Analyzers.csproj",
+      "src\\BenchmarkDotNet.TestAdapter\\BenchmarkDotNet.TestAdapter.csproj",
+      "src\\BenchmarkDotNet\\BenchmarkDotNet.csproj"
+    ]
+  }
+}
diff --git a/samples/BenchmarkDotNet.Samples.Sandbox/.runsettings b/samples/BenchmarkDotNet.Samples.Sandbox/.runsettings
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <RunConfiguration>
+    <EnvironmentVariables>
+      <!-- Set `Debug` or `DebugInProcess` to when debugging benchmark -->
+      <BENCHMARKDOTNET_CONFIG></BENCHMARKDOTNET_CONFIG>
+    </EnvironmentVariables>
+  </RunConfiguration>
+</RunSettings>
diff --git a/samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet.Samples.Sandbox.csproj b/samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet.Samples.Sandbox.csproj
@@ -0,0 +1,38 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="..\..\build\common.props" />
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFrameworks>net8.0;net9.0;net10.0</TargetFrameworks>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="$([MSBuild]::IsOSPlatform('Windows'))">
+    <TargetFrameworks>$(TargetFrameworks);net48</TargetFrameworks>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <!-- Disable entry point generation as this project has it's own entry point -->
+    <GenerateProgramFile>false</GenerateProgramFile>
+    <!-- Disable parallel tests between TargetFrameworks -->
+    <TestTfmsInParallel>false</TestTfmsInParallel>
+    <!-- Enable .runsettings to use custom enviroment variable -->
+    <RunSettingsFilePath>$(MSBuildProjectDirectory)\.runsettings</RunSettingsFilePath>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\BenchmarkDotNet\BenchmarkDotNet.csproj" />
+    <ProjectReference Include="..\..\src\BenchmarkDotNet.Analyzers\BenchmarkDotNet.Analyzers.csproj">
+      <OutputItemType>Analyzer</OutputItemType>
+      <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+    </ProjectReference>
+  </ItemGroup>
+
+  <!-- Enable VS TestAdapter intergration -->
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.1" />
+    <ProjectReference Include="..\..\src\BenchmarkDotNet.TestAdapter\BenchmarkDotNet.TestAdapter.csproj" />
+  </ItemGroup>
+
+  <Import Project="..\..\build\common.targets" />
+</Project>
diff --git a/samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/BaseBenchmarkConfig.cs b/samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/BaseBenchmarkConfig.cs
@@ -0,0 +1,121 @@
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Diagnosers;
+using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Order;
+using BenchmarkDotNet.Reports;
+using Perfolizer.Horology;
+
+namespace BenchmarkDotNet;
+
+public class BaseBenchmarkConfig : ManualConfig
+{
+    public BaseBenchmarkConfig()
+    {
+        WithSummaryStyle(SummaryStyle.Default.WithMaxParameterColumnWidth(40)); // Default: 20 chars
+        WithBuildTimeout(TimeSpan.FromMinutes(10)); // Default: 120 seconds
+
+        WithOrderer(new DefaultOrderer());
+        WithUnionRule(ConfigUnionRule.Union);
+        WithArtifactsPath(DefaultConfig.Instance.ArtifactsPath!);
+
+#if DEBUG
+        // Allow benchmarks for debug build.
+        WithOptions(ConfigOptions.DisableOptimizationsValidator);
+#endif
+
+        // Enable following settings for debugging
+        // WithOptions(ConfigOptions.StopOnFirstError);
+        // WithOptions(ConfigOptions.KeepBenchmarkFiles);
+        // WithOptions(ConfigOptions.GenerateMSBuildBinLog);
+    }
+
+    // Use ShortRun based settings (LaunchCount=1 IterationCount=3 WarmupCount = 3)
+    // And use RecommendedConfig setting that used by `dotnet/performance` repository.
+    // https://github.com/dotnet/performance/blob/main/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs
+    protected virtual Job GetBaseJobConfig() =>
+        Job.Default
+           .WithLaunchCount(1)
+           .WithWarmupCount(3)
+           .WithIterationTime(TimeInterval.FromMilliseconds(250)) // Default: 500 [ms]
+           .WithMinIterationCount(15)  // Default: 15
+           .WithMaxIterationCount(20); // Default: 100
+
+    /// <summary>
+    /// Add configurations.
+    /// </summary>
+    protected void AddConfigurations()
+    {
+        AddAnalyzers();
+        AddColumnHidingRules();
+        AddColumnProviders();
+        AddDiagnosers();
+        AddEventProcessors();
+        AddExporters();
+        AddFilters();
+        AddHardwareCounters();
+        AddLoggers();
+        AddLogicalGroupRules();
+        AddValidators();
+    }
+
+    protected virtual void AddAnalyzers()
+    {
+        AddAnalyser(DefaultConfig.Instance.GetAnalysers().ToArray());
+    }
+
+    protected virtual void AddColumnHidingRules()
+    {
+    }
+
+    protected virtual void AddColumnProviders()
+    {
+        AddColumnProvider(DefaultColumnProviders.Instance);
+    }
+
+    protected virtual void AddDiagnosers()
+    {
+        AddDiagnoser(MemoryDiagnoser.Default);
+        AddDiagnoser(new ExceptionDiagnoser(new ExceptionDiagnoserConfig(displayExceptionsIfZeroValue: false)));
+
+#if NETCOREAPP3_0_OR_GREATER
+        AddDiagnoser(new ThreadingDiagnoser(new ThreadingDiagnoserConfig(displayCompletedWorkItemCountWhenZero: false, displayLockContentionWhenZero: false)));
+#endif
+    }
+
+    protected virtual void AddExporters()
+    {
+        // Use ConsoleMarkdownExporter to disable group higligting with `**`.
+        AddExporter(MarkdownExporter.Console);
+    }
+
+    protected virtual void AddEventProcessors()
+    {
+    }
+
+    protected virtual void AddFilters()
+    {
+        AddFilter(TargetFrameworkFilter.Instance);
+    }
+
+    protected virtual void AddHardwareCounters()
+    {
+    }
+
+    protected virtual void AddLoggers()
+    {
+        AddLogger(ConsoleLogger.Default);
+    }
+
+    protected virtual void AddLogicalGroupRules()
+    {
+    }
+
+    protected virtual void AddValidators()
+    {
+        AddValidator(DefaultConfig.Instance.GetValidators().ToArray());
+    }
+}
diff --git a/samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/DebugBenchmarkConfig.cs b/samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/DebugBenchmarkConfig.cs
@@ -0,0 +1,28 @@
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Toolchains.InProcess.Emit;
+using System.Runtime.InteropServices;
+
+namespace BenchmarkDotNet;
+
+public class DebugBenchmarkConfig : BaseBenchmarkConfig
+{
+    public DebugBenchmarkConfig()
+    {
+        // Configure base job config
+        var baseJobConfig = GetBaseJobConfig();
+
+        // Add benchmark job.
+        AddJob(baseJobConfig.WithCustomBuildConfiguration("Debug")
+                            .WithWarmupCount(1)
+                            .WithStrategy(RunStrategy.Monitoring)
+                            .WithId($"Debug({RuntimeInformation.FrameworkDescription})"));
+
+        // Set DebugConfig comatible option
+        WithOptions(ConfigOptions.KeepBenchmarkFiles);
+
+        // Configure additional settings.
+        AddConfigurations();
+    }
+}
diff --git a/.../BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/DebugInProcessBenchmarkConfig.cs b/.../BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/DebugInProcessBenchmarkConfig.cs
@@ -0,0 +1,40 @@
+using BenchmarkDotNet.Analysers;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Toolchains.InProcess.Emit;
+using System.Runtime.InteropServices;
+
+namespace BenchmarkDotNet;
+
+public class DebugInProcessBenchmarkConfig : BaseBenchmarkConfig
+{
+    public DebugInProcessBenchmarkConfig() : base()
+    {
+        // Configure base job config
+        var baseJobConfig = GetBaseJobConfig();
+
+        // Add benchmark job.
+        AddJob(baseJobConfig.WithToolchain(InProcessEmitToolchain.Default)
+                            .WithWarmupCount(1)
+                            .WithStrategy(RunStrategy.Monitoring)
+                            .WithId($"DebugInProcess({RuntimeInformation.FrameworkDescription})"));
+
+        // Set DebugConfig comatible option
+        WithOptions(ConfigOptions.KeepBenchmarkFiles);
+
+        // Configure additional settings.
+        AddConfigurations();
+    }
+
+    protected override void AddAnalyzers()
+    {
+        // Exclude MinIterationTimeAnalyser that cause warning when using RunStrategy.Monitoring.
+        var analyzers = DefaultConfig.Instance
+                                     .GetAnalysers()
+                                     .Where(x => x is not MinIterationTimeAnalyser)
+                                     .ToArray();
+
+        AddAnalyser(analyzers);
+    }
+}
diff --git a/samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/DefaultBenchmarkConfig.cs b/samples/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/DefaultBenchmarkConfig.cs
@@ -0,0 +1,22 @@
+using BenchmarkDotNet.Jobs;
+using System.Runtime.InteropServices;
+
+namespace BenchmarkDotNet;
+
+public class DefaultBenchmarkConfig : BaseBenchmarkConfig
+{
+    public DefaultBenchmarkConfig() : base()
+    {
+        // Configure base job config
+        var baseJobConfig = GetBaseJobConfig();
+
+        // Create benchmark job.
+        var job = baseJobConfig.WithId($"Default({RuntimeInformation.FrameworkDescription})");
+
+        // Add job.
+        AddJob(job);
+
+        // Configure additional settings.
+        AddConfigurations();
+    }
+}
diff --git a/...enchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/TargetFrameworksBenchmarkConfig.cs b/...enchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/Configs/TargetFrameworksBenchmarkConfig.cs
@@ -0,0 +1,50 @@
+using BenchmarkDotNet.Analysers;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Toolchains.CsProj;
+
+namespace BenchmarkDotNet;
+
+/// <summary>
+/// BenchmarkConfig that run benchmarks for multiple target frameworks.
+/// </summary>
+public class TargetFrameworksBenchmarkConfig : BaseBenchmarkConfig
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="DefaultBenchmarkConfig"/> class.
+    /// </summary>
+    public TargetFrameworksBenchmarkConfig() : base()
+    {
+        // Configure base job config
+        var baseJobConfig = GetBaseJobConfig();
+
+        // Add jobs
+        AddJob(baseJobConfig.WithToolchain(CsProjCoreToolchain.NetCoreApp80).WithId(".NET 8").AsBaseline());
+        AddJob(baseJobConfig.WithToolchain(CsProjCoreToolchain.NetCoreApp90).WithId(".NET 9"));
+        AddJob(baseJobConfig.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0).WithId(".NET 10"));
+
+        // AddJob(baseJobConfig.WithToolchain(CsProjClassicNetToolchain.Net48).WithId(".NET Framework 4.8"));
+
+        // Configure additional settings.
+        AddConfigurations();
+    }
+
+    protected override void AddLogicalGroupRules()
+    {
+        // Grouping benchmarks by method.
+        // Note: When following conditions are met. BaselineCustomAnalyzer raise warning. See: https://github.com/dotnet/BenchmarkDotNet/issues/2956
+        // 1. Job contains Baseline=true
+        // 2. Benchmark method contains Baseline=true
+        // 3. Enable grouping with method.
+        AddLogicalGroupRules(
+        [
+            BenchmarkLogicalGroupRule.ByMethod,
+        ]);
+    }
+
+    protected override void AddColumnHidingRules()
+    {
+        HideColumns(Column.Toolchain); // Toolchain information are shown at JobId column.
+    }
+}
diff --git a/...s/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/ExtensionMethods/SummariesExtensions.cs b/...s/BenchmarkDotNet.Samples.Sandbox/BenchmarkDotNet/ExtensionMethods/SummariesExtensions.cs
@@ -0,0 +1,23 @@
+using BenchmarkDotNet.Reports;
+
+namespace BenchmarkDotNet;
+
+public static class SummariesExtensions
+{
+    public static bool HasError(this Summary[] summaries)
+    {
+        if (summaries.Length == 0)
+        {
+            var hashSet = new HashSet<string>(["--help", "--list", "--info", "--version"]);
+
+            var args = Environment.GetCommandLineArgs();
+            return !args.Any(hashSet.Contains);
+        }
+
+        if (summaries.Any(x => x.HasCriticalValidationErrors))
+            return true;
+
+        return summaries.Any(x => x.Reports.Any(r => !r.Success));
+    }
+}
+