From 6d0be780a16f74ea7888350f0f96bbd43404516c Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Fri, 27 Mar 2026 10:00:24 -0600 Subject: [PATCH 1/5] Add Dictionary and HashSet Remove/Contains benchmarks with multi-size coverage Add focused benchmarks for Dictionary and HashSet operations at sizes 512 and 8192 (past L1/L2 cache) with int, string, and Guid type args: - Remove/RemoveTrue.cs: steady-state remove hit (remove + re-add) - Remove/RemoveFalse.cs: remove miss (absent keys) - Dictionary/DictionaryTryRemove.cs: Remove(key, out value) overload - Contains/HashSetContains.cs: HashSet Contains hit and miss - Contains/DictionaryContainsKey.cs: Dictionary ContainsKey hit and miss Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Contains/DictionaryContainsKey.cs | 57 ++++++++++++++++ .../Contains/HashSetContains.cs | 57 ++++++++++++++++ .../Dictionary/DictionaryTryRemove.cs | 46 +++++++++++++ .../System.Collections/Remove/RemoveFalse.cs | 65 ++++++++++++++++++ .../System.Collections/Remove/RemoveTrue.cs | 68 +++++++++++++++++++ 5 files changed, 293 insertions(+) create mode 100644 src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs create mode 100644 src/benchmarks/micro/libraries/System.Collections/Contains/HashSetContains.cs create mode 100644 src/benchmarks/micro/libraries/System.Collections/Dictionary/DictionaryTryRemove.cs create mode 100644 src/benchmarks/micro/libraries/System.Collections/Remove/RemoveFalse.cs create mode 100644 src/benchmarks/micro/libraries/System.Collections/Remove/RemoveTrue.cs diff --git a/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs b/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs new file mode 100644 index 00000000000..e4e81a33fbc --- /dev/null +++ b/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Extensions; +using MicroBenchmarks; + +namespace System.Collections +{ + [BenchmarkCategory(Categories.Libraries, Categories.Collections, Categories.GenericCollections)] + [GenericTypeArguments(typeof(int))] // value type + [GenericTypeArguments(typeof(string))] // reference type + [GenericTypeArguments(typeof(Guid))] // larger value type + public class DictionaryContainsKey + { + private T[] _found; + private T[] _notFound; + private Dictionary _dictionary; + + [Params(Utils.DefaultCollectionSize, 8192)] + public int Size; + + [GlobalSetup] + public void Setup() + { + var allKeys = ValuesGenerator.ArrayOfUniqueValues(Size * 2); + _found = allKeys.Skip(Size).Take(Size).ToArray(); + _notFound = allKeys.Take(Size).ToArray(); + _dictionary = _found.ToDictionary(k => k, k => default(T)!); + } + + [Benchmark] + public bool ContainsKeyTrue() + { + bool result = default; + Dictionary collection = _dictionary; + T[] found = _found; + for (int i = 0; i < found.Length; i++) + result ^= collection.ContainsKey(found[i]); + return result; + } + + [Benchmark] + public bool ContainsKeyFalse() + { + bool result = default; + Dictionary collection = _dictionary; + T[] notFound = _notFound; + for (int i = 0; i < notFound.Length; i++) + result ^= collection.ContainsKey(notFound[i]); + return result; + } + } +} diff --git a/src/benchmarks/micro/libraries/System.Collections/Contains/HashSetContains.cs b/src/benchmarks/micro/libraries/System.Collections/Contains/HashSetContains.cs new file mode 100644 index 00000000000..76e40c72f3f --- /dev/null +++ b/src/benchmarks/micro/libraries/System.Collections/Contains/HashSetContains.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Extensions; +using MicroBenchmarks; + +namespace System.Collections +{ + [BenchmarkCategory(Categories.Libraries, Categories.Collections, Categories.GenericCollections)] + [GenericTypeArguments(typeof(int))] // value type + [GenericTypeArguments(typeof(string))] // reference type + [GenericTypeArguments(typeof(Guid))] // larger value type + public class HashSetContains + { + private T[] _found; + private T[] _notFound; + private HashSet _hashSet; + + [Params(Utils.DefaultCollectionSize, 8192)] + public int Size; + + [GlobalSetup] + public void Setup() + { + var allKeys = ValuesGenerator.ArrayOfUniqueValues(Size * 2); + _found = allKeys.Skip(Size).Take(Size).ToArray(); + _notFound = allKeys.Take(Size).ToArray(); + _hashSet = new HashSet(_found); + } + + [Benchmark] + public bool ContainsTrue() + { + bool result = default; + HashSet collection = _hashSet; + T[] found = _found; + for (int i = 0; i < found.Length; i++) + result ^= collection.Contains(found[i]); + return result; + } + + [Benchmark] + public bool ContainsFalse() + { + bool result = default; + HashSet collection = _hashSet; + T[] notFound = _notFound; + for (int i = 0; i < notFound.Length; i++) + result ^= collection.Contains(notFound[i]); + return result; + } + } +} diff --git a/src/benchmarks/micro/libraries/System.Collections/Dictionary/DictionaryTryRemove.cs b/src/benchmarks/micro/libraries/System.Collections/Dictionary/DictionaryTryRemove.cs new file mode 100644 index 00000000000..4f56da15583 --- /dev/null +++ b/src/benchmarks/micro/libraries/System.Collections/Dictionary/DictionaryTryRemove.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Extensions; +using MicroBenchmarks; + +namespace System.Collections +{ + [BenchmarkCategory(Categories.Libraries, Categories.Collections, Categories.GenericCollections)] + [GenericTypeArguments(typeof(int), typeof(int))] // small value type key+value + [GenericTypeArguments(typeof(string), typeof(string))] // reference type key+value + [GenericTypeArguments(typeof(Guid), typeof(int))] // larger value type key + public class DictionaryTryRemove + { + private Dictionary _dictionary; + private TKey[] _keys; + + [Params(Utils.DefaultCollectionSize, 8192)] + public int Size; + + [GlobalSetup] + public void Setup() + { + _keys = ValuesGenerator.ArrayOfUniqueValues(Size); + _dictionary = _keys.ToDictionary(k => k, k => default(TValue)!); + } + + [Benchmark] + public bool TryRemove_Hit() + { + var dict = _dictionary; + var keys = _keys; + bool result = false; + for (int i = 0; i < keys.Length; i++) + { + result = dict.Remove(keys[i], out _); + dict[keys[i]] = default!; + } + return result; + } + } +} diff --git a/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveFalse.cs b/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveFalse.cs new file mode 100644 index 00000000000..0c592600c55 --- /dev/null +++ b/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveFalse.cs @@ -0,0 +1,65 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Extensions; +using MicroBenchmarks; + +namespace System.Collections +{ + [BenchmarkCategory(Categories.Libraries, Categories.Collections, Categories.GenericCollections)] + [GenericTypeArguments(typeof(int))] // value type + [GenericTypeArguments(typeof(string))] // reference type + [GenericTypeArguments(typeof(Guid))] // larger value type + public class RemoveFalse + { + private T[] _missingKeys; + private HashSet _hashSet; + private Dictionary _dictionary; + + [Params(Utils.DefaultCollectionSize, 8192)] + public int Size; + + private T[] Setup() + { + var allKeys = ValuesGenerator.ArrayOfUniqueValues(Size * 2); + _missingKeys = allKeys.Take(Size).ToArray(); + return allKeys.Skip(Size).Take(Size).ToArray(); + } + + [GlobalSetup(Target = nameof(HashSet))] + public void SetupHashSet() => _hashSet = new HashSet(Setup()); + + [Benchmark] + public bool HashSet() + { + bool result = false; + var collection = _hashSet; + var keys = _missingKeys; + for (int i = 0; i < keys.Length; i++) + result = collection.Remove(keys[i]); + return result; + } + + [GlobalSetup(Target = nameof(Dictionary))] + public void SetupDictionary() + { + var source = Setup(); + _dictionary = source.ToDictionary(k => k, k => default(T)!); + } + + [Benchmark] + public bool Dictionary() + { + bool result = false; + var collection = _dictionary; + var keys = _missingKeys; + for (int i = 0; i < keys.Length; i++) + result = collection.Remove(keys[i]); + return result; + } + } +} diff --git a/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveTrue.cs b/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveTrue.cs new file mode 100644 index 00000000000..3353043575d --- /dev/null +++ b/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveTrue.cs @@ -0,0 +1,68 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Extensions; +using MicroBenchmarks; + +namespace System.Collections +{ + [BenchmarkCategory(Categories.Libraries, Categories.Collections, Categories.GenericCollections)] + [GenericTypeArguments(typeof(int))] // value type + [GenericTypeArguments(typeof(string))] // reference type + [GenericTypeArguments(typeof(Guid))] // larger value type + public class RemoveTrue + { + private T[] _keys; + private HashSet _hashSet; + private Dictionary _dictionary; + + [Params(Utils.DefaultCollectionSize, 8192)] + public int Size; + + [GlobalSetup(Target = nameof(HashSet))] + public void SetupHashSet() + { + _keys = ValuesGenerator.ArrayOfUniqueValues(Size); + _hashSet = new HashSet(_keys); + } + + [Benchmark] + public bool HashSet() + { + bool result = false; + var collection = _hashSet; + var keys = _keys; + for (int i = 0; i < keys.Length; i++) + { + result = collection.Remove(keys[i]); + collection.Add(keys[i]); + } + return result; + } + + [GlobalSetup(Target = nameof(Dictionary))] + public void SetupDictionary() + { + _keys = ValuesGenerator.ArrayOfUniqueValues(Size); + _dictionary = _keys.ToDictionary(k => k, k => default(T)!); + } + + [Benchmark] + public bool Dictionary() + { + bool result = false; + var collection = _dictionary; + var keys = _keys; + for (int i = 0; i < keys.Length; i++) + { + result = collection.Remove(keys[i]); + collection[keys[i]] = default!; + } + return result; + } + } +} From 8c8219a51a56e262fb7fec54bff53902e20abbbb Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Fri, 27 Mar 2026 14:00:42 -0600 Subject: [PATCH 2/5] Add DictionaryContainsKeyLarge: 1M-entry lookup benchmark Probes 512 keys into a 1M-entry dictionary per invocation to measure lookup behavior when the hash table far exceeds CPU cache, while keeping per-call time low enough for stable BDN statistics (~0.5% noise). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Contains/DictionaryContainsKey.cs | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs b/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs index e4e81a33fbc..c508e95017e 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs @@ -54,4 +54,52 @@ public bool ContainsKeyFalse() return result; } } + + /// + /// Measures ContainsKey on a 1M-entry dictionary to capture behavior when + /// the hash table far exceeds CPU cache. Probes only 512 keys per invocation + /// so BDN gets enough iterations for stable statistics. + /// + [BenchmarkCategory(Categories.Libraries, Categories.Collections, Categories.GenericCollections)] + public class DictionaryContainsKeyLarge + { + private const int DictSize = 1_000_000; + private const int ProbeCount = 512; + + private int[] _found; + private int[] _notFound; + private Dictionary _dictionary; + + [GlobalSetup] + public void Setup() + { + var allKeys = ValuesGenerator.ArrayOfUniqueValues(DictSize + ProbeCount); + var inDict = allKeys.Take(DictSize).ToArray(); + _found = inDict.Take(ProbeCount).ToArray(); + _notFound = allKeys.Skip(DictSize).Take(ProbeCount).ToArray(); + _dictionary = inDict.ToDictionary(k => k, k => k); + } + + [Benchmark] + public bool ContainsKeyTrue() + { + bool result = default; + var collection = _dictionary; + var found = _found; + for (int i = 0; i < found.Length; i++) + result ^= collection.ContainsKey(found[i]); + return result; + } + + [Benchmark] + public bool ContainsKeyFalse() + { + bool result = default; + var collection = _dictionary; + var notFound = _notFound; + for (int i = 0; i < notFound.Length; i++) + result ^= collection.ContainsKey(notFound[i]); + return result; + } + } } From 632d971bd22008c48d334090edf3b3879ec24a50 Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Fri, 27 Mar 2026 15:13:16 -0600 Subject: [PATCH 3/5] Increase DictionaryContainsKeyLarge probe count to 8192 512 probes fit in L1 cache, masking the large-table effect. 8192 probes (~512 KB working set) push into L2/L3, giving realistic cache-miss behavior with ~1-2% noise. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../System.Collections/Contains/DictionaryContainsKey.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs b/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs index c508e95017e..ea883068b82 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs @@ -56,15 +56,16 @@ public bool ContainsKeyFalse() } /// - /// Measures ContainsKey on a 1M-entry dictionary to capture behavior when - /// the hash table far exceeds CPU cache. Probes only 512 keys per invocation - /// so BDN gets enough iterations for stable statistics. + /// Measures ContainsKey on a 1M-entry dictionary (~20 MB) to capture behavior + /// when the hash table far exceeds L1/L2 cache. Probes 8192 keys per invocation + /// for realistic cache-miss pressure while keeping per-call time low enough + /// for stable BDN statistics (~1-2% noise). /// [BenchmarkCategory(Categories.Libraries, Categories.Collections, Categories.GenericCollections)] public class DictionaryContainsKeyLarge { private const int DictSize = 1_000_000; - private const int ProbeCount = 512; + private const int ProbeCount = 8192; private int[] _found; private int[] _notFound; From a7667c91c5504c3d0553ca3331ba6da28d7fe015 Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Fri, 27 Mar 2026 19:55:08 -0600 Subject: [PATCH 4/5] Use Add instead of indexer for re-add after Remove After a successful Remove, use Add() instead of the indexer to re-add the entry. Add is semantically clearer (key is known absent), fails loudly if the remove didn't work, and avoids the indexer's extra exists-check overhead that can dilute the Remove signal. Matches the HashSet path which already uses Add. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../System.Collections/Dictionary/DictionaryTryRemove.cs | 2 +- .../micro/libraries/System.Collections/Remove/RemoveTrue.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/benchmarks/micro/libraries/System.Collections/Dictionary/DictionaryTryRemove.cs b/src/benchmarks/micro/libraries/System.Collections/Dictionary/DictionaryTryRemove.cs index 4f56da15583..42c5f33945c 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Dictionary/DictionaryTryRemove.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Dictionary/DictionaryTryRemove.cs @@ -38,7 +38,7 @@ public bool TryRemove_Hit() for (int i = 0; i < keys.Length; i++) { result = dict.Remove(keys[i], out _); - dict[keys[i]] = default!; + dict.Add(keys[i], default!); } return result; } diff --git a/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveTrue.cs b/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveTrue.cs index 3353043575d..48c8f84b26e 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveTrue.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveTrue.cs @@ -60,7 +60,7 @@ public bool Dictionary() for (int i = 0; i < keys.Length; i++) { result = collection.Remove(keys[i]); - collection[keys[i]] = default!; + collection.Add(keys[i], default!); } return result; } From ee555c039a7178f4d5fe43f7d3e9f0053f2d8301 Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Mon, 30 Mar 2026 13:53:56 -0600 Subject: [PATCH 5/5] Remove Size=8192 from new benchmarks to match existing collection benchmark sizes The 8192 size introduced alignment sensitivity that causes unreliable results across runs (up to 47% spread with [MemoryRandomization], and 3x bimodal swings on hybrid-core CPUs like i9-14900K). All existing cross-collection benchmarks use only DefaultCollectionSize (512). DictionaryContainsKeyLarge (1M entries, 8192 probes) is unaffected as it has a different design. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../System.Collections/Contains/DictionaryContainsKey.cs | 2 +- .../libraries/System.Collections/Contains/HashSetContains.cs | 2 +- .../System.Collections/Dictionary/DictionaryTryRemove.cs | 2 +- .../micro/libraries/System.Collections/Remove/RemoveFalse.cs | 2 +- .../micro/libraries/System.Collections/Remove/RemoveTrue.cs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs b/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs index ea883068b82..257035a3773 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Contains/DictionaryContainsKey.cs @@ -20,7 +20,7 @@ public class DictionaryContainsKey private T[] _notFound; private Dictionary _dictionary; - [Params(Utils.DefaultCollectionSize, 8192)] + [Params(Utils.DefaultCollectionSize)] public int Size; [GlobalSetup] diff --git a/src/benchmarks/micro/libraries/System.Collections/Contains/HashSetContains.cs b/src/benchmarks/micro/libraries/System.Collections/Contains/HashSetContains.cs index 76e40c72f3f..1347cb4b801 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Contains/HashSetContains.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Contains/HashSetContains.cs @@ -20,7 +20,7 @@ public class HashSetContains private T[] _notFound; private HashSet _hashSet; - [Params(Utils.DefaultCollectionSize, 8192)] + [Params(Utils.DefaultCollectionSize)] public int Size; [GlobalSetup] diff --git a/src/benchmarks/micro/libraries/System.Collections/Dictionary/DictionaryTryRemove.cs b/src/benchmarks/micro/libraries/System.Collections/Dictionary/DictionaryTryRemove.cs index 42c5f33945c..27461911168 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Dictionary/DictionaryTryRemove.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Dictionary/DictionaryTryRemove.cs @@ -19,7 +19,7 @@ public class DictionaryTryRemove private Dictionary _dictionary; private TKey[] _keys; - [Params(Utils.DefaultCollectionSize, 8192)] + [Params(Utils.DefaultCollectionSize)] public int Size; [GlobalSetup] diff --git a/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveFalse.cs b/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveFalse.cs index 0c592600c55..8f1455b5585 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveFalse.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveFalse.cs @@ -20,7 +20,7 @@ public class RemoveFalse private HashSet _hashSet; private Dictionary _dictionary; - [Params(Utils.DefaultCollectionSize, 8192)] + [Params(Utils.DefaultCollectionSize)] public int Size; private T[] Setup() diff --git a/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveTrue.cs b/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveTrue.cs index 48c8f84b26e..0f5ce48b49a 100644 --- a/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveTrue.cs +++ b/src/benchmarks/micro/libraries/System.Collections/Remove/RemoveTrue.cs @@ -20,7 +20,7 @@ public class RemoveTrue private HashSet _hashSet; private Dictionary _dictionary; - [Params(Utils.DefaultCollectionSize, 8192)] + [Params(Utils.DefaultCollectionSize)] public int Size; [GlobalSetup(Target = nameof(HashSet))]