--shuffle, --shuffle-seed

pyperformance/cli.py

@@ -148,6 +148,17 @@ def parse_args():
         default=None,
         help="number of skipped values per run used to warmup the benchmark",
     )
+    cmd.add_argument(
+        "--shuffle",
+        action="store_true",
+        help="Run selected benchmarks in random order",
+    )
+    cmd.add_argument(
+        "--shuffle-seed",
+        type=int,
+        default=None,
+        help="Seed used when shuffling benchmarks (implies --shuffle)",
+    )
     filter_opts(cmd)
 
     # show

pyperformance/run.py

@@ -1,6 +1,7 @@
 import hashlib
 import json
 import os
+import random
 import sys
 import time
 import traceback
@@ -67,13 +68,26 @@ def get_loops_from_file(filename):
     return loops
 
 
+def order_benchmarks(benchmarks, *, shuffle=False, shuffle_seed=None):
+    seq = list(benchmarks)
+    if shuffle or shuffle_seed is not None:
+        rng = random.Random(shuffle_seed)
+        rng.shuffle(seq)
+        return seq
+    return sorted(seq)
+
+
 def run_benchmarks(should_run, python, options):
     if options.same_loops is not None:
         loops = get_loops_from_file(options.same_loops)
     else:
         loops = {}
 
-    to_run = sorted(should_run)
+    to_run = order_benchmarks(
+        should_run,
+        shuffle=getattr(options, "shuffle", False),
+        shuffle_seed=getattr(options, "shuffle_seed", None),
+    )
 
     info = _pythoninfo.get_info(python)
     runid = get_run_id(info)

pyperformance/tests/test_run_order.py

@@ -0,0 +1,34 @@
+import unittest
+
+from pyperformance.run import order_benchmarks
+
+
+class OrderBenchmarksTests(unittest.TestCase):
+    def setUp(self):
+        self.benchmarks = ["b", "a", "c"]
+
+    def test_default_is_sorted(self):
+        ordered = order_benchmarks(self.benchmarks)
+        self.assertEqual(ordered, ["a", "b", "c"])
+
+    def test_shuffle_seed_implies_shuffle(self):
+        ordered = order_benchmarks(self.benchmarks, shuffle_seed=123)
+        self.assertEqual(ordered, ["c", "a", "b"])
+
+    def test_shuffle_flag_without_seed_changes_order(self):
+        sorted_expected = sorted(self.benchmarks)
+
+        shuffled_orders = []
+        for seed in range(10):
+            ordered = order_benchmarks(self.benchmarks, shuffle=True, shuffle_seed=seed)
+            self.assertCountEqual(ordered, self.benchmarks)
+            shuffled_orders.append(tuple(ordered))
+
+        self.assertTrue(
+            any(order != tuple(sorted_expected) for order in shuffled_orders),
+            "Shuffle should change order for at least one seed in range(10)",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()