From 4d4bb51eef921b18025d89f9fcf6ce0037429467 Mon Sep 17 00:00:00 2001
From: ipezygj <231574583+ipezygj@users.noreply.github.com>
Date: Sat, 8 Aug 2026 14:20:50 +0300
Subject: [PATCH] Add hot-run-stability.py: run-to-run spread report from
 published results

The ranking takes the smaller of the 2nd and 3rd runtime per query, but
both hot runs are already stored in every results file - so the timing
noise that decides whether a small gap between adjacent systems is real
can be measured directly from the published data. This script reports,
per results file, the median and p90 relative spread between the two hot
runs and the share of queries where they differ by more than a threshold.

Across the 962 active result files the median per-file hot-run spread is
2.9%, but the tail is heavy: some systems' hot runs differ by 50%+ on
most queries, which is larger than many adjacent-rank gaps on the board.
---
 hot-run-stability.py | 121 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 121 insertions(+)
 create mode 100644 hot-run-stability.py

diff --git a/hot-run-stability.py b/hot-run-stability.py
new file mode 100644
index 0000000000..0ff85b32dd
--- /dev/null
+++ b/hot-run-stability.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+
+"""Report hot-run run-to-run stability of ClickBench results.
+
+The ranking uses the smaller of the 2nd and 3rd runtime per query (best of
+the two hot runs). Both hot runs are already stored in every results file,
+so the run-to-run spread — the timing noise that decides whether a small
+gap between adjacent systems is real — can be measured directly from the
+published data. This script does that:
+
+  per query:   rel_spread = |run2 - run3| / min(run2, run3)
+  per file:    median and p90 of rel_spread, and the share of queries whose
+               hot runs differ by more than a threshold (default 10%)
+
+Usage:
+
+  ./hot-run-stability.py clickhouse/results/20260808/*.json
+  ./hot-run-stability.py --threshold 0.2 duckdb/results/*/*.json
+  ./hot-run-stability.py            # all latest active results
+
+With no arguments, the latest results file per (system, machine) is used,
+mirroring the selection in validate-results.py.
+"""
+
+import argparse
+import json
+import re
+import statistics
+import sys
+from pathlib import Path
+
+DATE_DIR_RE = re.compile(r"^\d{8}$")
+SKIP_SYSTEMS = {"hardware", "versions", "gravitons"}
+
+
+def find_active_result_files(root):
+    latest = {}
+    for path in sorted(root.glob("*/results/*/*.json")):
+        relative_path = path.relative_to(root)
+        if relative_path.parts[0] in SKIP_SYSTEMS:
+            continue
+        system, _, date_dir, filename = relative_path.parts[:4]
+        if not DATE_DIR_RE.match(date_dir):
+            continue
+        key = (system, filename)
+        if key not in latest or date_dir > latest[key].parts[2]:
+            latest[key] = relative_path
+    return [root / p for p in sorted(latest.values())]
+
+
+def hot_run_spreads(result):
+    spreads = []
+    for runs in result:
+        if len(runs) < 3:
+            continue
+        hot = [r for r in (runs[1], runs[2]) if isinstance(r, (int, float)) and r > 0]
+        if len(hot) < 2:
+            continue
+        spreads.append(abs(hot[0] - hot[1]) / min(hot))
+    return spreads
+
+
+def analyze_file(path, threshold):
+    with open(path, encoding="utf-8") as f:
+        data = json.load(f)
+    spreads = hot_run_spreads(data.get("result", []))
+    if not spreads:
+        return None
+    return {
+        "path": str(path),
+        "system": data.get("system", "?"),
+        "machine": data.get("machine", "?"),
+        "queries": len(spreads),
+        "median_spread": statistics.median(spreads),
+        "p90_spread": statistics.quantiles(spreads, n=10)[-1] if len(spreads) >= 2 else spreads[0],
+        "share_over_threshold": sum(s > threshold for s in spreads) / len(spreads),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("files", nargs="*", help="results JSON files (default: latest active results)")
+    parser.add_argument("--threshold", type=float, default=0.1,
+                        help="relative spread above which a query counts as unstable (default 0.1 = 10%%)")
+    args = parser.parse_args()
+
+    root = Path(__file__).parent
+    paths = [Path(f) for f in args.files] if args.files else find_active_result_files(root)
+
+    rows = []
+    for path in paths:
+        try:
+            row = analyze_file(path, args.threshold)
+        except (json.JSONDecodeError, OSError) as e:
+            print(f"skipping {path}: {e}", file=sys.stderr)
+            continue
+        if row:
+            rows.append(row)
+
+    if not rows:
+        print("no analyzable results", file=sys.stderr)
+        return 1
+
+    rows.sort(key=lambda r: r["median_spread"], reverse=True)
+    print(f"{'system':40.40} {'machine':20.20} {'median':>8} {'p90':>8} {'>' + format(args.threshold, '.0%'):>8}")
+    for r in rows:
+        print(
+            f"{r['system']:40.40} {r['machine']:20.20} "
+            f"{r['median_spread']:8.1%} {r['p90_spread']:8.1%} {r['share_over_threshold']:8.0%}"
+        )
+
+    all_spreads = [r["median_spread"] for r in rows]
+    print(
+        f"\n{len(rows)} result files; median of per-file median spreads: "
+        f"{statistics.median(all_spreads):.1%}"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())