From a78f33bdfc3e4ffe28667a79a5b67e550d021da2 Mon Sep 17 00:00:00 2001 From: fishidaho Date: Thu, 17 Sep 2026 20:44:10 -0700 Subject: [PATCH] Loosen the vs-scipy timing ceilings past CI runner noise `matvec_vs_scipy` and `matmat_vs_scipy` were gated at a ratio of 1.0 -- at least as fast as scipy -- from a measurement of ~0.25 taken on a workstation. Shared GitHub runners measure 1.09 to 1.35 for the same commit, so the gate failed on changes that cannot affect throughput, including a docstring-only one. Re-running an unchanged commit turned a failure into a pass. Ceilings to 2.5 and the margin to 10.0, which clears the worst observed runner figure by ~85% while still catching a 2x regression. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Rvu3cf8ZL7F5EPX22eo6Je --- benchmarks/README.md | 5 +++-- benchmarks/baselines.json | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 66669e1..192e67c 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -25,8 +25,9 @@ building its input. **Throughput relative to scipy**, never absolute seconds. The same work is timed through scipy in the same process and the ratio recorded, which cancels -most of the difference between machines. Still the noisiest metric, so its -gate is much looser. +most of the difference between machines. It does not cancel all of it -- a +shared CI runner has measured 5x what a workstation does on the same commit -- +so this is much the noisiest metric and its gate is correspondingly loose. Each case runs in its own subprocess, since measurement state and JIT warm-up leak between them otherwise. diff --git a/benchmarks/baselines.json b/benchmarks/baselines.json index 251c274..3cf556b 100644 --- a/benchmarks/baselines.json +++ b/benchmarks/baselines.json @@ -5,7 +5,7 @@ "indices_bytes_per_nonzero": 1.1, "vs_scipy_ratio": 1.1, "peak_alloc_mb": 2.0, - "time_ratio_vs_scipy": 4.0, + "time_ratio_vs_scipy": 10.0, "peak_alloc_mb_view": 2.0, "time_ratio_view_over_materialize": 4.0, "cpu_ratio_1t_cp10k_log1p": 1.5, @@ -27,10 +27,10 @@ "peak_alloc_mb": 407.2383 }, "matvec_vs_scipy": { - "time_ratio_vs_scipy": 1.0 + "time_ratio_vs_scipy": 2.5 }, "matmat_vs_scipy": { - "time_ratio_vs_scipy": 1.0 + "time_ratio_vs_scipy": 2.5 }, "minor_extrema_peak_mb": { "peak_alloc_mb": 132.5453