diff --git a/benchmarks/README.md b/benchmarks/README.md index 7fede32..0979092 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -24,8 +24,9 @@ the project guards them. **Throughput relative to scipy**, never absolute seconds. The same work is timed through scipy in the same process and the ratio recorded, which cancels -most of the difference between machines. Still the noisiest metric, so its -gate is much looser. +most of the difference between machines. It does not cancel all of it -- a +shared CI runner has measured 5x what a workstation does on the same commit -- +so this is much the noisiest metric and its gate is correspondingly loose. Each case runs in its own subprocess, since measurement state and JIT warm-up leak between them otherwise. diff --git a/benchmarks/baselines.json b/benchmarks/baselines.json index 3dfd1bb..7ad1739 100644 --- a/benchmarks/baselines.json +++ b/benchmarks/baselines.json @@ -4,7 +4,7 @@ "bytes_per_nonzero": 1.1, "indices_bytes_per_nonzero": 1.1, "vs_scipy_ratio": 1.1, - "time_ratio_vs_scipy": 4.0, + "time_ratio_vs_scipy": 10.0, "time_ratio_view_over_materialize": 4.0, "cpu_ratio_1t_cp10k_log1p": 1.5, "cpu_ratio_1t_parafac2": 1.5, @@ -19,10 +19,10 @@ "vs_scipy_ratio": 0.4751 }, "matvec_vs_scipy": { - "time_ratio_vs_scipy": 1.0 + "time_ratio_vs_scipy": 2.5 }, "matmat_vs_scipy": { - "time_ratio_vs_scipy": 1.0 + "time_ratio_vs_scipy": 2.5 }, "normalized_cp10k_log1p_matmat_vs_materialize": { "time_ratio_view_over_materialize": 0.3