Create Descriptive Statistics feature

.github/workflows/benchmarks.yml

@@ -14,15 +14,18 @@ permissions:
 
 env:
   CARGO_TERM_COLOR: always
-  # Ten percent is a useful default for noisy shared CI runners. Adjust this
-  # repository-level value if the project needs a stricter or looser boundary.
-  BENCHER_THRESHOLD: "0.10"
+  # Forty percent keeps normal shared-runner noise from failing the report.
+  # Adjust this repository-level value if a stricter or looser boundary is needed.
+  BENCHER_THRESHOLD: "0.30"
+  # Keep every benchmark target in CI, but omit the largest input size. Local
+  # runs use all sizes unless this variable is set explicitly.
+  BENCHMARK_MAX_INPUT_SIZE: "1000"
   # A relative path is valid from the checkout and is available at workflow
   # parse time, unlike the runner context used by the previous configuration.
   CARGO_TARGET_DIR: target
 
 jobs:
-  benchmark:
+  benchmark_tests:
     name: Test and benchmark
     runs-on: ubuntu-latest
     steps:
@@ -42,6 +45,33 @@ jobs:
       - name: Run functionality tests
         run: cargo test --locked
 
+  benchmark:
+    name: Benchmark (${{ matrix.target }})
+    needs: benchmark_tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        target:
+          - sort
+          - summary_statistics
+          - deviation
+          - core_operations
+    steps:
+      - name: Check out source
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+
+      - name: Cache Rust build artifacts
+        uses: Swatinem/rust-cache@v2
+
       # Bencher Cloud requires a project and API key. Keeping the check in a
       # step makes forked pull requests safely fall back to artifacts because
       # GitHub does not expose repository secrets to them.
@@ -68,6 +98,7 @@ jobs:
         env:
           BENCHER_API_KEY: ${{ secrets.BENCHER_API_KEY }}
           BENCHER_PROJECT: ${{ vars.BENCHER_PROJECT }}
+          BENCHER_CI_ID: ${{ matrix.target }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           set -euo pipefail
@@ -89,6 +120,7 @@ jobs:
             --threshold-upper-boundary "$BENCHER_THRESHOLD"
             --thresholds-reset
             --adapter rust_criterion
+            --ci-id "$BENCHER_CI_ID"
             --github-actions "$GITHUB_TOKEN"
           )
 
@@ -104,7 +136,7 @@ jobs:
 
           # Do not expose the Bencher key to the benchmarked crate.
           env -u BENCHER_API_KEY bencher "${bencher_args[@]}" \
-            'for bench in sort summary_statistics deviation core_operations; do cargo bench --locked --bench "$bench"; done'
+            'cargo bench --locked --bench "$BENCHER_CI_ID"'
 
       - name: Fetch master for local Criterion comparison
         if: steps.bencher.outputs.enabled != 'true' && (github.event_name != 'push' || github.ref_name != 'master')
@@ -117,18 +149,17 @@ jobs:
           set -euo pipefail
 
           master_worktree="$RUNNER_TEMP/ndarray-stats-master"
-          baseline_targets="$RUNNER_TEMP/criterion-baseline-targets"
-          : > "$baseline_targets"
+          baseline_target="$RUNNER_TEMP/criterion-baseline-target"
+          benchmark_target="${{ matrix.target }}"
+          : > "$baseline_target"
           git worktree add --detach "$master_worktree" origin/master
-          for bench in sort summary_statistics deviation core_operations; do
-            if grep -q "name = \"$bench\"" "$master_worktree/Cargo.toml"; then
-              cargo bench --manifest-path "$master_worktree/Cargo.toml" \
-                --locked --bench "$bench" -- --save-baseline master
-              echo "$bench" >> "$baseline_targets"
-            else
-              echo "Skipping $bench: it is not present on master yet."
-            fi
-          done
+          if grep -q "name = \"$benchmark_target\"" "$master_worktree/Cargo.toml"; then
+            cargo bench --manifest-path "$master_worktree/Cargo.toml" \
+              --locked --bench "$benchmark_target" -- --save-baseline master
+            echo "$benchmark_target" >> "$baseline_target"
+          else
+            echo "Skipping $benchmark_target: it is not present on master yet."
+          fi
           git worktree remove --force "$master_worktree"
 
       - name: Run local Criterion comparison
@@ -137,23 +168,20 @@ jobs:
         run: |
           set -euo pipefail
 
+          benchmark_target="${{ matrix.target }}"
           if [[ "$GITHUB_EVENT_NAME" != "push" || "$GITHUB_REF_NAME" != "master" ]]; then
-            for bench in sort summary_statistics deviation core_operations; do
-              if grep -Fxq "$bench" "$RUNNER_TEMP/criterion-baseline-targets"; then
-                cargo bench --locked --bench "$bench" -- --baseline master 2>&1 | \
-                  tee -a "$RUNNER_TEMP/criterion-summary.txt"
-              else
-                echo "No master baseline for $bench; measuring without comparison." | \
-                  tee -a "$RUNNER_TEMP/criterion-summary.txt"
-                cargo bench --locked --bench "$bench" 2>&1 | \
-                  tee -a "$RUNNER_TEMP/criterion-summary.txt"
-              fi
-            done
-          else
-            for bench in sort summary_statistics deviation core_operations; do
-              cargo bench --locked --bench "$bench" 2>&1 | \
+            if grep -Fxq "$benchmark_target" "$RUNNER_TEMP/criterion-baseline-target"; then
+              cargo bench --locked --bench "$benchmark_target" -- --baseline master 2>&1 | \
+                tee -a "$RUNNER_TEMP/criterion-summary.txt"
+            else
+              echo "No master baseline for $benchmark_target; measuring without comparison." | \
                 tee -a "$RUNNER_TEMP/criterion-summary.txt"
-            done
+              cargo bench --locked --bench "$benchmark_target" 2>&1 | \
+                tee -a "$RUNNER_TEMP/criterion-summary.txt"
+            fi
+          else
+            cargo bench --locked --bench "$benchmark_target" 2>&1 | \
+              tee -a "$RUNNER_TEMP/criterion-summary.txt"
           fi
 
       - name: Create custom benchmark summary
@@ -162,6 +190,7 @@ jobs:
         env:
           BENCHER_ENABLED: ${{ steps.bencher.outputs.enabled }}
           BENCHER_PROJECT: ${{ vars.BENCHER_PROJECT }}
+          BENCHER_CI_ID: ${{ matrix.target }}
         run: |
           set -euo pipefail
 
@@ -297,13 +326,14 @@ jobs:
             context_row "Runner" "$RUNNER_OS / ubuntu-latest"
             context_row "Rust" "$rust_version"
             context_row "Features" "default"
-            context_row "Benchmark targets" "sort, summary_statistics, deviation, core_operations"
+            context_row "Benchmark target" "$BENCHER_CI_ID"
             context_row "Input shapes" "1-D n; correlation 3 x n; histogram n x 2"
+            context_row "Input size cap" "n <= $BENCHMARK_MAX_INPUT_SIZE"
             context_row "Backend" "$backend"
             context_row "Comparison" "$baseline_info"
             context_row "Bencher project" "${BENCHER_PROJECT:-not configured}"
             if [[ "$BENCHER_ENABLED" == "true" ]]; then
-              context_row "Threshold" "10% upper boundary"
+              context_row "Threshold" "40% upper boundary"
             else
               context_row "Threshold" "not enforced by artifact fallback"
             fi
@@ -315,10 +345,55 @@ jobs:
         if: always() && steps.bencher.outputs.enabled != 'true'
         uses: actions/upload-artifact@v4
         with:
-          name: criterion-${{ github.run_id }}
+          name: criterion-${{ github.run_id }}-${{ matrix.target }}
           path: |
             ${{ runner.temp }}/criterion-summary.txt
             ${{ runner.temp }}/custom-benchmark-summary.md
             target/criterion
           if-no-files-found: warn
           retention-days: 14
+
+  bencher_report:
+    name: Bencher Report
+    if: always() && github.event_name == 'pull_request'
+    needs: benchmark
+    runs-on: ubuntu-latest
+    steps:
+      # Bencher intentionally marks its GitHub Check as failed when it finds an
+      # alert. Keep those reports visible, but make alerts informational for PRs.
+      - name: Keep Bencher reports non-blocking
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            try {
+              const { data } = await github.rest.checks.listForRef({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                ref: context.sha,
+                per_page: 100,
+              });
+              const bencherReports = data.check_runs.filter(({ name }) =>
+                name.startsWith('Bencher Report (')
+              );
+
+              for (const report of bencherReports) {
+                try {
+                  await github.rest.checks.update({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                    check_run_id: report.id,
+                    conclusion: 'neutral',
+                  });
+                  core.info(`${report.name} is informational for this pull request.`);
+                } catch (error) {
+                  core.warning(`Could not update ${report.name}: ${error.message}`);
+                }
+              }
+
+              if (bencherReports.length === 0) {
+                core.info(`No external Bencher report checks found on ${context.sha}.`);
+              }
+            } catch (error) {
+              core.warning(`Could not update Bencher report checks: ${error.message}`);
+            }

benches/common/mod.rs

@@ -0,0 +1,22 @@
+use std::env;
+
+/// Return the benchmark input sizes, optionally applying the CI size cap.
+pub(crate) fn benchmark_lengths() -> Vec<usize> {
+    const DEFAULT_LENGTHS: [usize; 4] = [10, 100, 1_000, 10_000];
+
+    if let Ok(max_length) = env::var("BENCHMARK_MAX_INPUT_SIZE") {
+        if let Ok(max_length) = max_length.parse::<usize>() {
+            let lengths: Vec<_> = DEFAULT_LENGTHS
+                .iter()
+                .copied()
+                .filter(|length| *length <= max_length)
+                .collect();
+
+            if !lengths.is_empty() {
+                return lengths;
+            }
+        }
+    }
+
+    DEFAULT_LENGTHS.to_vec()
+}

benches/core_operations.rs

@@ -8,8 +8,10 @@ use ndarray_stats::histogram::{strategies::Auto, GridBuilder, HistogramExt};
 use ndarray_stats::{interpolate::Linear, CorrelationExt, DeviationExt, EntropyExt, Quantile1dExt};
 use noisy_float::types::n64;
 
+mod common;
+
 fn mean(c: &mut Criterion) {
-    let lens = vec![10, 100, 1000, 10000];
+    let lens = common::benchmark_lengths();
     let mut group = c.benchmark_group("mean");
     group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
     for len in &lens {
@@ -23,7 +25,7 @@ fn mean(c: &mut Criterion) {
 }
 
 fn quantiles_mut(c: &mut Criterion) {
-    let lens = vec![10, 100, 1000, 10000];
+    let lens = common::benchmark_lengths();
     let quantile_indexes = Array1::from_vec(vec![n64(0.25), n64(0.5), n64(0.75)]);
     let mut group = c.benchmark_group("quantiles_mut");
     group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
@@ -46,7 +48,7 @@ fn quantiles_mut(c: &mut Criterion) {
 }
 
 fn pearson_correlation(c: &mut Criterion) {
-    let lens = vec![10, 100, 1000, 10000];
+    let lens = common::benchmark_lengths();
     let mut group = c.benchmark_group("pearson_correlation");
     group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
     for len in &lens {
@@ -59,7 +61,7 @@ fn pearson_correlation(c: &mut Criterion) {
 }
 
 fn spearman_correlation(c: &mut Criterion) {
-    let lens = vec![10, 100, 1000, 10000];
+    let lens = common::benchmark_lengths();
     let mut group = c.benchmark_group("spearman_correlation");
     group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
     for len in &lens {
@@ -74,7 +76,7 @@ fn spearman_correlation(c: &mut Criterion) {
 }
 
 fn kendall_tau(c: &mut Criterion) {
-    let lens = vec![10, 100, 1000, 10000];
+    let lens = common::benchmark_lengths();
     let mut group = c.benchmark_group("kendall_tau");
     group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
     for len in &lens {
@@ -89,7 +91,7 @@ fn kendall_tau(c: &mut Criterion) {
 }
 
 fn entropy(c: &mut Criterion) {
-    let lens = vec![10, 100, 1000, 10000];
+    let lens = common::benchmark_lengths();
     let mut group = c.benchmark_group("entropy");
     group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
     for len in &lens {
@@ -103,7 +105,7 @@ fn entropy(c: &mut Criterion) {
 }
 
 fn histogram(c: &mut Criterion) {
-    let lens = vec![10, 100, 1000, 10000];
+    let lens = common::benchmark_lengths();
     let mut group = c.benchmark_group("histogram");
     group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
     for len in &lens {
@@ -123,7 +125,7 @@ fn histogram(c: &mut Criterion) {
 }
 
 fn l1_dist(c: &mut Criterion) {
-    let lens = vec![10, 100, 1000, 10000];
+    let lens = common::benchmark_lengths();
     let mut group = c.benchmark_group("l1_dist");
     group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
     for len in &lens {

benches/deviation.rs

@@ -6,8 +6,10 @@ use ndarray_rand::rand_distr::Uniform;
 use ndarray_rand::RandomExt;
 use ndarray_stats::DeviationExt;
 
+mod common;
+
 fn sq_l2_dist(c: &mut Criterion) {
-    let lens = vec![10, 100, 1000, 10000];
+    let lens = common::benchmark_lengths();
     let mut group = c.benchmark_group("sq_l2_dist");
     group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
     for len in &lens {

benches/sort.rs

@@ -5,8 +5,10 @@ use ndarray::prelude::*;
 use ndarray_stats::Sort1dExt;
 use rand::prelude::*;
 
+mod common;
+
 fn get_from_sorted_mut(c: &mut Criterion) {
-    let lens = vec![10, 100, 1000, 10000];
+    let lens = common::benchmark_lengths();
     let mut group = c.benchmark_group("get_from_sorted_mut");
     group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
     for len in &lens {
@@ -30,7 +32,7 @@ fn get_from_sorted_mut(c: &mut Criterion) {
 }
 
 fn get_many_from_sorted_mut(c: &mut Criterion) {
-    let lens = vec![10, 100, 1000, 10000];
+    let lens = common::benchmark_lengths();
     let mut group = c.benchmark_group("get_many_from_sorted_mut");
     group.plot_config(PlotConfiguration::default().summary_scale(AxisScale::Logarithmic));
     for len in &lens {