From 7423bdeb0e2e2997844537aeb4fac8faafa05358 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 18 Jul 2025 17:26:55 +0200 Subject: [PATCH 01/13] fix rocksdb build on fedora bz backporting an upstream patch --- build-bench-env.sh | 3 +++ patches/rocksdb_build.patch | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 patches/rocksdb_build.patch diff --git a/build-bench-env.sh b/build-bench-env.sh index 31f05de8..327badf7 100755 --- a/build-bench-env.sh +++ b/build-bench-env.sh @@ -763,6 +763,9 @@ if test "$setup_rocksdb" = "1"; then fi cd "rocksdb-$version_rocksdb" + set +e + patch -p1 -N -r- < ../../patches/rocksdb_build.patch > /dev/null + set -e DISABLE_WARNING_AS_ERROR=1 DISABLE_JEMALLOC=1 ROCKSDB_DISABLE_TCMALLOC=1 make db_bench -j $procs [ "$CI" ] && find . -name '*.o' -delete popd diff --git a/patches/rocksdb_build.patch b/patches/rocksdb_build.patch new file mode 100644 index 00000000..138708cb --- /dev/null +++ b/patches/rocksdb_build.patch @@ -0,0 +1,13 @@ +# backported from https://github.com/facebook/rocksdb/commit/a4f2d46ab6065eb43ee781b0d57c29787780d935 +diff --git a/db/blob/blob_file_meta.h b/db/blob/blob_file_meta.h +index d7c8a124336..2e47726f8d1 100644 +--- a/db/blob/blob_file_meta.h ++++ b/db/blob/blob_file_meta.h +@@ -6,6 +6,7 @@ + #pragma once + + #include ++#include + #include + #include + #include From 5b538fffa2b69f5e70c7c42a3aec7f31ad4b4d75 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 18 Jul 2025 17:51:41 +0200 Subject: [PATCH 02/13] fix another instance of missing cstdint --- patches/rocksdb_build.patch | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/patches/rocksdb_build.patch b/patches/rocksdb_build.patch index 138708cb..4002f7c8 100644 --- a/patches/rocksdb_build.patch +++ b/patches/rocksdb_build.patch @@ -11,3 +11,18 @@ index d7c8a124336..2e47726f8d1 100644 #include #include #include + +# backported from https://github.com/facebook/rocksdb/commit/e812a3d3b32ac159f2df99bcae4ca55173e790d5 +diff --git a/include/rocksdb/trace_record.h b/include/rocksdb/trace_record.h +index 8f9c3ee2f0f..d0f1532aa3a 100644 +--- a/include/rocksdb/trace_record.h ++++ b/include/rocksdb/trace_record.h +@@ -8,7 +8,7 @@ + #include + #include + #include +- ++#include + #include "rocksdb/rocksdb_namespace.h" + #include "rocksdb/slice.h" + #include "rocksdb/status.h" From 5b7b23e4e4bd9045923fcf855ec506694917b9ce Mon Sep 17 00:00:00 2001 From: derSteFfi Date: Sat, 19 Jul 2025 10:18:51 +0200 Subject: [PATCH 03/13] Fix build on fedora (#241) * fix rocksdb build on fedora bz backporting an upstream patch * fix another instance of missing cstdint * barnes: fix build on fedora --------- Co-authored-by: Stephan Bauroth --- bench/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt index 2f6c6d70..f9c95e3f 100644 --- a/bench/CMakeLists.txt +++ b/bench/CMakeLists.txt @@ -55,6 +55,7 @@ target_compile_options(espresso PRIVATE $<$:-std=gnu89>) target_link_libraries(espresso m) add_executable(barnes ${barnes_sources}) +target_compile_options(barnes PRIVATE $<$:-std=gnu89>) target_link_libraries(barnes m) add_executable(larson larson/larson.cpp) From 7577c4c1749efccb8523a6e7bf2ea449cc9134c6 Mon Sep 17 00:00:00 2001 From: derSteFfi Date: Wed, 23 Jul 2025 21:25:15 +0200 Subject: [PATCH 04/13] Add compiling linux as a benchmark Add building linux as a benchmark, build in RAM to be slightly faster Co-authored-by: Stephan Bauroth --- bench.sh | 15 ++++++++++++++- build-bench-env.sh | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/bench.sh b/bench.sh index 6708f798..c03fe2de 100755 --- a/bench.sh +++ b/bench.sh @@ -14,7 +14,7 @@ alloc_libs="sys=" # mapping from allocator to its .so as "= pushd "$leanmldir" run_test_cmd "mathlib" "$leandir/bin/leanpkg build" popd;; + linux) + builddir="/tmp/linux_build" + pushd "$linux_dir" + run_test_cmd "linux" "make O=$builddir -j $procs" + popd;; redis) # https://redis.io/docs/reference/optimization/benchmarks/ redis_tail="1" diff --git a/build-bench-env.sh b/build-bench-env.sh index 327badf7..2f4f026b 100755 --- a/build-bench-env.sh +++ b/build-bench-env.sh @@ -70,6 +70,7 @@ readonly version_redis=6.2.7 readonly version_lean=21d264a66d53b0a910178ae7d9529cb5886a39b6 # build fix for recent compilers readonly version_rocksdb=8.1.1 readonly version_lua=v5.4.7 +readonly version_linux=6.5.1 # HTTP-downloaded files checksums readonly sha256sum_sh6bench="506354d66b9eebef105d757e055bc55e8d4aea1e7b51faab3da35b0466c923a1" @@ -109,6 +110,7 @@ setup_bench=0 setup_lean=0 setup_redis=0 setup_rocksdb=0 +setup_linux=0 # various setup_packages=0 @@ -168,6 +170,7 @@ while : ; do setup_lean=$flag_arg setup_redis=$flag_arg setup_rocksdb=$flag_arg + setup_linux=$flag_arg setup_bench=$flag_arg setup_packages=$flag_arg ;; @@ -215,6 +218,8 @@ while : ; do setup_redis=$flag_arg;; rocksdb) setup_rocksdb=$flag_arg;; + linux) + setup_linux=$flag_arg;; rp) setup_rp=$flag_arg;; sc) @@ -281,6 +286,7 @@ while : ; do echo " packages setup required packages" echo " redis setup redis benchmark" echo " rocksdb setup rocksdb benchmark" + echo " linux setup linux benchmark" echo "" echo "Prefix an option with 'no-' to disable an option" exit 0;; @@ -851,6 +857,18 @@ if test "$setup_bench" = "1"; then cmake --build out/bench --parallel $procs fi +if test "$setup_linux" = "1"; then + phase "fetch linux" + pushd "$devdir" + if test -d "linux-$version_linux"; then + echo "$devdir/linux-$version_linux already exists; no need to download it" + else + wget --no-verbose "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-$version_linux.tar.xz" + tar xf "linux-$version_linux.tar.xz" + rm "./linux-$version_linux.tar.xz" + fi + popd +fi curdir=`pwd` From 5634fe9fa770bb25c7c4e74d9ba94ef3e2654ced Mon Sep 17 00:00:00 2001 From: jvoisin Date: Tue, 5 Aug 2025 14:11:11 +0200 Subject: [PATCH 05/13] bump snmalloc from 0.7.1 to 0.7.2 --- build-bench-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-bench-env.sh b/build-bench-env.sh index 2f4f026b..92665c5f 100755 --- a/build-bench-env.sh +++ b/build-bench-env.sh @@ -59,7 +59,7 @@ readonly version_sc=master # unmaintained since 2016 readonly version_scudo=main readonly version_sg=master # ~unmaintained since 2021 readonly version_sm=master # ~unmaintained since 2017 -readonly version_sn=0.7.1 +readonly version_sn=0.7.2 readonly version_tbb=v2021.9.0 readonly version_tc=gperftools-2.16.90 readonly version_tcg=98fd24303c7b5ef5e30da625f11fb623a5e038b6 # 2025-07-18 From bc2f1b0cab348fcfe5d5d5aa1d8bfa0e632a6f53 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Tue, 5 Aug 2025 16:53:58 +0100 Subject: [PATCH 06/13] Very minor fixes to Ubuntu install --- build-bench-env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-bench-env.sh b/build-bench-env.sh index 92665c5f..690975b6 100755 --- a/build-bench-env.sh +++ b/build-bench-env.sh @@ -379,7 +379,7 @@ function aptinstall { echo "" echo "> $SUDO apt install $1" echo "" - $SUDO apt install --no-install-recommends $1 + $SUDO apt install -y --no-install-recommends $1 } function dnfinstall { @@ -447,7 +447,7 @@ if test "$setup_packages" = "1"; then elif grep -q -e 'ID=debian' -e 'ID=ubuntu' /etc/os-release 2>/dev/null; then echo "updating package database... ($SUDO apt update)" $SUDO apt update -qq - aptinstall "g++ clang lld llvm-dev unzip dos2unix linuxinfo bc libgmp-dev wget \ + aptinstall "build-essential git gpg g++ clang lld llvm-dev unzip dos2unix linuxinfo bc libgmp-dev wget \ cmake python3 ruby ninja-build libtool autoconf sed ghostscript time \ curl automake libatomic1 libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev \ liblz4-dev libzstd-dev libreadline-dev pkg-config gawk util-linux" From 4354fe9df80ab9d5b11d5ce1362f76a7bbe5d991 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 8 Aug 2025 21:16:17 +0100 Subject: [PATCH 07/13] Cache some of the build using Docker (#244) Rewrite CI pipeline to use Docker. This enables the base image of the benchmarks to be built first, and then each allocator is tested inside that base image. This enables considerable parallelism without much duplication. --- .dockerignore | 4 ++ .github/workflows/all.yml | 124 ++++++--------------------------- .github/workflows/platform.yml | 100 ++++++++++++++++++++++++++ Dockerfile | 44 ++++++++++++ build-bench-env.sh | 5 +- 5 files changed, 171 insertions(+), 106 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/platform.yml create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..4717ae1d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.github +.git +Dockerfile +README.md \ No newline at end of file diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml index e89ce102..930eee13 100644 --- a/.github/workflows/all.yml +++ b/.github/workflows/all.yml @@ -1,111 +1,27 @@ name: Build and run everything + +# The following should ensure that the workflow only runs a single set of actions +# for each PR. But it will not apply this to pushes to the main branch. +concurrency: + group: ${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + on: push: pull_request: schedule: - cron: '0 0 * * 1' + jobs: - build-all-alpine: - runs-on: ubuntu-latest - container: - image: alpine:latest - steps: - - name: Check out repository code - uses: actions/checkout@v3 - - name: Install bash - run: apk add bash git - - name: Silence some git warnings - run: | - git config --global advice.detachedHead false - git config --global init.defaultBranch main - - name: Install and build all benchmarks and allocators - # dh: glibc-specific - # fg: Uses execinfo.h, which is a GNU extension - # gd: ? - # hd: glibc-specific - # lf: crashes redis server - # lt: return type 'struct mallinfo' is incomplete - # mesh/nomesh: infinite loop? - # pa: can't setup depot_tools and goma - # sm: ../src/supermalloc.h:10:31: error: expected initializer before '__THROW' - # tcg: [...] specifies less restrictive attribute than its target [...] - # lp: /__w/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c:218:22: error: call to undeclared function 'pthread_getname_np'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] - # rp: rpmalloc/rpmalloc.c:980:9: error: unsafe pointer arithmetic [-Werror,-Wunsafe-buffer-usage] - run: ./build-bench-env.sh all no-dh no-hd no-sm no-mesh no-nomesh no-pa no-gd no-fg no-lf no-lt no-tcg no-lp no-rp - - name: Run everything. - run: | - cd out/bench - ../../bench.sh alla allt - build-all-ubuntu: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@v3 - - name: Silence some git warnings - run: | - git config --global advice.detachedHead false - git config --global init.defaultBranch main - - name: Install and build all benchmarks and allocators - # fg: crashes on redis - # gd: infinite loop in the redis benchmark - # lt: breaks on sh8benchN - # lf: crashes redis server - # ff: crashes on modern ubuntu: https://github.com/bwickman97/ffmalloc/issues/5 - # hoard: crashes on rocksdb - # pa: python3: can't open file '/__w/mimalloc-bench/mimalloc-bench/extern/pa/partition_alloc_builder/tools/rust/update_rust.py': [Errno 2] No such file or directory - # sn: /usr/bin/../lib/gcc/x86_64-linux-gnu/13/../../../../include/c++/13/chrono:2320:48: error: call to consteval function 'std::chrono::hh_mm_ss::_S_fractional_width' is not a constant expression - # lp: /home/runner/work/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/hotbit_heap_inlines.h:64:51: error: too few arguments to function call, expected 3, have 2 - # tcg: ERROR: Error computing the main repository mapping: protobuf@29.0 depends on rules_fuzzing@0.5.2 with compatibility level 0, but depends on rules_fuzzing@0.5.1 with compatibility level 1 which is different - run: ./build-bench-env.sh all no-lean no-gd no-ff no-fg no-lt no-lf no-hd no-pa no-sn no-lp no-tcg - - name: Run everything. - run: | - cd out/bench - ../../bench.sh alla allt - build-all-fedora: - runs-on: ubuntu-latest - container: - image: fedora:latest - steps: - - name: Check out repository code - uses: actions/checkout@v3 - - name: Install git - run: sudo dnf -y --quiet --nodocs install git - - name: Silence some git warnings - run: | - git config --global advice.detachedHead false - git config --global init.defaultBranch main - - name: Install and build all benchmarks and allocators - # gd: infinite loop in the redis benchmark - # mesh/nomesh: error: '__malloc_hook' was not declared in this scope; - # mi: error: '__malloc_hook' was not declared in this scope; - # rp: mixing declarations and code is incompatible with standards before C99 - # lf: crashes redis server - # fg: crashes redis server - # pa: python3: can't open file '/__w/mimalloc-bench/mimalloc-bench/extern/pa/partition_alloc_builder/tools/rust/update_rust.py': [Errno 2] No such file or directory - # lp: /__w/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c:218:22: error: call to undeclared function 'pthread_getname_np'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] - # tcg: https://github.com/bazelbuild/bazel/issues/19295 bazel5 is required but unavailable - run: ./build-bench-env.sh all no-lean no-mi no-mesh no-nomesh no-gd no-rp no-lf no-fg no-pa no-lp no-sh8 no-tcg - - name: Run everything. - run: | - cd out/bench - ../../bench.sh alla allt - build-all-osx: - runs-on: macos-11 - steps: - - name: Check out repository code - uses: actions/checkout@v3 - - name: Silence some git warnings - run: | - git config --global advice.detachedHead false - git config --global init.defaultBranch main - - name: Install and build all benchmarks and allocators - # ff:ffmalloc.c:1140:14: error: implicit declaration of function 'sched_getcpu' is invalid in C99 [-Werror,-Wimplicit-function-declaration] - # fg: unknown type name 'pthread_spinlock_t'; did you mean 'pthread_rwlock_t'? - # gd: so many errors - # pa: ninja: error: '../../buildtools/third_party/libc++/trunk/src/utility.cpp', needed by 'obj/buildtools/third_party/libc++/libc++/utility.o', missing and no known rule to make it - # lp: /home/runner/work/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/hotbit_heap_inlines.h:64:51: error: too few arguments to function call, expected 3, have 2 - run: ./build-bench-env.sh all no-lean no-gd no-ff no-fg no-pa no-lp - - name: Run everything. - run: | - cd out/bench - ../../bench.sh alla allt + build-and-run: + strategy: + matrix: + platform: [ubuntu, alpine, fedora] + exclude: + # Fedora is not currently building lean. + - platform: fedora + fail-fast: false + uses: ./.github/workflows/platform.yml + name: ${{ matrix.platform }} + with: + platform: ${{ matrix.platform }} \ No newline at end of file diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml new file mode 100644 index 00000000..4463bece --- /dev/null +++ b/.github/workflows/platform.yml @@ -0,0 +1,100 @@ +name: Build and run everything +on: + workflow_call: + inputs: + platform: + description: 'Base container image to build on' + required: true + type: string + +jobs: + build-base-with-docker: + runs-on: ubuntu-latest + name: Base Container + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build benchmarks in Docker + id: docker_build + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile + push: false + tags: bench + target: bench-env + build-args: | + platform=${{ inputs.platform }} + cache-from: type=gha,scope=buildkit_${{ inputs.platform }} + cache-to: type=gha,mode=max,scope=buildkit_${{ inputs.platform }} + + build-and-benchmark-allocator: + runs-on: ubuntu-latest + needs: build-base-with-docker + strategy: + matrix: + allocator: [ff, fg, gd, hd, hm, iso, je, lf, lp, lt, mesh, mng, mi, mi2, nomesh, pa, rp, scudo, sm, sn, sg, tbb, tc, yal, tcg] + testsuite: [allt] + # Used to provide a platform-specific predicate + predicate: [true, false] + exclude: + # Always exclude build when predicate is false. + - predicate: false + + # Not currently working on any platform + - allocator: pa + + # Alpine Specific disables. + # dh: glibc-specific + - allocator: dh + predicate: ${{ inputs.platform == 'alpine' }} + # fg: Uses execinfo.h, which is a GNU extension + - allocator: fg + predicate: ${{ inputs.platform == 'alpine' }} + # gd: ? + - allocator: gd + predicate: ${{ inputs.platform == 'alpine' }} + # hd: glibc-specific + - allocator: hd + predicate: ${{ inputs.platform == 'alpine' }} + # lp: /__w/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c:218:22: error: call to undeclared function 'pthread_getname_np'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] + - allocator: lp + predicate: ${{ inputs.platform == 'alpine' }} + # lt: return type 'struct mallinfo' is incomplete + - allocator: lt + predicate: ${{ inputs.platform == 'alpine' }} + # sm: ../src/supermalloc.h:10:31: error: expected initializer before '__THROW' + - allocator: sm + predicate: ${{ inputs.platform == 'alpine' }} + # tcg: [...] specifies less restrictive attribute than its target [...] + - allocator: tcg + predicate: ${{ inputs.platform == 'alpine' }} + fail-fast: false + name: ${{ matrix.allocator }} - build and benchmark + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build allocator and run benchmarks in Docker + id: docker_build + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile + push: false + target: benchmark + ## If allocators in lt,lf,fg,gd then use a smaller set of tests. + ## They were generally failing on lean or redis, and lf was also failing on mstress. + build-args: | + platform=${{ inputs.platform }} + allocator=${{ matrix.allocator }} + benchs=${{ (contains('lt,lf,gd', matrix.allocator) && 'cfrac espresso barnes larson-sized mstress rptest gs lua') || ('fg' == matrix.allocator && 'cfrac espresso barnes gs lua') || 'allt' }} + repeats=1 + cache-from: type=gha,scope=buildkit_${{ inputs.platform }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..fe54d991 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,44 @@ +ARG platform=ubuntu + +FROM ubuntu:24.04 AS ubuntu + + +FROM fedora:latest AS fedora + + +FROM alpine:latest AS alpine +RUN apk add --no-cache bash + + +FROM ${platform} AS bench-env + +# Pull mimalloc-bench +RUN mkdir -p /mimalloc-bench +COPY . /mimalloc-bench + +WORKDIR /mimalloc-bench +# Install dependencies +RUN ./build-bench-env.sh packages + +# Build benchmarks +RUN ./build-bench-env.sh bench + +RUN ./build-bench-env.sh redis + +RUN ./build-bench-env.sh rocksdb + +RUN ./build-bench-env.sh lean + +FROM bench-env AS benchmark + +WORKDIR /mimalloc-bench + +ARG allocator=mi +ARG benchs=cfrac +ARG repeats=1 + +RUN ./build-bench-env.sh $allocator + +# Run benchmarks +WORKDIR /mimalloc-bench/out/bench +RUN ../../bench.sh $allocator $benchs -r=$repeats \ No newline at end of file diff --git a/build-bench-env.sh b/build-bench-env.sh index 690975b6..4cd9fed1 100755 --- a/build-bench-env.sh +++ b/build-bench-env.sh @@ -441,7 +441,7 @@ if test "$setup_packages" = "1"; then # no 'apt update' equivalent needed on Fedora dnfinstall "gcc-c++ clang lld llvm-devel unzip dos2unix bc gmp-devel wget gawk \ cmake python3 ruby ninja-build libtool autoconf git patch time sed \ - ghostscript libatomic which gflags-devel xz readline-devel snappy-devel" + ghostscript libatomic libstdc++ which gflags-devel xz readline-devel snappy-devel" # bazel5 is broken on the copr: https://github.com/bazelbuild/bazel/issues/19295 #dnfinstallbazel elif grep -q -e 'ID=debian' -e 'ID=ubuntu' /etc/os-release 2>/dev/null; then @@ -656,7 +656,8 @@ if test "$setup_sn" = "1"; then else mkdir -p release cd release - env CXX=clang++ cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Release + # CXX11_DESTRUCTORS is needed as broken on Alpine without, should be fixed in the future upstrem. + env CXX=clang++ cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Release -DSNMALLOC_CLEANUP=CXX11_DESTRUCTORS cd .. fi cd release From 5a816098c5209838fcacc36364edede7c94d89d5 Mon Sep 17 00:00:00 2001 From: Matthew Parkinson Date: Fri, 8 Aug 2025 21:16:50 +0100 Subject: [PATCH 08/13] Add script to talk to bencher.dev. (#242) --- scripts/bencher.dev.py | 109 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 scripts/bencher.dev.py diff --git a/scripts/bencher.dev.py b/scripts/bencher.dev.py new file mode 100644 index 00000000..9bd43e19 --- /dev/null +++ b/scripts/bencher.dev.py @@ -0,0 +1,109 @@ +# This script converts the benchmark outputs to the format required by bencher.dev. +# It creates a file for each allocator with the results in JSON format. +# It requires numpy for statistical calculations. +# It generates file names of the form bencher.dev..json. +# Output files will contain the mean, high, and low values for memory and time for each benchmark: +# { +# "":{ +# "memory":{ +# "value": +# "high-value": +# "low-value": +# } +# "time":{ +# "value": +# "high-value": +# "low-value": +# } +# } +# } +# +# These can be submitted to bencher.dev for analysis, e.g. a github action step like: +# # Upload to graphing service +# - uses: bencherdev/bencher@main +# - name: Upload benchmark results to Bencher +# run: | +# bencher run \ +# --project snmalloc \ +# --token '${{ secrets.BENCHER_DEV_API_TOKEN }}' \ +# --branch ${{ github.ref_name }} \ +# --adapter json \ +# --err \ +# --file bencher.dev.sn.json + +import re +import sys +import collections +try: + import numpy as np +except ImportError: + print('You need to install numpy.') + sys.exit(1) + +if len(sys.argv) != 2: + print('Usage: %s benchres.csv' % sys.argv[0]) + print('Where benchres.csv is the output of the benchmark script. I.e.') + print(' mimalloc-bench/out/bench/benchres.csv') + print() + print('The script generates a file per allocator for submission to bencher.dev.') + sys.exit(1) + +parse_line = re.compile('^([^ ]+) +([^ ]+) +([0-9:.]+) +([0-9]+) [0-9:.]+ [0-9:.]+ [0-9:.]+ [0-9:.]+$') +data = [] +test_names = set() + +# read in the data +with open(sys.argv[1]) as f: + for l in f.readlines(): + match = parse_line.search(l) + if not match: + continue + test_name, alloc_name, time_string, memory = match.groups() + time_split = time_string.split(':') + time_taken = 0 + test_names.add(test_name) + if len(time_split) == 2: + time_taken = int(time_split[0]) * 60 + float(time_split[1]) + else: + time_taken = float(time_split[0]) + data.append({"Benchmark":test_name, "Allocator":alloc_name, "Time":time_taken, "Memory":int(memory)}) + +# Output data in json of the form +# +# { +# "":{ +# "memory":{ +# "value": +# "high-value": +# "low-value": +# } +# "time":{ +# "value": +# "high-value": +# "low-value": +# } +# } +# } + +import json + +for alloc in set(d["Allocator"] for d in data if d["Benchmark"] == test_name): + output = {} + for test_name in test_names: + output[test_name] = { + "memory": { + "value": float(np.mean([d["Memory"] for d in data if d["Benchmark"] == test_name])), + "high-value": float(np.max([d["Memory"] for d in data if d["Benchmark"] == test_name])), + "low-value": float(np.min([d["Memory"] for d in data if d["Benchmark"] == test_name])), + }, + "time": { + "value": float(np.mean([d["Time"] for d in data if d["Benchmark"] == test_name])), + "high-value": float(np.max([d["Time"] for d in data if d["Benchmark"] == test_name])), + "low-value": float(np.min([d["Time"] for d in data if d["Benchmark"] == test_name])), + } + } + result = json.dumps(output, indent=2) + with open(f"bencher.dev.{alloc}.json", "w") as f: + f.write(result) + print(f"Output written to bencher.dev.{alloc}.json") + From ca9ae25e16f462bddf0e5da94f552232e8e68d1e Mon Sep 17 00:00:00 2001 From: jvoisin Date: Thu, 23 Oct 2025 16:00:23 +0200 Subject: [PATCH 09/13] Bump snmalloc --- build-bench-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-bench-env.sh b/build-bench-env.sh index 4cd9fed1..bc5a5a7c 100755 --- a/build-bench-env.sh +++ b/build-bench-env.sh @@ -59,7 +59,7 @@ readonly version_sc=master # unmaintained since 2016 readonly version_scudo=main readonly version_sg=master # ~unmaintained since 2021 readonly version_sm=master # ~unmaintained since 2017 -readonly version_sn=0.7.2 +readonly version_sn=0.7.3 readonly version_tbb=v2021.9.0 readonly version_tc=gperftools-2.16.90 readonly version_tcg=98fd24303c7b5ef5e30da625f11fb623a5e038b6 # 2025-07-18 From 59bc997278865545bba19eb5932fa8d1f63f99cb Mon Sep 17 00:00:00 2001 From: jvoisin Date: Sun, 25 Jan 2026 20:07:11 +0100 Subject: [PATCH 10/13] Bump gperftools from 2.16.90 to 2.18 --- build-bench-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-bench-env.sh b/build-bench-env.sh index bc5a5a7c..3223dc7f 100755 --- a/build-bench-env.sh +++ b/build-bench-env.sh @@ -61,7 +61,7 @@ readonly version_sg=master # ~unmaintained since 2021 readonly version_sm=master # ~unmaintained since 2017 readonly version_sn=0.7.3 readonly version_tbb=v2021.9.0 -readonly version_tc=gperftools-2.16.90 +readonly version_tc=gperftools-2.18 readonly version_tcg=98fd24303c7b5ef5e30da625f11fb623a5e038b6 # 2025-07-18 readonly version_yal=main From 6a00416ab724eeb98a9b2ac3913f6e790fe8c91f Mon Sep 17 00:00:00 2001 From: jvoisin Date: Mon, 23 Feb 2026 10:35:21 +0100 Subject: [PATCH 11/13] Bump tcg in the hope of fixing its build --- build-bench-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-bench-env.sh b/build-bench-env.sh index 3223dc7f..425f9de7 100755 --- a/build-bench-env.sh +++ b/build-bench-env.sh @@ -62,7 +62,7 @@ readonly version_sm=master # ~unmaintained since 2017 readonly version_sn=0.7.3 readonly version_tbb=v2021.9.0 readonly version_tc=gperftools-2.18 -readonly version_tcg=98fd24303c7b5ef5e30da625f11fb623a5e038b6 # 2025-07-18 +readonly version_tcg=81f4e44f23f2936303f9404fee7315119b9df623 # 2026-02-23 readonly version_yal=main # benchmark versions From b52c0718c16ba38fcd8f7ef058cca470f60ed002 Mon Sep 17 00:00:00 2001 From: jvoisin Date: Tue, 24 Mar 2026 17:34:35 +0100 Subject: [PATCH 12/13] Bump snmalloc to 0.7.4 --- build-bench-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-bench-env.sh b/build-bench-env.sh index 425f9de7..7efa222b 100755 --- a/build-bench-env.sh +++ b/build-bench-env.sh @@ -59,7 +59,7 @@ readonly version_sc=master # unmaintained since 2016 readonly version_scudo=main readonly version_sg=master # ~unmaintained since 2021 readonly version_sm=master # ~unmaintained since 2017 -readonly version_sn=0.7.3 +readonly version_sn=0.7.4 readonly version_tbb=v2021.9.0 readonly version_tc=gperftools-2.18 readonly version_tcg=81f4e44f23f2936303f9404fee7315119b9df623 # 2026-02-23 From efac5035f5d3d9f6de23c5167e9bd0ec9d1314e1 Mon Sep 17 00:00:00 2001 From: newell-romario Date: Fri, 3 Apr 2026 04:35:44 -0700 Subject: [PATCH 13/13] Add rmalloc --- bench.sh | 3 ++- build-bench-env.sh | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/bench.sh b/bench.sh index c03fe2de..e794d6f2 100755 --- a/bench.sh +++ b/bench.sh @@ -6,7 +6,7 @@ # Allocators and tests # -------------------------------------------------------------------- -readonly alloc_all="sys dh ff fg gd hd hm hml iso je lf lp lt mi mi-sec mi2 mi2-sec mng mesh nomesh pa rp sc scudo sg sm sn sn-sec tbb tc tcg mi-dbg mi2-dbg xmi xsmi xmi-dbg yal" +readonly alloc_all="sys dh ff fg gd hd hm hml iso je lf lp lt mi mi-sec mi2 mi2-sec mng mesh nomesh pa rp sc scudo sg sm sn sn-sec tbb tc tcg mi-dbg mi2-dbg xmi xsmi xmi-dbg yal rmalloc" readonly alloc_secure="dh ff gd hm hml iso mi-sec mi2-sec mng pa scudo sg sn-sec sg" alloc_run="" # allocators to run (expanded by command line options) alloc_installed="sys" # later expanded to include all installed allocators @@ -125,6 +125,7 @@ alloc_lib_add "tbb" "$lib_tbb" alloc_lib_add "tc" "$localdevdir/tc/.libs/libtcmalloc_minimal$extso" alloc_lib_add "tcg" "$localdevdir/tcg/bazel-bin/tcmalloc/libtcmalloc$extso" alloc_lib_add "yal" "$localdevdir/yal/yalloc$extso" +alloc_lib_add "rmalloc" "$localdevdir/rmalloc/lib/librmalloc$extso" alloc_lib_add "mi" "$localdevdir/mi/out/release/libmimalloc$extso" alloc_lib_add "mi-sec" "$localdevdir/mi/out/secure/libmimalloc-secure$extso" diff --git a/build-bench-env.sh b/build-bench-env.sh index 7efa222b..767d2de3 100755 --- a/build-bench-env.sh +++ b/build-bench-env.sh @@ -64,6 +64,7 @@ readonly version_tbb=v2021.9.0 readonly version_tc=gperftools-2.18 readonly version_tcg=81f4e44f23f2936303f9404fee7315119b9df623 # 2026-02-23 readonly version_yal=main +readonly version_rmalloc=master # benchmark versions readonly version_redis=6.2.7 @@ -104,6 +105,7 @@ setup_tbb=0 setup_tc=0 setup_tcg=0 setup_yal=0 +setup_rmalloc=0 # bigger benchmarks setup_bench=0 @@ -147,6 +149,7 @@ while : ; do setup_tbb=$flag_arg setup_tc=$flag_arg setup_yal=$flag_arg + setup_rmalloc=$flag_arg if [ -z "$darwin" ]; then setup_tcg=$flag_arg # lacking 'malloc.h' setup_dh=$flag_arg @@ -176,6 +179,8 @@ while : ; do ;; bench) setup_bench=$flag_arg;; + rmalloc) + setup_rmalloc=$flag_arg;; ff) setup_ff=$flag_arg;; fg) @@ -280,6 +285,7 @@ while : ; do echo " tc setup tcmalloc ($version_tc)" echo " tcg setup Google's tcmalloc ($version_tcg)" echo " yal setup yalloc ($version_yal)" + echo " rmalloc setup rmalloc($version_rmalloc)" echo "" echo " bench build all local benchmarks" echo " lean setup lean 3 benchmark" @@ -467,6 +473,13 @@ if test "$setup_packages" = "1"; then fi fi +if test "$setup_rmalloc" = "1"; then + checkout rmalloc $version_rmalloc https://github.com/newell-romario/rmalloc + cmake -DCMAKE_BUILD_TYPE=RELEASE . + make + popd +fi + if test "$setup_hm" = "1"; then checkout hm $version_hm https://github.com/GrapheneOS/hardened_malloc make CONFIG_NATIVE=true CONFIG_WERROR=false VARIANT=light -j $proc