From ed29a80f18c5aba5432c2c11831c8daa5d0e9695 Mon Sep 17 00:00:00 2001 From: bomanaps Date: Wed, 20 May 2026 17:46:56 +0100 Subject: [PATCH 1/2] switch to jemalloc allocator --- lean_client/Cargo.lock | 33 +++++++++++++++++++++++ lean_client/Cargo.toml | 13 +++++++++ lean_client/src/main.rs | 60 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 106 insertions(+) diff --git a/lean_client/Cargo.lock b/lean_client/Cargo.lock index c0de67c..4801b66 100644 --- a/lean_client/Cargo.lock +++ b/lean_client/Cargo.lock @@ -2987,6 +2987,8 @@ dependencies = [ "parking_lot", "reqwest", "ssz", + "tikv-jemalloc-ctl", + "tikv-jemallocator", "tokio", "tracing", "tracing-subscriber", @@ -6157,6 +6159,37 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "tikv-jemalloc-ctl" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "661f1f6a57b3a36dc9174a2c10f19513b4866816e13425d3e418b11cc37bc24c" +dependencies = [ + "libc", + "paste", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "time" version = "0.3.47" diff --git a/lean_client/Cargo.toml b/lean_client/Cargo.toml index 5f05c80..d9eda7d 100644 --- a/lean_client/Cargo.toml +++ b/lean_client/Cargo.toml @@ -293,6 +293,8 @@ ssz-types = "0.3" itertools = "0.14" test-generator = "0.3.1" thiserror = "2" +tikv-jemallocator = { version = "0.6", features = ["stats", "unprefixed_malloc_on_supported_platforms"] } +tikv-jemalloc-ctl = { version = "0.6", features = ["use_std"] } tiny-keccak = "2.0.2" tokio = { version = "1.0", features = ["full"] } tower = { version = "0.5", features = ["util"] } @@ -335,3 +337,14 @@ tracing-subscriber = { workspace = true } validator = { workspace = true } xmss = { workspace = true } reqwest = { workspace = true } +tikv-jemallocator = { workspace = true, optional = true } +tikv-jemalloc-ctl = { workspace = true, optional = true } + +[features] +default = ["jemalloc"] +jemalloc = ["dep:tikv-jemallocator"] +jemalloc_profiling = ["jemalloc", "dep:tikv-jemalloc-ctl", "tikv-jemallocator/profiling"] + +[profile.release] +debug = "line-tables-only" +strip = "none" diff --git a/lean_client/src/main.rs b/lean_client/src/main.rs index 60e5f7d..3fcbfea 100644 --- a/lean_client/src/main.rs +++ b/lean_client/src/main.rs @@ -1,3 +1,12 @@ +#[cfg(all(feature = "jemalloc", not(target_env = "msvc")))] +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +#[cfg(all(feature = "jemalloc_profiling", not(target_env = "msvc")))] +#[allow(non_upper_case_globals)] +#[unsafe(export_name = "malloc_conf")] +static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0"; + use anyhow::{Context as _, Result}; use clap::Parser; use containers::{ @@ -48,6 +57,54 @@ use xmss::{PublicKey, Signature}; mod aggregation; +#[cfg(all(feature = "jemalloc_profiling", unix))] +fn install_jemalloc_dump_signal_handler() { + use std::ffi::CString; + use tokio::signal::unix::{SignalKind, signal}; + + tokio::spawn(async move { + let mut stream = match signal(SignalKind::user_defined1()) { + Ok(s) => s, + Err(e) => { + warn!("failed to install SIGUSR1 handler for jemalloc dump: {e}"); + return; + } + }; + info!("SIGUSR1 handler installed; send `kill -USR1 ` to dump jemalloc heap profile"); + + loop { + stream.recv().await; + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + let path = format!("/tmp/grandine-heap-{ts}.prof"); + + info!(path = %path, "SIGUSR1 received, writing jemalloc heap dump"); + + let c_path = match CString::new(path.as_bytes()) { + Ok(p) => p, + Err(e) => { + warn!("invalid path for jemalloc dump: {e}"); + continue; + } + }; + + let result = unsafe { + tikv_jemalloc_ctl::raw::write( + b"prof.dump\0", + c_path.as_ptr() as *mut std::os::raw::c_char, + ) + }; + + match result { + Ok(()) => info!(path = %path, "jemalloc heap dump written"), + Err(e) => warn!("jemalloc heap dump failed: {e}"), + } + } + }); +} + fn load_node_key(path: &str) -> Result> { let hex_str = std::fs::read_to_string(path)?.trim().to_string(); let bytes = hex::decode(&hex_str)?; @@ -409,6 +466,9 @@ async fn main() -> Result<()> { git_version::git_version!(args = ["--always", "--abbrev=8"]), ); + #[cfg(all(feature = "jemalloc_profiling", unix))] + install_jemalloc_dump_signal_handler(); + let args = Args::parse(); for feature in args.features { From fe99c9374e0099fe6fdf52121120bf8ea2183c9d Mon Sep 17 00:00:00 2001 From: bomanaps Date: Thu, 21 May 2026 14:48:33 +0100 Subject: [PATCH 2/2] switch to jemalloc allocator --- lean_client/Cargo.lock | 12 --------- lean_client/Cargo.toml | 13 +-------- lean_client/src/main.rs | 58 +---------------------------------------- 3 files changed, 2 insertions(+), 81 deletions(-) diff --git a/lean_client/Cargo.lock b/lean_client/Cargo.lock index 4801b66..768fc92 100644 --- a/lean_client/Cargo.lock +++ b/lean_client/Cargo.lock @@ -2987,7 +2987,6 @@ dependencies = [ "parking_lot", "reqwest", "ssz", - "tikv-jemalloc-ctl", "tikv-jemallocator", "tokio", "tracing", @@ -6159,17 +6158,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "tikv-jemalloc-ctl" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "661f1f6a57b3a36dc9174a2c10f19513b4866816e13425d3e418b11cc37bc24c" -dependencies = [ - "libc", - "paste", - "tikv-jemalloc-sys", -] - [[package]] name = "tikv-jemalloc-sys" version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" diff --git a/lean_client/Cargo.toml b/lean_client/Cargo.toml index d9eda7d..4e5880f 100644 --- a/lean_client/Cargo.toml +++ b/lean_client/Cargo.toml @@ -294,7 +294,6 @@ itertools = "0.14" test-generator = "0.3.1" thiserror = "2" tikv-jemallocator = { version = "0.6", features = ["stats", "unprefixed_malloc_on_supported_platforms"] } -tikv-jemalloc-ctl = { version = "0.6", features = ["use_std"] } tiny-keccak = "2.0.2" tokio = { version = "1.0", features = ["full"] } tower = { version = "0.5", features = ["util"] } @@ -337,14 +336,4 @@ tracing-subscriber = { workspace = true } validator = { workspace = true } xmss = { workspace = true } reqwest = { workspace = true } -tikv-jemallocator = { workspace = true, optional = true } -tikv-jemalloc-ctl = { workspace = true, optional = true } - -[features] -default = ["jemalloc"] -jemalloc = ["dep:tikv-jemallocator"] -jemalloc_profiling = ["jemalloc", "dep:tikv-jemalloc-ctl", "tikv-jemallocator/profiling"] - -[profile.release] -debug = "line-tables-only" -strip = "none" +tikv-jemallocator = { workspace = true } diff --git a/lean_client/src/main.rs b/lean_client/src/main.rs index 3fcbfea..3ad1ad0 100644 --- a/lean_client/src/main.rs +++ b/lean_client/src/main.rs @@ -1,12 +1,7 @@ -#[cfg(all(feature = "jemalloc", not(target_env = "msvc")))] +#[cfg(not(target_env = "msvc"))] #[global_allocator] static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; -#[cfg(all(feature = "jemalloc_profiling", not(target_env = "msvc")))] -#[allow(non_upper_case_globals)] -#[unsafe(export_name = "malloc_conf")] -static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0"; - use anyhow::{Context as _, Result}; use clap::Parser; use containers::{ @@ -57,54 +52,6 @@ use xmss::{PublicKey, Signature}; mod aggregation; -#[cfg(all(feature = "jemalloc_profiling", unix))] -fn install_jemalloc_dump_signal_handler() { - use std::ffi::CString; - use tokio::signal::unix::{SignalKind, signal}; - - tokio::spawn(async move { - let mut stream = match signal(SignalKind::user_defined1()) { - Ok(s) => s, - Err(e) => { - warn!("failed to install SIGUSR1 handler for jemalloc dump: {e}"); - return; - } - }; - info!("SIGUSR1 handler installed; send `kill -USR1 ` to dump jemalloc heap profile"); - - loop { - stream.recv().await; - let ts = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); - let path = format!("/tmp/grandine-heap-{ts}.prof"); - - info!(path = %path, "SIGUSR1 received, writing jemalloc heap dump"); - - let c_path = match CString::new(path.as_bytes()) { - Ok(p) => p, - Err(e) => { - warn!("invalid path for jemalloc dump: {e}"); - continue; - } - }; - - let result = unsafe { - tikv_jemalloc_ctl::raw::write( - b"prof.dump\0", - c_path.as_ptr() as *mut std::os::raw::c_char, - ) - }; - - match result { - Ok(()) => info!(path = %path, "jemalloc heap dump written"), - Err(e) => warn!("jemalloc heap dump failed: {e}"), - } - } - }); -} - fn load_node_key(path: &str) -> Result> { let hex_str = std::fs::read_to_string(path)?.trim().to_string(); let bytes = hex::decode(&hex_str)?; @@ -466,9 +413,6 @@ async fn main() -> Result<()> { git_version::git_version!(args = ["--always", "--abbrev=8"]), ); - #[cfg(all(feature = "jemalloc_profiling", unix))] - install_jemalloc_dump_signal_handler(); - let args = Args::parse(); for feature in args.features {