From 9033d7163f8dd3923b42c253d77bd48d9f840674 Mon Sep 17 00:00:00 2001 From: Vesko Karaganev Date: Fri, 17 Feb 2023 12:50:42 +0000 Subject: [PATCH 1/5] Generate jemalloc_internal_types.h using autoconf Make the file autoconf-generated. In the follow-up change, the representation of the mallocx flags will be made configurable at compile-time. (cherry picked from commit 82caac43b38eda300a54b1e28327a60ed7235c1f) --- .gitignore | 1 + configure.ac | 4 ++++ ...jemalloc_internal_types.h => jemalloc_internal_types.h.in} | 0 3 files changed, 5 insertions(+) rename include/jemalloc/internal/{jemalloc_internal_types.h => jemalloc_internal_types.h.in} (100%) diff --git a/.gitignore b/.gitignore index 95dbaa5fd2..42d6398b76 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ /include/jemalloc/internal/jemalloc_preamble.h /include/jemalloc/internal/jemalloc_internal_defs.h +/include/jemalloc/internal/jemalloc_internal_types.h /include/jemalloc/internal/private_namespace.gen.h /include/jemalloc/internal/private_namespace.h /include/jemalloc/internal/private_namespace_jet.gen.h diff --git a/configure.ac b/configure.ac index e57d0667e4..78f247c7e6 100644 --- a/configure.ac +++ b/configure.ac @@ -1312,6 +1312,7 @@ cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_macros.h.in" cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_protos.h.in" cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_typedefs.h.in" cfgoutputs_in="${cfgoutputs_in} include/jemalloc/internal/jemalloc_preamble.h.in" +cfgoutputs_in="${cfgoutputs_in} include/jemalloc/internal/jemalloc_internal_types.h.in" cfgoutputs_in="${cfgoutputs_in} test/test.sh.in" cfgoutputs_in="${cfgoutputs_in} test/include/test/jemalloc_test.h.in" @@ -1324,6 +1325,7 @@ cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_macros.h" cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_protos.h" cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_typedefs.h" cfgoutputs_out="${cfgoutputs_out} include/jemalloc/internal/jemalloc_preamble.h" +cfgoutputs_out="${cfgoutputs_out} include/jemalloc/internal/jemalloc_internal_types.h" cfgoutputs_out="${cfgoutputs_out} test/test.sh" cfgoutputs_out="${cfgoutputs_out} test/include/test/jemalloc_test.h" @@ -1336,11 +1338,13 @@ cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_macros.h:include/jem cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_protos.h:include/jemalloc/jemalloc_protos.h.in" cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_typedefs.h:include/jemalloc/jemalloc_typedefs.h.in" cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_preamble.h" +cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_internal_types.h" cfgoutputs_tup="${cfgoutputs_tup} test/test.sh:test/test.sh.in" cfgoutputs_tup="${cfgoutputs_tup} test/include/test/jemalloc_test.h:test/include/test/jemalloc_test.h.in" cfghdrs_in="include/jemalloc/jemalloc_defs.h.in" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/jemalloc_internal_defs.h.in" +cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/jemalloc_internal_types.h.in" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_symbols.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_namespace.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_namespace.sh" diff --git a/include/jemalloc/internal/jemalloc_internal_types.h b/include/jemalloc/internal/jemalloc_internal_types.h.in similarity index 100% rename from include/jemalloc/internal/jemalloc_internal_types.h rename to include/jemalloc/internal/jemalloc_internal_types.h.in From 2680cd4190ec57639789a29d0d9965a1aaed5393 Mon Sep 17 00:00:00 2001 From: Vesko Karaganev Date: Wed, 1 Feb 2023 15:01:06 +0000 Subject: [PATCH 2/5] Add --with-lg-tcache-limit to increase the max supported tcaches In the Couchbase Server Data Service (KV Engine), we use one tcache per arena, per thread (multipies). We have one arena per bucket, and we currently support 30 buckets, but might eventually bump this up to 100 buckets per instance. However, because we allocate a tcache per thread as well, the number of tcache which we use ends up being num-buckets x num-threads. Tcaches are allocated when a thread decides to use a bucket's arena, allocating tcaches lazily, so not all threads will need a tcache, but most will. jemalloc can automatically allocate and make use of a tcache, but we've found that can result in incorrect accounting of memory stats per-arena, which is a deal-breaker for us, because we rely on some of these stats to calculate per-arena memory fragmentation. The old limit on the number of tcaches was largely dependant on the representation of the tcache ID in the 32-bit flags through which it was specified to je_mallocx. --- Flag bits: a: arena t: tcache .: configurable, dependent on --with-lg-tcache-limit: The higher this value, the more of the configurable bits will be teated as tcache bits (the rest are arena bits). 0: unused z: zero n: alignment Old representation: aaaaaaaa aaaatttt tttttttt 0znnnnnn New representation: aaaaaaa. ........ ....tttt 0znnnnnn Before this change, we had 12 bits for the arena index, 12 bits for the tcache ID, giving us 4094 (tcache 0 and 1 reserved) tcaches and 4096 arenas. By using more bits from the arena index representation, we can increase the representation of the tcache ID to be wider/narrower, allowing up to 2^17 valid tcache IDs to be specified (but 0 and 1 are reserved by jemalloc). In this example, however, the number of areans falls to 128. (cherry picked from commit 89033437882ccc253b15a0e131594f0df0f8b58c) --- configure.ac | 39 +++++++++++++++++++ .../internal/jemalloc_internal_types.h.in | 13 ++++--- include/jemalloc/jemalloc_macros.h.in | 8 ++-- scripts/gen_run_tests.py | 1 + test/unit/background_thread_enable.c | 7 ++++ 5 files changed, 59 insertions(+), 9 deletions(-) diff --git a/configure.ac b/configure.ac index 78f247c7e6..69f702824f 100644 --- a/configure.ac +++ b/configure.ac @@ -1957,6 +1957,43 @@ if test "x${je_cv_gcc_builtin_popcountl}" = "xyes" ; then AC_DEFINE([JEMALLOC_INTERNAL_POPCOUNTLL], [__builtin_popcountll], [ ]) fi +dnl Configure maximum number of tcaches/arenas +dnl Some tests assume at least 10 tcaches can be created, so we need at least +dnl 4-bits to represent the tcache IDs. +dnl The upper limit of 17 is not a technical limitation, rather, it is what our +dnl unit tests pass at. Some tests allocate areans and never destroy them, which +dnl means we run out of areans during tests. But we can't "support" a limit we +dnl cannot test... +AC_ARG_WITH([lg_tcache_limit], + [AS_HELP_STRING([--with-lg-tcache-limit=], + [Base 2 log of maximum number of tcaches (between 4 and 17). Trade off with number of arenas, which will be 2 ^ (24 - lg-cache-limit).])]) + +if test "x$with_lg_tcache_limit" = "x"; then + # Default is 4096 tcaches, 4096 arenas + with_lg_tcache_limit=12 +fi + +if ! test "$with_lg_tcache_limit" -ge 4 -a "$with_lg_tcache_limit" -le 17; then + AC_MSG_ERROR([lg-cache-limit should be between 4 and 17.]) +fi + +dnl Internal macros +tcache_and_arena_repr_bits=24 +tcache_repr_bits="$with_lg_tcache_limit" +arena_repr_bits="$(($tcache_and_arena_repr_bits-$tcache_repr_bits))" +tcache_shift=8 +arena_shift="$((32-$arena_repr_bits))" +arenas_all="$((2**$arena_repr_bits))" +arenas_destroyed="$(($arenas_all+1))" +AC_SUBST([tcache_repr_bits]) +AC_SUBST([arena_repr_bits]) + +dnl Public macros +AC_SUBST([tcache_shift]) +AC_SUBST([arena_shift]) +AC_SUBST([arenas_all]) +AC_SUBST([arenas_destroyed]) + AC_ARG_WITH([lg_quantum], [AS_HELP_STRING([--with-lg-quantum=], [Base 2 log of minimum allocation alignment])]) @@ -3092,6 +3129,8 @@ AC_MSG_RESULT([]) AC_MSG_RESULT([JEMALLOC_PREFIX : ${JEMALLOC_PREFIX}]) AC_MSG_RESULT([JEMALLOC_PRIVATE_NAMESPACE]) AC_MSG_RESULT([ : ${JEMALLOC_PRIVATE_NAMESPACE}]) +AC_MSG_RESULT([tcache_repr_bits : ${tcache_repr_bits}]) +AC_MSG_RESULT([arena_repr_bits : ${arena_repr_bits}]) AC_MSG_RESULT([install_suffix : ${install_suffix}]) AC_MSG_RESULT([malloc_conf : ${config_malloc_conf}]) AC_MSG_RESULT([documentation : ${enable_doc}]) diff --git a/include/jemalloc/internal/jemalloc_internal_types.h.in b/include/jemalloc/internal/jemalloc_internal_types.h.in index 0ade5461ba..0e81d53945 100644 --- a/include/jemalloc/internal/jemalloc_internal_types.h.in +++ b/include/jemalloc/internal/jemalloc_internal_types.h.in @@ -33,17 +33,20 @@ typedef enum malloc_init_e malloc_init_t; * * a: arena * t: tcache + * .: configurable, dependent on --with-lg-tcache-limit: + * The higher this value, the more of the configurable bits will be teated as + * tcache bits (the rest are arena bits). * 0: unused * z: zero * n: alignment * - * aaaaaaaa aaaatttt tttttttt 0znnnnnn + * aaaaaaa. ........ ....tttt 0znnnnnn */ -#define MALLOCX_ARENA_BITS 12 -#define MALLOCX_TCACHE_BITS 12 +#define MALLOCX_ARENA_BITS @arena_repr_bits@ +#define MALLOCX_TCACHE_BITS @tcache_repr_bits@ #define MALLOCX_LG_ALIGN_BITS 6 -#define MALLOCX_ARENA_SHIFT 20 -#define MALLOCX_TCACHE_SHIFT 8 +#define MALLOCX_ARENA_SHIFT @arena_shift@ +#define MALLOCX_TCACHE_SHIFT @tcache_shift@ #define MALLOCX_ARENA_MASK \ ((unsigned)(((1U << MALLOCX_ARENA_BITS) - 1) << MALLOCX_ARENA_SHIFT)) /* NB: Arena index bias decreases the maximum number of arenas by 1. */ diff --git a/include/jemalloc/jemalloc_macros.h.in b/include/jemalloc/jemalloc_macros.h.in index 06f47b8a14..826ff0226f 100644 --- a/include/jemalloc/jemalloc_macros.h.in +++ b/include/jemalloc/jemalloc_macros.h.in @@ -25,12 +25,12 @@ * Bias tcache index bits so that 0 encodes "automatic tcache management", and 1 * encodes MALLOCX_TCACHE_NONE. */ -#define MALLOCX_TCACHE(tc) ((int)(((tc)+2) << 8)) +#define MALLOCX_TCACHE(tc) ((int)(((tc)+2) << @tcache_shift@)) #define MALLOCX_TCACHE_NONE MALLOCX_TCACHE(-1) /* * Bias arena index bits so that 0 encodes "use an automatically chosen arena". */ -#define MALLOCX_ARENA(a) ((((int)(a))+1) << 20) +#define MALLOCX_ARENA(a) ((((int)(a))+1) << @arena_shift@) /* * Use as arena index in "arena..{purge,decay,dss}" and @@ -44,12 +44,12 @@ * mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", NULL, NULL, NULL, * 0); */ -#define MALLCTL_ARENAS_ALL 4096 +#define MALLCTL_ARENAS_ALL @arenas_all@ /* * Use as arena index in "stats.arenas..*" mallctl interfaces to select * destroyed arenas. */ -#define MALLCTL_ARENAS_DESTROYED 4097 +#define MALLCTL_ARENAS_DESTROYED @arenas_destroyed@ #if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW) # define JEMALLOC_CXX_THROW noexcept (true) diff --git a/scripts/gen_run_tests.py b/scripts/gen_run_tests.py index 7c3075f9f6..9db353369a 100755 --- a/scripts/gen_run_tests.py +++ b/scripts/gen_run_tests.py @@ -42,6 +42,7 @@ def powerset(items): '--disable-stats', '--enable-opt-safety-checks', '--with-lg-page=16', + '--with-lg-tcache-limit=15', ] if bits_64: possible_config_opts.append('--with-lg-vaddr=56') diff --git a/test/unit/background_thread_enable.c b/test/unit/background_thread_enable.c index 57f26c4bfd..7030da626d 100644 --- a/test/unit/background_thread_enable.c +++ b/test/unit/background_thread_enable.c @@ -13,6 +13,13 @@ max_test_narenas(void) { * approximation. */ unsigned ret = 10 * ncpus; + // Limit the max to avoid running out of arenas. There are quite a few + // tests where arenas.create is called but arena..destroy is not, so the + // arena remains. Once those are fixed, we should be able to change this to + // a larger limit capped at ARENAS_LIMIT. + if (ret > 50) { + ret = 50; + } /* Limit the max to avoid VM exhaustion on 32-bit . */ return ret > 256 ? 256 : ret; From eed2a1a2551f01e8dd92cb3ac62fdb5ebbe3fb82 Mon Sep 17 00:00:00 2001 From: Trond Norbye Date: Wed, 29 Jan 2025 10:32:19 +0100 Subject: [PATCH 3/5] Allow arena 0 to be specified explicitly As seen in our tests when we override the global new allocator and perform allocations from the context we're cleaning up thread local data we may end up calling je_mallocx and explicitly specify arena 0 which cause jemalloc to fail an assert (before it explicitly change the allocation to use arena 0). This asssert is currently blocking our upgrade of folly, and the effect of allowing arena 0 to be specified is harmless in this context (as jemalloc would change it to 0 on the next statement) See https://github.com/jemalloc/jemalloc/issues/2791 See https://jira.issues.couchbase.com/browse/MB-65040 (cherry picked from commit 13f495b273ec164f960b659ec4c3045fa468a52a) --- src/jemalloc.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index 8d341ba37e..c026099b3d 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1592,9 +1592,10 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { * We should never specify particular arenas or tcaches from * within our internal allocations. */ - assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC - || dopts->tcache_ind == TCACHE_IND_NONE); - assert(dopts->arena_ind == ARENA_IND_AUTOMATIC); + assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC || + dopts->tcache_ind == TCACHE_IND_NONE); + assert(dopts->arena_ind == ARENA_IND_AUTOMATIC || + dopts->arena_ind == 0); dopts->tcache_ind = TCACHE_IND_NONE; /* We know that arena 0 has already been initialized. */ dopts->arena_ind = 0; From 1665de0433a19352dcce142c28cd70dc158d0d1f Mon Sep 17 00:00:00 2001 From: Vesko Karaganev Date: Fri, 4 Apr 2025 12:53:48 +0100 Subject: [PATCH 4/5] Add je_thread_cleanup() and nullify the tsd when freeing On Windows, we have seen that the tls callback is called before C++ thread_local destructors run. This means that malloc_tsd_dalloc is called before potential further alloc/free is done, leaving us with a dangling tsd_t returned by tsd_wrapper_get(). In order to workaround this, first we need to reset the tls storage to NULL. It can now get re-created if needed. Also, provide a new exported symbol je_tls_cleanup() which allows users which alloc/free at thread_local destruction to manually cleanup a potential re-created tsd_t. The contract for the je_thread_cleanup() method is that it can be called at any point of the thread lifetime, and it will clear the thread tsd_t. The tsd_t will be re-created automatically if required again. Calling je_thread_cleanup() at the end of the thread lifetime ensures that there is not a memory leak in threads which allocate during thread exit. (cherry picked from commit fc0eda64ab9a12ec06e80283020b3461570195ed) --- include/jemalloc/internal/tsd_win.h | 4 ++++ include/jemalloc/jemalloc_protos.h.in | 1 + src/jemalloc.c | 8 ++++++++ 3 files changed, 13 insertions(+) diff --git a/include/jemalloc/internal/tsd_win.h b/include/jemalloc/internal/tsd_win.h index 8b22bec18d..6158d1a077 100644 --- a/include/jemalloc/internal/tsd_win.h +++ b/include/jemalloc/internal/tsd_win.h @@ -52,6 +52,10 @@ tsd_cleanup_wrapper(void) { } } malloc_tsd_dalloc(wrapper); + if (!TlsSetValue(tsd_tsd, NULL)) { + malloc_write(": Error clearing TSD\n"); + abort(); + } return false; } diff --git a/include/jemalloc/jemalloc_protos.h.in b/include/jemalloc/jemalloc_protos.h.in index 21e4dc579a..a090dafeea 100644 --- a/include/jemalloc/jemalloc_protos.h.in +++ b/include/jemalloc/jemalloc_protos.h.in @@ -61,6 +61,7 @@ JEMALLOC_EXPORT void JEMALLOC_NOTHROW @je_@malloc_stats_print( const char *opts); JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @je_@malloc_usable_size( JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW; +JEMALLOC_EXPORT void JEMALLOC_NOTHROW @je_@thread_cleanup() JEMALLOC_CXX_THROW; #ifdef JEMALLOC_HAVE_MALLOC_SIZE JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @je_@malloc_size( const void *ptr); diff --git a/src/jemalloc.c b/src/jemalloc.c index c026099b3d..42f1f50cfc 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -23,6 +23,7 @@ #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/thread_event.h" +#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/util.h" #include "jemalloc/internal/conf.h" @@ -3086,6 +3087,13 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { return ret; } +JEMALLOC_EXPORT void JEMALLOC_NOTHROW +je_thread_cleanup() { + #if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) + _malloc_thread_cleanup(); + #endif +} + #ifdef JEMALLOC_HAVE_MALLOC_SIZE JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_size(const void *ptr) { From 3d596c11f2d0ffbf984e525fc22976ff5773e2d7 Mon Sep 17 00:00:00 2001 From: Vesko Karaganev Date: Wed, 25 Jun 2025 10:52:04 +0100 Subject: [PATCH 5/5] Add je_thread_cleanup to public symbols list Otherwise, the symbol name is not correct on macOS. Before: nm -gU lib/libjemalloc.a | grep je_th 0000000000006c74 T _je_je_thread_cleanup 0000000000000818 D _je_thp_mode_names After: nm -gU lib/libjemalloc.a | grep je_th 0000000000006c74 T _je_thread_cleanup 0000000000000818 D _je_thp_mode_names (cherry picked from commit 59aa12eeacfed4c876a8b4b54f477dfbc69f1b9f) --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 69f702824f..26519bbd96 100644 --- a/configure.ac +++ b/configure.ac @@ -1205,7 +1205,7 @@ AC_ARG_WITH([export], fi] ) -public_syms="aligned_alloc calloc dallocx free free_sized free_aligned_sized mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_conf_2_conf_harder malloc_message malloc_stats_print malloc_usable_size mallocx smallocx_${jemalloc_version_gid} nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx" +public_syms="aligned_alloc calloc dallocx free free_sized free_aligned_sized mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_conf_2_conf_harder malloc_message malloc_stats_print malloc_usable_size thread_cleanup mallocx smallocx_${jemalloc_version_gid} nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx" dnl Check for additional platform-specific public API functions. AC_CHECK_FUNC([memalign], [AC_DEFINE([JEMALLOC_OVERRIDE_MEMALIGN], [ ], [ ])