diff --git a/.gitignore b/.gitignore index 95dbaa5fd2..42d6398b76 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ /include/jemalloc/internal/jemalloc_preamble.h /include/jemalloc/internal/jemalloc_internal_defs.h +/include/jemalloc/internal/jemalloc_internal_types.h /include/jemalloc/internal/private_namespace.gen.h /include/jemalloc/internal/private_namespace.h /include/jemalloc/internal/private_namespace_jet.gen.h diff --git a/configure.ac b/configure.ac index e57d0667e4..26519bbd96 100644 --- a/configure.ac +++ b/configure.ac @@ -1205,7 +1205,7 @@ AC_ARG_WITH([export], fi] ) -public_syms="aligned_alloc calloc dallocx free free_sized free_aligned_sized mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_conf_2_conf_harder malloc_message malloc_stats_print malloc_usable_size mallocx smallocx_${jemalloc_version_gid} nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx" +public_syms="aligned_alloc calloc dallocx free free_sized free_aligned_sized mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_conf_2_conf_harder malloc_message malloc_stats_print malloc_usable_size thread_cleanup mallocx smallocx_${jemalloc_version_gid} nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx" dnl Check for additional platform-specific public API functions. AC_CHECK_FUNC([memalign], [AC_DEFINE([JEMALLOC_OVERRIDE_MEMALIGN], [ ], [ ]) @@ -1312,6 +1312,7 @@ cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_macros.h.in" cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_protos.h.in" cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_typedefs.h.in" cfgoutputs_in="${cfgoutputs_in} include/jemalloc/internal/jemalloc_preamble.h.in" +cfgoutputs_in="${cfgoutputs_in} include/jemalloc/internal/jemalloc_internal_types.h.in" cfgoutputs_in="${cfgoutputs_in} test/test.sh.in" cfgoutputs_in="${cfgoutputs_in} test/include/test/jemalloc_test.h.in" @@ -1324,6 +1325,7 @@ cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_macros.h" cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_protos.h" cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_typedefs.h" cfgoutputs_out="${cfgoutputs_out} include/jemalloc/internal/jemalloc_preamble.h" +cfgoutputs_out="${cfgoutputs_out} include/jemalloc/internal/jemalloc_internal_types.h" cfgoutputs_out="${cfgoutputs_out} test/test.sh" cfgoutputs_out="${cfgoutputs_out} test/include/test/jemalloc_test.h" @@ -1336,11 +1338,13 @@ cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_macros.h:include/jem cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_protos.h:include/jemalloc/jemalloc_protos.h.in" cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_typedefs.h:include/jemalloc/jemalloc_typedefs.h.in" cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_preamble.h" +cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_internal_types.h" cfgoutputs_tup="${cfgoutputs_tup} test/test.sh:test/test.sh.in" cfgoutputs_tup="${cfgoutputs_tup} test/include/test/jemalloc_test.h:test/include/test/jemalloc_test.h.in" cfghdrs_in="include/jemalloc/jemalloc_defs.h.in" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/jemalloc_internal_defs.h.in" +cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/jemalloc_internal_types.h.in" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_symbols.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_namespace.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_namespace.sh" @@ -1953,6 +1957,43 @@ if test "x${je_cv_gcc_builtin_popcountl}" = "xyes" ; then AC_DEFINE([JEMALLOC_INTERNAL_POPCOUNTLL], [__builtin_popcountll], [ ]) fi +dnl Configure maximum number of tcaches/arenas +dnl Some tests assume at least 10 tcaches can be created, so we need at least +dnl 4-bits to represent the tcache IDs. +dnl The upper limit of 17 is not a technical limitation, rather, it is what our +dnl unit tests pass at. Some tests allocate areans and never destroy them, which +dnl means we run out of areans during tests. But we can't "support" a limit we +dnl cannot test... +AC_ARG_WITH([lg_tcache_limit], + [AS_HELP_STRING([--with-lg-tcache-limit=], + [Base 2 log of maximum number of tcaches (between 4 and 17). Trade off with number of arenas, which will be 2 ^ (24 - lg-cache-limit).])]) + +if test "x$with_lg_tcache_limit" = "x"; then + # Default is 4096 tcaches, 4096 arenas + with_lg_tcache_limit=12 +fi + +if ! test "$with_lg_tcache_limit" -ge 4 -a "$with_lg_tcache_limit" -le 17; then + AC_MSG_ERROR([lg-cache-limit should be between 4 and 17.]) +fi + +dnl Internal macros +tcache_and_arena_repr_bits=24 +tcache_repr_bits="$with_lg_tcache_limit" +arena_repr_bits="$(($tcache_and_arena_repr_bits-$tcache_repr_bits))" +tcache_shift=8 +arena_shift="$((32-$arena_repr_bits))" +arenas_all="$((2**$arena_repr_bits))" +arenas_destroyed="$(($arenas_all+1))" +AC_SUBST([tcache_repr_bits]) +AC_SUBST([arena_repr_bits]) + +dnl Public macros +AC_SUBST([tcache_shift]) +AC_SUBST([arena_shift]) +AC_SUBST([arenas_all]) +AC_SUBST([arenas_destroyed]) + AC_ARG_WITH([lg_quantum], [AS_HELP_STRING([--with-lg-quantum=], [Base 2 log of minimum allocation alignment])]) @@ -3088,6 +3129,8 @@ AC_MSG_RESULT([]) AC_MSG_RESULT([JEMALLOC_PREFIX : ${JEMALLOC_PREFIX}]) AC_MSG_RESULT([JEMALLOC_PRIVATE_NAMESPACE]) AC_MSG_RESULT([ : ${JEMALLOC_PRIVATE_NAMESPACE}]) +AC_MSG_RESULT([tcache_repr_bits : ${tcache_repr_bits}]) +AC_MSG_RESULT([arena_repr_bits : ${arena_repr_bits}]) AC_MSG_RESULT([install_suffix : ${install_suffix}]) AC_MSG_RESULT([malloc_conf : ${config_malloc_conf}]) AC_MSG_RESULT([documentation : ${enable_doc}]) diff --git a/include/jemalloc/internal/jemalloc_internal_types.h b/include/jemalloc/internal/jemalloc_internal_types.h.in similarity index 93% rename from include/jemalloc/internal/jemalloc_internal_types.h rename to include/jemalloc/internal/jemalloc_internal_types.h.in index 0ade5461ba..0e81d53945 100644 --- a/include/jemalloc/internal/jemalloc_internal_types.h +++ b/include/jemalloc/internal/jemalloc_internal_types.h.in @@ -33,17 +33,20 @@ typedef enum malloc_init_e malloc_init_t; * * a: arena * t: tcache + * .: configurable, dependent on --with-lg-tcache-limit: + * The higher this value, the more of the configurable bits will be teated as + * tcache bits (the rest are arena bits). * 0: unused * z: zero * n: alignment * - * aaaaaaaa aaaatttt tttttttt 0znnnnnn + * aaaaaaa. ........ ....tttt 0znnnnnn */ -#define MALLOCX_ARENA_BITS 12 -#define MALLOCX_TCACHE_BITS 12 +#define MALLOCX_ARENA_BITS @arena_repr_bits@ +#define MALLOCX_TCACHE_BITS @tcache_repr_bits@ #define MALLOCX_LG_ALIGN_BITS 6 -#define MALLOCX_ARENA_SHIFT 20 -#define MALLOCX_TCACHE_SHIFT 8 +#define MALLOCX_ARENA_SHIFT @arena_shift@ +#define MALLOCX_TCACHE_SHIFT @tcache_shift@ #define MALLOCX_ARENA_MASK \ ((unsigned)(((1U << MALLOCX_ARENA_BITS) - 1) << MALLOCX_ARENA_SHIFT)) /* NB: Arena index bias decreases the maximum number of arenas by 1. */ diff --git a/include/jemalloc/internal/tsd_win.h b/include/jemalloc/internal/tsd_win.h index 8b22bec18d..6158d1a077 100644 --- a/include/jemalloc/internal/tsd_win.h +++ b/include/jemalloc/internal/tsd_win.h @@ -52,6 +52,10 @@ tsd_cleanup_wrapper(void) { } } malloc_tsd_dalloc(wrapper); + if (!TlsSetValue(tsd_tsd, NULL)) { + malloc_write(": Error clearing TSD\n"); + abort(); + } return false; } diff --git a/include/jemalloc/jemalloc_macros.h.in b/include/jemalloc/jemalloc_macros.h.in index 06f47b8a14..826ff0226f 100644 --- a/include/jemalloc/jemalloc_macros.h.in +++ b/include/jemalloc/jemalloc_macros.h.in @@ -25,12 +25,12 @@ * Bias tcache index bits so that 0 encodes "automatic tcache management", and 1 * encodes MALLOCX_TCACHE_NONE. */ -#define MALLOCX_TCACHE(tc) ((int)(((tc)+2) << 8)) +#define MALLOCX_TCACHE(tc) ((int)(((tc)+2) << @tcache_shift@)) #define MALLOCX_TCACHE_NONE MALLOCX_TCACHE(-1) /* * Bias arena index bits so that 0 encodes "use an automatically chosen arena". */ -#define MALLOCX_ARENA(a) ((((int)(a))+1) << 20) +#define MALLOCX_ARENA(a) ((((int)(a))+1) << @arena_shift@) /* * Use as arena index in "arena..{purge,decay,dss}" and @@ -44,12 +44,12 @@ * mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", NULL, NULL, NULL, * 0); */ -#define MALLCTL_ARENAS_ALL 4096 +#define MALLCTL_ARENAS_ALL @arenas_all@ /* * Use as arena index in "stats.arenas..*" mallctl interfaces to select * destroyed arenas. */ -#define MALLCTL_ARENAS_DESTROYED 4097 +#define MALLCTL_ARENAS_DESTROYED @arenas_destroyed@ #if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW) # define JEMALLOC_CXX_THROW noexcept (true) diff --git a/include/jemalloc/jemalloc_protos.h.in b/include/jemalloc/jemalloc_protos.h.in index 21e4dc579a..a090dafeea 100644 --- a/include/jemalloc/jemalloc_protos.h.in +++ b/include/jemalloc/jemalloc_protos.h.in @@ -61,6 +61,7 @@ JEMALLOC_EXPORT void JEMALLOC_NOTHROW @je_@malloc_stats_print( const char *opts); JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @je_@malloc_usable_size( JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW; +JEMALLOC_EXPORT void JEMALLOC_NOTHROW @je_@thread_cleanup() JEMALLOC_CXX_THROW; #ifdef JEMALLOC_HAVE_MALLOC_SIZE JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @je_@malloc_size( const void *ptr); diff --git a/scripts/gen_run_tests.py b/scripts/gen_run_tests.py index 7c3075f9f6..9db353369a 100755 --- a/scripts/gen_run_tests.py +++ b/scripts/gen_run_tests.py @@ -42,6 +42,7 @@ def powerset(items): '--disable-stats', '--enable-opt-safety-checks', '--with-lg-page=16', + '--with-lg-tcache-limit=15', ] if bits_64: possible_config_opts.append('--with-lg-vaddr=56') diff --git a/src/jemalloc.c b/src/jemalloc.c index 8d341ba37e..42f1f50cfc 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -23,6 +23,7 @@ #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/thread_event.h" +#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/util.h" #include "jemalloc/internal/conf.h" @@ -1592,9 +1593,10 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { * We should never specify particular arenas or tcaches from * within our internal allocations. */ - assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC - || dopts->tcache_ind == TCACHE_IND_NONE); - assert(dopts->arena_ind == ARENA_IND_AUTOMATIC); + assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC || + dopts->tcache_ind == TCACHE_IND_NONE); + assert(dopts->arena_ind == ARENA_IND_AUTOMATIC || + dopts->arena_ind == 0); dopts->tcache_ind = TCACHE_IND_NONE; /* We know that arena 0 has already been initialized. */ dopts->arena_ind = 0; @@ -3085,6 +3087,13 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { return ret; } +JEMALLOC_EXPORT void JEMALLOC_NOTHROW +je_thread_cleanup() { + #if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) + _malloc_thread_cleanup(); + #endif +} + #ifdef JEMALLOC_HAVE_MALLOC_SIZE JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_size(const void *ptr) { diff --git a/test/unit/background_thread_enable.c b/test/unit/background_thread_enable.c index 57f26c4bfd..7030da626d 100644 --- a/test/unit/background_thread_enable.c +++ b/test/unit/background_thread_enable.c @@ -13,6 +13,13 @@ max_test_narenas(void) { * approximation. */ unsigned ret = 10 * ncpus; + // Limit the max to avoid running out of arenas. There are quite a few + // tests where arenas.create is called but arena..destroy is not, so the + // arena remains. Once those are fixed, we should be able to change this to + // a larger limit capped at ARENAS_LIMIT. + if (ret > 50) { + ret = 50; + } /* Limit the max to avoid VM exhaustion on 32-bit . */ return ret > 256 ? 256 : ret;