5.3.1 couchbase

.gitignore

@@ -23,6 +23,7 @@
 
 /include/jemalloc/internal/jemalloc_preamble.h
 /include/jemalloc/internal/jemalloc_internal_defs.h
+/include/jemalloc/internal/jemalloc_internal_types.h
 /include/jemalloc/internal/private_namespace.gen.h
 /include/jemalloc/internal/private_namespace.h
 /include/jemalloc/internal/private_namespace_jet.gen.h

configure.ac

@@ -1205,7 +1205,7 @@ AC_ARG_WITH([export],
 fi]
 )
 
-public_syms="aligned_alloc calloc dallocx free free_sized free_aligned_sized mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_conf_2_conf_harder malloc_message malloc_stats_print malloc_usable_size mallocx smallocx_${jemalloc_version_gid} nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx"
+public_syms="aligned_alloc calloc dallocx free free_sized free_aligned_sized mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_conf_2_conf_harder malloc_message malloc_stats_print malloc_usable_size thread_cleanup mallocx smallocx_${jemalloc_version_gid} nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx"
 dnl Check for additional platform-specific public API functions.
 AC_CHECK_FUNC([memalign],
 	      [AC_DEFINE([JEMALLOC_OVERRIDE_MEMALIGN], [ ], [ ])
@@ -1312,6 +1312,7 @@ cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_macros.h.in"
 cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_protos.h.in"
 cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_typedefs.h.in"
 cfgoutputs_in="${cfgoutputs_in} include/jemalloc/internal/jemalloc_preamble.h.in"
+cfgoutputs_in="${cfgoutputs_in} include/jemalloc/internal/jemalloc_internal_types.h.in"
 cfgoutputs_in="${cfgoutputs_in} test/test.sh.in"
 cfgoutputs_in="${cfgoutputs_in} test/include/test/jemalloc_test.h.in"
 
@@ -1324,6 +1325,7 @@ cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_macros.h"
 cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_protos.h"
 cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_typedefs.h"
 cfgoutputs_out="${cfgoutputs_out} include/jemalloc/internal/jemalloc_preamble.h"
+cfgoutputs_out="${cfgoutputs_out} include/jemalloc/internal/jemalloc_internal_types.h"
 cfgoutputs_out="${cfgoutputs_out} test/test.sh"
 cfgoutputs_out="${cfgoutputs_out} test/include/test/jemalloc_test.h"
 
@@ -1336,11 +1338,13 @@ cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_macros.h:include/jem
 cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_protos.h:include/jemalloc/jemalloc_protos.h.in"
 cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_typedefs.h:include/jemalloc/jemalloc_typedefs.h.in"
 cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_preamble.h"
+cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_internal_types.h"
 cfgoutputs_tup="${cfgoutputs_tup} test/test.sh:test/test.sh.in"
 cfgoutputs_tup="${cfgoutputs_tup} test/include/test/jemalloc_test.h:test/include/test/jemalloc_test.h.in"
 
 cfghdrs_in="include/jemalloc/jemalloc_defs.h.in"
 cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/jemalloc_internal_defs.h.in"
+cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/jemalloc_internal_types.h.in"
 cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_symbols.sh"
 cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_namespace.sh"
 cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_namespace.sh"
@@ -1953,6 +1957,43 @@ if test "x${je_cv_gcc_builtin_popcountl}" = "xyes" ; then
   AC_DEFINE([JEMALLOC_INTERNAL_POPCOUNTLL], [__builtin_popcountll], [ ])
 fi
 
+dnl Configure maximum number of tcaches/arenas
+dnl Some tests assume at least 10 tcaches can be created, so we need at least
+dnl 4-bits to represent the tcache IDs.
+dnl The upper limit of 17 is not a technical limitation, rather, it is what our
+dnl unit tests pass at. Some tests allocate areans and never destroy them, which
+dnl means we run out of areans during tests. But we can't "support" a limit we
+dnl cannot test...
+AC_ARG_WITH([lg_tcache_limit],
+  [AS_HELP_STRING([--with-lg-tcache-limit=<lg-cache-limit>],
+  [Base 2 log of maximum number of tcaches (between 4 and 17). Trade off with number of arenas, which will be 2 ^ (24 - lg-cache-limit).])])
+
+if test "x$with_lg_tcache_limit" = "x"; then
+  # Default is 4096 tcaches, 4096 arenas
+  with_lg_tcache_limit=12
+fi
+
+if ! test "$with_lg_tcache_limit" -ge 4 -a "$with_lg_tcache_limit" -le 17; then
+  AC_MSG_ERROR([lg-cache-limit should be between 4 and 17.])
+fi
+
+dnl Internal macros
+tcache_and_arena_repr_bits=24
+tcache_repr_bits="$with_lg_tcache_limit"
+arena_repr_bits="$(($tcache_and_arena_repr_bits-$tcache_repr_bits))"
+tcache_shift=8
+arena_shift="$((32-$arena_repr_bits))"
+arenas_all="$((2**$arena_repr_bits))"
+arenas_destroyed="$(($arenas_all+1))"
+AC_SUBST([tcache_repr_bits])
+AC_SUBST([arena_repr_bits])
+
+dnl Public macros
+AC_SUBST([tcache_shift])
+AC_SUBST([arena_shift])
+AC_SUBST([arenas_all])
+AC_SUBST([arenas_destroyed])
+
 AC_ARG_WITH([lg_quantum],
   [AS_HELP_STRING([--with-lg-quantum=<lg-quantum>],
    [Base 2 log of minimum allocation alignment])])
@@ -3088,6 +3129,8 @@ AC_MSG_RESULT([])
 AC_MSG_RESULT([JEMALLOC_PREFIX    : ${JEMALLOC_PREFIX}])
 AC_MSG_RESULT([JEMALLOC_PRIVATE_NAMESPACE])
 AC_MSG_RESULT([                   : ${JEMALLOC_PRIVATE_NAMESPACE}])
+AC_MSG_RESULT([tcache_repr_bits   : ${tcache_repr_bits}])
+AC_MSG_RESULT([arena_repr_bits    : ${arena_repr_bits}])
 AC_MSG_RESULT([install_suffix     : ${install_suffix}])
 AC_MSG_RESULT([malloc_conf        : ${config_malloc_conf}])
 AC_MSG_RESULT([documentation      : ${enable_doc}])

include/jemalloc/internal/jemalloc_internal_types.h → include/jemalloc/internal/jemalloc_internal_types.h.in

@@ -33,17 +33,20 @@ typedef enum malloc_init_e malloc_init_t;
  *
  * a: arena
  * t: tcache
+ * .: configurable, dependent on --with-lg-tcache-limit:
+ *    The higher this value, the more of the configurable bits will be teated as
+ *    tcache bits (the rest are arena bits).
  * 0: unused
  * z: zero
  * n: alignment
  *
- * aaaaaaaa aaaatttt tttttttt 0znnnnnn
+ * aaaaaaa. ........ ....tttt 0znnnnnn
  */
-#define MALLOCX_ARENA_BITS 12
-#define MALLOCX_TCACHE_BITS 12
+#define MALLOCX_ARENA_BITS @arena_repr_bits@
+#define MALLOCX_TCACHE_BITS @tcache_repr_bits@
 #define MALLOCX_LG_ALIGN_BITS 6
-#define MALLOCX_ARENA_SHIFT 20
-#define MALLOCX_TCACHE_SHIFT 8
+#define MALLOCX_ARENA_SHIFT @arena_shift@
+#define MALLOCX_TCACHE_SHIFT @tcache_shift@
 #define MALLOCX_ARENA_MASK                                                     \
 	((unsigned)(((1U << MALLOCX_ARENA_BITS) - 1) << MALLOCX_ARENA_SHIFT))
 /* NB: Arena index bias decreases the maximum number of arenas by 1. */

include/jemalloc/internal/tsd_win.h

@@ -52,6 +52,10 @@ tsd_cleanup_wrapper(void) {
 		}
 	}
 	malloc_tsd_dalloc(wrapper);
+	if (!TlsSetValue(tsd_tsd, NULL)) {
+		malloc_write("<jemalloc>: Error clearing TSD\n");
+		abort();
+	}
 	return false;
 }
 

include/jemalloc/jemalloc_macros.h.in

@@ -25,12 +25,12 @@
  * Bias tcache index bits so that 0 encodes "automatic tcache management", and 1
  * encodes MALLOCX_TCACHE_NONE.
  */
-#define MALLOCX_TCACHE(tc)	((int)(((tc)+2) << 8))
+#define MALLOCX_TCACHE(tc)	((int)(((tc)+2) << @tcache_shift@))
 #define MALLOCX_TCACHE_NONE	MALLOCX_TCACHE(-1)
 /*
  * Bias arena index bits so that 0 encodes "use an automatically chosen arena".
  */
-#define MALLOCX_ARENA(a)	((((int)(a))+1) << 20)
+#define MALLOCX_ARENA(a)	((((int)(a))+1) << @arena_shift@)
 
 /*
  * Use as arena index in "arena.<i>.{purge,decay,dss}" and
@@ -44,12 +44,12 @@
  *   mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", NULL, NULL, NULL,
  *       0);
  */
-#define MALLCTL_ARENAS_ALL	4096
+#define MALLCTL_ARENAS_ALL	@arenas_all@
 /*
  * Use as arena index in "stats.arenas.<i>.*" mallctl interfaces to select
  * destroyed arenas.
  */
-#define MALLCTL_ARENAS_DESTROYED	4097
+#define MALLCTL_ARENAS_DESTROYED	@arenas_destroyed@
 
 #if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW)
 #  define JEMALLOC_CXX_THROW noexcept (true)

include/jemalloc/jemalloc_protos.h.in

@@ -61,6 +61,7 @@ JEMALLOC_EXPORT void JEMALLOC_NOTHROW	@je_@malloc_stats_print(
     const char *opts);
 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW	@je_@malloc_usable_size(
     JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW;
+JEMALLOC_EXPORT void JEMALLOC_NOTHROW	@je_@thread_cleanup() JEMALLOC_CXX_THROW;
 #ifdef JEMALLOC_HAVE_MALLOC_SIZE
 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW	@je_@malloc_size(
     const void *ptr);

scripts/gen_run_tests.py

@@ -42,6 +42,7 @@ def powerset(items):
     '--disable-stats',
     '--enable-opt-safety-checks',
     '--with-lg-page=16',
+    '--with-lg-tcache-limit=15',
 ]
 if bits_64:
     possible_config_opts.append('--with-lg-vaddr=56')

src/jemalloc.c

@@ -23,6 +23,7 @@
 #include "jemalloc/internal/sz.h"
 #include "jemalloc/internal/ticker.h"
 #include "jemalloc/internal/thread_event.h"
+#include "jemalloc/internal/tsd.h"
 #include "jemalloc/internal/util.h"
 
 #include "jemalloc/internal/conf.h"
@@ -1592,9 +1593,10 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) {
 		 * We should never specify particular arenas or tcaches from
 		 * within our internal allocations.
 		 */
-		assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC
-		    || dopts->tcache_ind == TCACHE_IND_NONE);
-		assert(dopts->arena_ind == ARENA_IND_AUTOMATIC);
+		assert(dopts->tcache_ind == TCACHE_IND_AUTOMATIC ||
+		    dopts->tcache_ind == TCACHE_IND_NONE);
+		assert(dopts->arena_ind == ARENA_IND_AUTOMATIC ||
+		    dopts->arena_ind == 0);
 		dopts->tcache_ind = TCACHE_IND_NONE;
 		/* We know that arena 0 has already been initialized. */
 		dopts->arena_ind = 0;
@@ -3085,6 +3087,13 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) {
 	return ret;
 }
 
+JEMALLOC_EXPORT void JEMALLOC_NOTHROW
+je_thread_cleanup() {
+	#if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32)
+	_malloc_thread_cleanup();
+	#endif
+}
+
 #ifdef JEMALLOC_HAVE_MALLOC_SIZE
 JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
 je_malloc_size(const void *ptr) {

test/unit/background_thread_enable.c

@@ -13,6 +13,13 @@ max_test_narenas(void) {
 	 * approximation.
 	 */
 	unsigned ret = 10 * ncpus;
+	// Limit the max to avoid running out of arenas. There are quite a few
+	// tests where arenas.create is called but arena.<i>.destroy is not, so the
+	// arena remains. Once those are fixed, we should be able to change this to
+	// a larger limit capped at ARENAS_LIMIT.
+	if (ret > 50) {
+		ret = 50;
+	}
 
 	/* Limit the max to avoid VM exhaustion on 32-bit . */
 	return ret > 256 ? 256 : ret;