diff --git a/ci/unit/required_packages.sh b/ci/unit/required_packages.sh index 4790812779c..7e3dfe6a8a8 100755 --- a/ci/unit/required_packages.sh +++ b/ci/unit/required_packages.sh @@ -36,6 +36,7 @@ pkgs="boost-python3$PY_MINOR_VER-devel \ $(utils/rpms/package_version.sh pmdk debug pmem) \ fuse3 \ gotestsum \ + gperftools-devel \ hwloc-devel \ libasan \ libipmctl-devel \ diff --git a/docs/dev/development.md b/docs/dev/development.md index d267a2ede76..01bf9c43b35 100644 --- a/docs/dev/development.md +++ b/docs/dev/development.md @@ -173,6 +173,32 @@ ASan support in DAOS is still experimental and has the following known issues: ASan is not fully integrated with the DAOS regression testing framework. Some tests, such as those using Valgrind, may fail due to false positives. +### Heap Profiler + +To debug memory leaks which are properly deleted when the application stop, the +[Gperftools Heap Profiler](https://gperftools.github.io/gperftools/heapprofile.html) can be used. +This heap profiler mostly rely on the [TCMalloc](https://github.com/google/tcmalloc) memory +allocator. This memory allocator can either be linked to DAOS executables or loaded at runtime +thanks to `LD_PRELOAD`. + +To profile heap allocation of the `daos_engine` with `LD_PRELOAD`, the following entries can be +added to the `env_vars` section of the `daos_server.yml` configuration file: +```yaml +engines: +- ... + env_vars: + ... + - LD_PRELOAD=/usr/lib64/libtcmalloc.so.4 + - HEAPPROFILE=/var/daos/hprof/daos_engine.1.hprof + - HEAP_PROFILE_INUSE_INTERVAL=1073741824 + - HEAP_PROFILE_ALLOCATION_INTERVAL=0 + - HEAP_PROFILE_TIME_INTERVAL=0 +``` + +To link DAOS executabls with the TCMalloc memory allocator, use the `HEAP_PROFILER=true` flag with +the `scons` command. To profile the daos engien heap allocation, the same `daos_server.yml` +configuration can be used without setting the `LD_PRELOAD` environment variable. + ## Go dependencies Developers contributing Go code may need to change the external dependencies diff --git a/site_scons/prereq_tools/base.py b/site_scons/prereq_tools/base.py index 42b8903f3ad..f19129d195f 100644 --- a/site_scons/prereq_tools/base.py +++ b/site_scons/prereq_tools/base.py @@ -517,6 +517,8 @@ def __init__(self, env, opts): ['warning', 'warn', 'error'], ignorecase=2)) opts.Add(('SANITIZERS', 'Instrument C code with google sanitizers', None)) opts.Add(BoolVariable('CMOCKA_FILTER_SUPPORTED', 'Allows to filter cmocka tests', False)) + opts.Add(BoolVariable('HEAP_PROFILER', 'Instrument C code with Gperftools Heap Profiler', + False)) opts.Update(self.__env) diff --git a/site_scons/site_tools/compiler_setup.py b/site_scons/site_tools/compiler_setup.py index a461bb29373..95bdcd2dad9 100644 --- a/site_scons/site_tools/compiler_setup.py +++ b/site_scons/site_tools/compiler_setup.py @@ -59,6 +59,11 @@ def _base_setup(env): env.AppendIfSupported(CCFLAGS=DESIRED_FLAGS) if 'SANITIZERS' in env and env['SANITIZERS'] != "": + + if "HEAP_PROFILER" in env and env['HEAP_PROFILER']: + print('Google Sanitizers and Gperftools.Heap.Profiler can not be mixed') + Exit(2) + cc = 'gcc' if 'COMPILER' in env: cc = env['COMPILER'] @@ -87,6 +92,10 @@ def _base_setup(env): env.AppendUnique(LINKFLAGS=flag) print(f"Enabling {flag.split('=')[1]} sanitizer for C code") + if 'HEAP_PROFILER' in env and env['HEAP_PROFILER']: + env.AppendUnique(LINKFLAGS="-ltcmalloc") + print("Enabling Gperftools Heap Profiler") + if '-Wmismatched-dealloc' in env['CCFLAGS']: env.AppendUnique(CPPDEFINES={'HAVE_DEALLOC': '1'}) @@ -222,6 +231,10 @@ def _check_func(env, func_name): denv["CCFLAGS"].remove(flag) denv["LINKFLAGS"].remove(flag) + # NOTE Remove Heap Profiler to not scramble the test output + if 'HEAP_PROFILER' in denv and denv['HEAP_PROFILER']: + denv["LINKFLAGS"].remove("-ltcmalloc") + config = Configure(denv) res = config.CheckFunc(func_name) config.Finish() diff --git a/utils/rpms/daos.changelog b/utils/rpms/daos.changelog index eb9c56b469c..6ce0f163a31 100644 --- a/utils/rpms/daos.changelog +++ b/utils/rpms/daos.changelog @@ -1,4 +1,7 @@ %changelog +* Mon May 04 2026 Cedric Koch-Hofer 2.9.100-4 +- Add gperftools-devel as a build dependency of daos-devel + * Fri Apr 24 2026 Tomasz Gromadzki 2.9.100-3 - Update PMDK to version 2.1.3-2 diff --git a/utils/rpms/daos.sh b/utils/rpms/daos.sh index 51f8c40e523..ac282d3c53b 100755 --- a/utils/rpms/daos.sh +++ b/utils/rpms/daos.sh @@ -399,7 +399,7 @@ TARGET_PATH="${daoshome}/python" list_files files "${SL_PREFIX}/lib/daos/python/*" append_install_list "${files[@]}" -EXTERNAL_DEPENDS=("${uuid_lib}") +EXTERNAL_DEPENDS=("${uuid_lib}" "${gperftools_dev}") DEPENDS=("daos-client = ${VERSION}-${RELEASE}") build_package "${daos_dev}" diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 254d641173e..8f0309c4823 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -24,7 +24,7 @@ Name: daos Version: 2.9.100 -Release: 3%{?relval}%{?dist} +Release: 4%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent diff --git a/utils/rpms/package_info.sh b/utils/rpms/package_info.sh index 5103980c8d0..ce580f56ec9 100644 --- a/utils/rpms/package_info.sh +++ b/utils/rpms/package_info.sh @@ -129,6 +129,9 @@ export daos_dev set_lib_name uuid lib libuuid libuuid1 libuuid1 export uuid_lib +set_lib_name gperftools dev gperftools gperftools libgoogle-perftools +export gperftools_dev + set_lib_name hdf5 lib hdf5 hdf5 hdf5 export hdf5_lib diff --git a/utils/scripts/install-el8.sh b/utils/scripts/install-el8.sh index 670ef8f6eef..5d75ca060d6 100755 --- a/utils/scripts/install-el8.sh +++ b/utils/scripts/install-el8.sh @@ -37,6 +37,7 @@ dnf --nodocs install ${dnf_install_args} \ git \ glibc-langpack-en \ golang \ + gperftools-devel \ graphviz \ help2man \ hdf5-devel \ diff --git a/utils/scripts/install-el9.sh b/utils/scripts/install-el9.sh index 21980fac63f..3b2a52c059d 100755 --- a/utils/scripts/install-el9.sh +++ b/utils/scripts/install-el9.sh @@ -35,6 +35,7 @@ dnf --nodocs install ${dnf_install_args} \ git \ glibc-langpack-en \ golang \ + gperftools-devel \ graphviz \ help2man \ hdf5-devel \ diff --git a/utils/scripts/install-leap15.sh b/utils/scripts/install-leap15.sh index 5029eb1000a..fb48162ea48 100755 --- a/utils/scripts/install-leap15.sh +++ b/utils/scripts/install-leap15.sh @@ -32,6 +32,7 @@ dnf --nodocs install ${dnf_install_args} \ git \ go \ go-race \ + gperftools-devel \ graphviz \ gzip \ hdf5-devel \ diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index 0db922daeb7..d2e122055d9 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -37,6 +37,7 @@ apt-get install ${apt_get_install_args} \ libcunit1-dev \ libdaxctl-dev \ libfuse3-dev \ + libgoogle-perftools-dev \ libhwloc-dev \ libibverbs-dev \ libjson-c-dev \