From 7f4e9c4d6cfc619c0f1f33c39c57fa7f1ed52b3f Mon Sep 17 00:00:00 2001 From: Cedric Koch-Hofer Date: Sat, 27 Sep 2025 00:22:13 +0000 Subject: [PATCH 1/4] DAOS-18857 build: Support of Gperftools Heap Profiler Add a new HEAP_PROFILER SCons boolean option to optionally link DAOS executables with the TCMalloc memory allocator from Gperftools, enabling heap profiling of DAOS processes such as daos_engine. When HEAP_PROFILER=true is set: - DAOS executables are linked with -ltcmalloc. - Google Sanitizers cannot be combined with the heap profiler (enforced at build time with a fatal error). - The tcmalloc linker flag is stripped from unit test environments to avoid scrambling test output. The Gperftools library can also be loaded at runtime via LD_PRELOAD without recompiling, by setting LD_PRELOAD to libtcmalloc.so and the relevant HEAPPROFILE env vars in daos_server.yml. Add gperftools as a build dependency across all supported distributions: - ci/unit/required_packages.sh and utils/scripts/install-*.sh: install gperftools-devel (EL/SUSE) or libgoogle-perftools-dev (Ubuntu). - utils/rpms/package_info.sh: define distro-aware `gperftools_dev` variable for use in FPM packaging. - utils/rpms/daos.sh: add `gperftools_dev` as a build dependency of the daos-devel package. - utils/rpms/daos.spec: retain gperftools-devel BuildRequires (legacy spec path) and bump release to 3. - utils/rpms/daos.changelog: add release 3 entry. Document the Heap Profiler feature and usage in docs/dev/development.md. Signed-off-by: Cedric Koch-Hofer --- ci/unit/required_packages.sh | 1 + docs/dev/development.md | 26 +++++++++++++++++++++++++ site_scons/prereq_tools/base.py | 2 ++ site_scons/site_tools/compiler_setup.py | 13 +++++++++++++ utils/rpms/daos.changelog | 3 +++ utils/rpms/daos.sh | 2 +- utils/rpms/daos.spec | 3 ++- utils/rpms/package_info.sh | 3 +++ utils/scripts/install-el8.sh | 1 + utils/scripts/install-el9.sh | 1 + utils/scripts/install-leap15.sh | 1 + utils/scripts/install-ubuntu.sh | 1 + 12 files changed, 55 insertions(+), 2 deletions(-) diff --git a/ci/unit/required_packages.sh b/ci/unit/required_packages.sh index 4790812779c..7e3dfe6a8a8 100755 --- a/ci/unit/required_packages.sh +++ b/ci/unit/required_packages.sh @@ -36,6 +36,7 @@ pkgs="boost-python3$PY_MINOR_VER-devel \ $(utils/rpms/package_version.sh pmdk debug pmem) \ fuse3 \ gotestsum \ + gperftools-devel \ hwloc-devel \ libasan \ libipmctl-devel \ diff --git a/docs/dev/development.md b/docs/dev/development.md index d267a2ede76..01bf9c43b35 100644 --- a/docs/dev/development.md +++ b/docs/dev/development.md @@ -173,6 +173,32 @@ ASan support in DAOS is still experimental and has the following known issues: ASan is not fully integrated with the DAOS regression testing framework. Some tests, such as those using Valgrind, may fail due to false positives. +### Heap Profiler + +To debug memory leaks which are properly deleted when the application stop, the +[Gperftools Heap Profiler](https://gperftools.github.io/gperftools/heapprofile.html) can be used. +This heap profiler mostly rely on the [TCMalloc](https://github.com/google/tcmalloc) memory +allocator. This memory allocator can either be linked to DAOS executables or loaded at runtime +thanks to `LD_PRELOAD`. + +To profile heap allocation of the `daos_engine` with `LD_PRELOAD`, the following entries can be +added to the `env_vars` section of the `daos_server.yml` configuration file: +```yaml +engines: +- ... + env_vars: + ... + - LD_PRELOAD=/usr/lib64/libtcmalloc.so.4 + - HEAPPROFILE=/var/daos/hprof/daos_engine.1.hprof + - HEAP_PROFILE_INUSE_INTERVAL=1073741824 + - HEAP_PROFILE_ALLOCATION_INTERVAL=0 + - HEAP_PROFILE_TIME_INTERVAL=0 +``` + +To link DAOS executabls with the TCMalloc memory allocator, use the `HEAP_PROFILER=true` flag with +the `scons` command. To profile the daos engien heap allocation, the same `daos_server.yml` +configuration can be used without setting the `LD_PRELOAD` environment variable. + ## Go dependencies Developers contributing Go code may need to change the external dependencies diff --git a/site_scons/prereq_tools/base.py b/site_scons/prereq_tools/base.py index 42b8903f3ad..f19129d195f 100644 --- a/site_scons/prereq_tools/base.py +++ b/site_scons/prereq_tools/base.py @@ -517,6 +517,8 @@ def __init__(self, env, opts): ['warning', 'warn', 'error'], ignorecase=2)) opts.Add(('SANITIZERS', 'Instrument C code with google sanitizers', None)) opts.Add(BoolVariable('CMOCKA_FILTER_SUPPORTED', 'Allows to filter cmocka tests', False)) + opts.Add(BoolVariable('HEAP_PROFILER', 'Instrument C code with Gperftools Heap Profiler', + False)) opts.Update(self.__env) diff --git a/site_scons/site_tools/compiler_setup.py b/site_scons/site_tools/compiler_setup.py index a461bb29373..95bdcd2dad9 100644 --- a/site_scons/site_tools/compiler_setup.py +++ b/site_scons/site_tools/compiler_setup.py @@ -59,6 +59,11 @@ def _base_setup(env): env.AppendIfSupported(CCFLAGS=DESIRED_FLAGS) if 'SANITIZERS' in env and env['SANITIZERS'] != "": + + if "HEAP_PROFILER" in env and env['HEAP_PROFILER']: + print('Google Sanitizers and Gperftools.Heap.Profiler can not be mixed') + Exit(2) + cc = 'gcc' if 'COMPILER' in env: cc = env['COMPILER'] @@ -87,6 +92,10 @@ def _base_setup(env): env.AppendUnique(LINKFLAGS=flag) print(f"Enabling {flag.split('=')[1]} sanitizer for C code") + if 'HEAP_PROFILER' in env and env['HEAP_PROFILER']: + env.AppendUnique(LINKFLAGS="-ltcmalloc") + print("Enabling Gperftools Heap Profiler") + if '-Wmismatched-dealloc' in env['CCFLAGS']: env.AppendUnique(CPPDEFINES={'HAVE_DEALLOC': '1'}) @@ -222,6 +231,10 @@ def _check_func(env, func_name): denv["CCFLAGS"].remove(flag) denv["LINKFLAGS"].remove(flag) + # NOTE Remove Heap Profiler to not scramble the test output + if 'HEAP_PROFILER' in denv and denv['HEAP_PROFILER']: + denv["LINKFLAGS"].remove("-ltcmalloc") + config = Configure(denv) res = config.CheckFunc(func_name) config.Finish() diff --git a/utils/rpms/daos.changelog b/utils/rpms/daos.changelog index 6dc2bedd177..a09625919a1 100644 --- a/utils/rpms/daos.changelog +++ b/utils/rpms/daos.changelog @@ -1,4 +1,7 @@ %changelog +* Mon Apr 21 2026 Cedric Koch-Hofer 2.9.100-3 +- Add gperftools-devel as a build dependency of daos-devel + * Wed Apr 08 2026 Cedric Koch-Hofer 2.9.100-2 - Fix ASAN ODR violation: move src/mgmt/rpc.c into new libdaos_mgmt_crtproto.so shared library to ensure mgmt_proto_fmt_v3/v4 globals are defined exactly diff --git a/utils/rpms/daos.sh b/utils/rpms/daos.sh index 51f8c40e523..ac282d3c53b 100755 --- a/utils/rpms/daos.sh +++ b/utils/rpms/daos.sh @@ -399,7 +399,7 @@ TARGET_PATH="${daoshome}/python" list_files files "${SL_PREFIX}/lib/daos/python/*" append_install_list "${files[@]}" -EXTERNAL_DEPENDS=("${uuid_lib}") +EXTERNAL_DEPENDS=("${uuid_lib}" "${gperftools_dev}") DEPENDS=("daos-client = ${VERSION}-${RELEASE}") build_package "${daos_dev}" diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 27fad1ef97c..4650f723b6f 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -24,7 +24,7 @@ Name: daos Version: 2.9.100 -Release: 2%{?relval}%{?dist} +Release: 3%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent @@ -134,6 +134,7 @@ BuildRequires: libasan %if (0%{?suse_version} > 0) BuildRequires: libasan8 %endif +BuildRequires: gperftools-devel Requires: openssl # This should only be temporary until we can get a stable upstream release diff --git a/utils/rpms/package_info.sh b/utils/rpms/package_info.sh index 5be9fb49ac8..7a78fc2ba4e 100644 --- a/utils/rpms/package_info.sh +++ b/utils/rpms/package_info.sh @@ -129,6 +129,9 @@ export daos_dev set_lib_name uuid lib libuuid libuuid1 libuuid1 export uuid_lib +set_lib_name gperftools dev gperftools gperftools libgoogle-perftools +export gperftools_dev + set_lib_name hdf5 lib hdf5 hdf5 hdf5 export hdf5_lib diff --git a/utils/scripts/install-el8.sh b/utils/scripts/install-el8.sh index 670ef8f6eef..5d75ca060d6 100755 --- a/utils/scripts/install-el8.sh +++ b/utils/scripts/install-el8.sh @@ -37,6 +37,7 @@ dnf --nodocs install ${dnf_install_args} \ git \ glibc-langpack-en \ golang \ + gperftools-devel \ graphviz \ help2man \ hdf5-devel \ diff --git a/utils/scripts/install-el9.sh b/utils/scripts/install-el9.sh index 21980fac63f..3b2a52c059d 100755 --- a/utils/scripts/install-el9.sh +++ b/utils/scripts/install-el9.sh @@ -35,6 +35,7 @@ dnf --nodocs install ${dnf_install_args} \ git \ glibc-langpack-en \ golang \ + gperftools-devel \ graphviz \ help2man \ hdf5-devel \ diff --git a/utils/scripts/install-leap15.sh b/utils/scripts/install-leap15.sh index 5029eb1000a..fb48162ea48 100755 --- a/utils/scripts/install-leap15.sh +++ b/utils/scripts/install-leap15.sh @@ -32,6 +32,7 @@ dnf --nodocs install ${dnf_install_args} \ git \ go \ go-race \ + gperftools-devel \ graphviz \ gzip \ hdf5-devel \ diff --git a/utils/scripts/install-ubuntu.sh b/utils/scripts/install-ubuntu.sh index 0db922daeb7..d2e122055d9 100755 --- a/utils/scripts/install-ubuntu.sh +++ b/utils/scripts/install-ubuntu.sh @@ -37,6 +37,7 @@ apt-get install ${apt_get_install_args} \ libcunit1-dev \ libdaxctl-dev \ libfuse3-dev \ + libgoogle-perftools-dev \ libhwloc-dev \ libibverbs-dev \ libjson-c-dev \ From 30159f63f7c169f5a0ff42332b3b2e1edc158632 Mon Sep 17 00:00:00 2001 From: Cedric Koch-Hofer Date: Mon, 4 May 2026 06:50:44 +0000 Subject: [PATCH 2/4] DAOS-18857 build: Fix merging conflict on release number Increment the DAOS release number. Signed-off-by: Cedric Koch-Hofer --- utils/rpms/daos.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 4650f723b6f..9867e47a56a 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -24,7 +24,7 @@ Name: daos Version: 2.9.100 -Release: 3%{?relval}%{?dist} +Release: 4%{?relval}%{?dist} Summary: DAOS Storage Engine License: BSD-2-Clause-Patent From faa76278d24971dedfd4df43fb975646a5eb8b1a Mon Sep 17 00:00:00 2001 From: Cedric Koch-Hofer Date: Wed, 6 May 2026 08:40:54 +0000 Subject: [PATCH 3/4] DAOS-18847 build: remove deprecated dependency Remove gperftools-devel dependency from deprecated rpm spec file. Signed-off-by: Cedric Koch-Hofer --- utils/rpms/daos.spec | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/rpms/daos.spec b/utils/rpms/daos.spec index 9867e47a56a..8f0309c4823 100644 --- a/utils/rpms/daos.spec +++ b/utils/rpms/daos.spec @@ -134,7 +134,6 @@ BuildRequires: libasan %if (0%{?suse_version} > 0) BuildRequires: libasan8 %endif -BuildRequires: gperftools-devel Requires: openssl # This should only be temporary until we can get a stable upstream release From c47d17be5c2551eafcb731bacd51e9f79673302c Mon Sep 17 00:00:00 2001 From: Cedric Koch-Hofer Date: Thu, 7 May 2026 07:09:39 +0000 Subject: [PATCH 4/4] DAOS-18857 build: Fix RPM changelog date capitalization Fix invalid RPM changelog date: 'may' -> 'May'. Lowercase month abbreviation caused rpmbuild to fail when fpm embedded the changelog into the generated spec file for all SUSE/Leap 15 packages. Signed-off-by: Cedric Koch-Hofer --- utils/rpms/daos.changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/rpms/daos.changelog b/utils/rpms/daos.changelog index a42c0faec36..6ce0f163a31 100644 --- a/utils/rpms/daos.changelog +++ b/utils/rpms/daos.changelog @@ -1,5 +1,5 @@ %changelog -* Mon may 04 2026 Cedric Koch-Hofer 2.9.100-4 +* Mon May 04 2026 Cedric Koch-Hofer 2.9.100-4 - Add gperftools-devel as a build dependency of daos-devel * Fri Apr 24 2026 Tomasz Gromadzki 2.9.100-3