jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-25 02:03:09 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-13479: Patch gperftools to remove 1GB limit on thread caches

Browse Source 
Upstream gperftools does not allow setting tcmalloc.max_total_thread_cache_bytes
to greater than 1GB. This moves to a new toolchain that has patched
gperftools to remove this limitation and allow setting
tcmalloc.max_total_thread_cache_bytes > 1GB. This also reads back the
value from tcmalloc and prints a warning if it doesn't match what we set.

Testing:
 - Set tcmalloc_max_total_thread_cache_bytes to 2GB and verified that
   the warning message doesn't appear. On unpatched versions of
   gperftools, the warning message does appear.

Change-Id: If78c8734c704090c12737a8c2a8456b73ea4b8e8
Reviewed-on: http://gerrit.cloudera.org:8080/22834
Reviewed-by: Michael Smith <michael.smith@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Reviewed-by: Joe McDonnell <joemcdonnell@cloudera.com>
This commit is contained in:
Joe McDonnell
2025-04-29 13:50:14 -07:00
parent 8a4803f895
commit 1157d6e10f
2 changed files with 25 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
be/src/runtime/exec-env.cc
View File

@@ -481,13 +481,32 @@ Status ExecEnv::Init() {
}
#if !defined(ADDRESS_SANITIZER) && !defined(THREAD_SANITIZER)
const static char* TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES =
"tcmalloc.max_total_thread_cache_bytes";
// Change the total TCMalloc thread cache size if necessary.
if (FLAGS_tcmalloc_max_total_thread_cache_bytes > 0 &&
!MallocExtension::instance()->SetNumericProperty(
"tcmalloc.max_total_thread_cache_bytes",
TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES,
FLAGS_tcmalloc_max_total_thread_cache_bytes)) {
return Status("Failed to change TCMalloc total thread cache size.");
return Status(Substitute("Failed to change {0}",
TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES));
}
// Read the value back from tcmalloc to verify it matches what we set.
size_t actual_max_total_thread_cache_bytes = 0;
bool retval = MallocExtension::instance()->GetNumericProperty(
TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES,
&actual_max_total_thread_cache_bytes);
if (!retval) {
return Status(Substitute("Could not retrieve value of {0}.",
TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES));
}
if (actual_max_total_thread_cache_bytes !=
FLAGS_tcmalloc_max_total_thread_cache_bytes) {
LOG(WARNING) << "Set " << TCMALLOC_MAX_TOTAL_THREAD_CACHE_BYTES << " to "
<< FLAGS_tcmalloc_max_total_thread_cache_bytes << " bytes but actually "
<< "using " << actual_max_total_thread_cache_bytes << " bytes.";
}
// A MemTracker for TCMalloc overhead which is the difference between the physical bytes
// reserved (TcmallocMetric::PHYSICAL_BYTES_RESERVED) and the bytes in use
// (TcmallocMetrics::BYTES_IN_USE). This overhead accounts for all the cached freelists

8
bin/impala-config.sh
View File

@@ -81,13 +81,13 @@ export USE_AVRO_CPP=${USE_AVRO_CPP:=false}
# moving to a different build of the toolchain, e.g. when a version is bumped or a
# compile option is changed. The build id can be found in the output of the toolchain
# build jobs, it is constructed from the build number and toolchain git hash prefix.
export IMPALA_TOOLCHAIN_BUILD_ID_AARCH64=63-3a0ac57d41
export IMPALA_TOOLCHAIN_BUILD_ID_X86_64=491-3a0ac57d41
export IMPALA_TOOLCHAIN_BUILD_ID_AARCH64=78-49a326be6f
export IMPALA_TOOLCHAIN_BUILD_ID_X86_64=509-49a326be6f
export IMPALA_TOOLCHAIN_REPO=\
${IMPALA_TOOLCHAIN_REPO:-https://github.com/cloudera/native-toolchain.git}
export IMPALA_TOOLCHAIN_BRANCH=${IMPALA_TOOLCHAIN_BRANCH:-master}
export IMPALA_TOOLCHAIN_COMMIT_HASH=\
${IMPALA_TOOLCHAIN_COMMIT_HASH-810d0f47572d9ac4036c34eb2ad6342d51d510cd}
${IMPALA_TOOLCHAIN_COMMIT_HASH-49a326be6f1836127b1189cea71ea9832b31b7aa}
# Compare the build ref in build IDs by removing everything up-to-and-including the
# first hyphen.
if [ "${IMPALA_TOOLCHAIN_BUILD_ID_AARCH64#*-}" \
@@ -135,7 +135,7 @@ export IMPALA_GFLAGS_VERSION=2.2.0-p2
unset IMPALA_GFLAGS_URL
export IMPALA_GLOG_VERSION=0.3.5-p3
unset IMPALA_GLOG_URL
export IMPALA_GPERFTOOLS_VERSION=2.10
export IMPALA_GPERFTOOLS_VERSION=2.10-p1
unset IMPALA_GPERFTOOLS_URL
export IMPALA_GTEST_VERSION=1.14.0
unset IMPALA_GTEST_URL