jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-19 18:12:08 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-5902: add ThreadSanitizer build

Browse Source 
This is sufficient to get Impala to come up and run queries with
thread sanitizer enabled.

I have not triaged or fixed the data races that are reported, that
is left for follow-on work.

Change-Id: I22f8faeefa5e157279c5973fe28bc573b7606d50
Reviewed-on: http://gerrit.cloudera.org:8080/7977
Reviewed-by: Tim Armstrong <tarmstrong@cloudera.com>
Tested-by: Impala Public Jenkins
This commit is contained in:
Tim Armstrong
2017-08-31 23:18:20 -07:00
committed by Impala Public Jenkins
parent be98aaacad
commit b1edaf215e
20 changed files with 105 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@@ -209,7 +209,8 @@ find_package(LlvmBinaries REQUIRED)
if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG"
OR "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER"
OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY"
OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN"
OR "${CMAKE_BUILD_TYPE}" STREQUAL "TSAN")
# Use the LLVM libaries with assertions for debug builds.
set(LLVM_ROOT ${LLVM_DEBUG_ROOT})
endif()

16
be/CMakeLists.txt
View File

@@ -110,6 +110,11 @@ SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -fno-wrapv")
# To ease debugging, turn off all optimizations:
SET(CXX_FLAGS_UBSAN "${CXX_FLAGS_UBSAN} -O0")
# Set the flags to the thread sanitizer, also known as "tsan"
# Turn on sanitizer and debug symbols to get stack traces:
SET(CXX_FLAGS_TSAN "${CXX_CLANG_FLAGS} -O1 -ggdb3 -fno-omit-frame-pointer")
SET(CXX_FLAGS_TSAN "${CXX_FLAGS_TSAN} -fsanitize=thread -DTHREAD_SANITIZER")
SET(CXX_FLAGS_TIDY "${CXX_CLANG_FLAGS}")
# Catching unused variables requires an optimization level greater than 0
SET(CXX_FLAGS_TIDY "${CXX_FLAGS_TIDY} -O1")
@@ -136,6 +141,8 @@ elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "TIDY")
SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_TIDY}")
elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_UBSAN}")
elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "TSAN")
SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_TSAN}")
else()
message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
endif()
@@ -159,7 +166,8 @@ if (CCACHE AND NOT DEFINED ENV{DISABLE_CCACHE})
set(RULE_LAUNCH_PREFIX ccache)
if ("${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER"
OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY"
OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN"
OR "${CMAKE_BUILD_TYPE}" STREQUAL "TSAN")
# Need to set CCACHE_CPP so that ccache calls clang with the original source file for
# both preprocessing and compilation. Otherwise, ccache will use clang to preprocess
# the file and then call clang with the preprocessed output if not cached. However,
@@ -294,7 +302,8 @@ add_definitions(-fPIC)
# set compile output directory
if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR
"${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER" OR
"${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
"${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN" OR
"${CMAKE_BUILD_TYPE}" STREQUAL "TSAN")
set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/debug/")
else()
set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/release/")
@@ -436,7 +445,8 @@ set (IMPALA_LINK_LIBS ${IMPALA_LINK_LIBS}
# sanitizer build. Address sanitizer is incompatible with tcmalloc (they both intercept
# malloc/free)
set (IMPALA_LINK_LIBS_NO_TCMALLOC ${IMPALA_LINK_LIBS})
if (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER")
if (NOT "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER" AND
NOT "${CMAKE_BUILD_TYPE}" STREQUAL "TSAN")
set (IMPALA_LINK_LIBS ${IMPALA_LINK_LIBS} tcmallocstatic)
endif()

18
be/src/common/init.cc
View File

@@ -137,17 +137,15 @@ static scoped_ptr<impala::Thread> pause_monitor;
BufferPool* buffer_pool = env->buffer_pool();
if (buffer_pool != nullptr) buffer_pool->Maintenance();
#ifndef ADDRESS_SANITIZER
// When using tcmalloc, the process limit as measured by our trackers will
// be out of sync with the process usage. The metric is refreshed whenever
// memory is consumed or released via a MemTracker, so on a system with
// queries executing it will be refreshed frequently. However if the system
// is idle, we need to refresh the tracker occasionally since untracked
// memory may be allocated or freed, e.g. by background threads.
// The process limit as measured by our trackers may get out of sync with the
// process usage if memory is allocated or freed without updating a MemTracker.
// The metric is refreshed whenever memory is consumed or released via a MemTracker,
// so on a system with queries executing it will be refreshed frequently. However
// if the system is idle, we need to refresh the tracker occasionally since
// untracked memory may be allocated or freed, e.g. by background threads.
if (env->process_mem_tracker() != nullptr) {
env->process_mem_tracker()->RefreshConsumptionFromMetric();
}
#endif
}
// Periodically refresh values of the aggregate memory metrics to ensure they are
// somewhat up-to-date.
@@ -198,7 +196,9 @@ void impala::InitCommonRuntime(int argc, char** argv, bool init_jvm,
impala::InitGoogleLoggingSafe(argv[0]);
// Breakpad needs flags and logging to initialize.
ABORT_IF_ERROR(RegisterMinidump(argv[0]));
#ifndef THREAD_SANITIZER
AtomicOps_x86CPUFeaturesInit();
#endif
impala::InitThreading();
impala::TimestampParser::Init();
impala::SeedOpenSSLRNG();
@@ -243,7 +243,7 @@ void impala::InitCommonRuntime(int argc, char** argv, bool init_jvm,
if (impala::KuduIsAvailable()) impala::InitKuduLogging();
#ifndef ADDRESS_SANITIZER
#if !defined(ADDRESS_SANITIZER) && !defined(THREAD_SANITIZER)
// tcmalloc and address sanitizer can not be used together
if (FLAGS_enable_process_lifetime_heap_profiling) {
HeapProfilerStart(FLAGS_heap_profile_dir.c_str());

18
be/src/gutil/atomicops-internals-tsan.h
View File

@@ -202,6 +202,24 @@ inline Atomic64 Release_CompareAndSwap(volatile Atomic64 *ptr,
return cmp;
}
inline Atomic32 Barrier_CompareAndSwap(volatile Atomic32 *ptr,
Atomic32 old_value,
Atomic32 new_value) {
Atomic32 cmp = old_value;
__tsan_atomic32_compare_exchange_strong(ptr, &cmp, new_value,
__tsan_memory_order_acq_rel, __tsan_memory_order_relaxed);
return cmp;
}
inline Atomic64 Barrier_CompareAndSwap(volatile Atomic64 *ptr,
Atomic64 old_value,
Atomic64 new_value) {
Atomic64 cmp = old_value;
__tsan_atomic64_compare_exchange_strong(ptr, &cmp, new_value,
__tsan_memory_order_acq_rel, __tsan_memory_order_relaxed);
return cmp;
}
inline void MemoryBarrier() {
__tsan_atomic_thread_fence(__tsan_memory_order_seq_cst);
}

2
be/src/runtime/bufferpool/system-allocator.cc
View File

@@ -46,7 +46,7 @@ static int64_t HUGE_PAGE_SIZE = 2LL * 1024 * 1024;
SystemAllocator::SystemAllocator(int64_t min_buffer_len)
: min_buffer_len_(min_buffer_len) {
DCHECK(BitUtil::IsPowerOf2(min_buffer_len));
#ifndef ADDRESS_SANITIZER
#if !defined(ADDRESS_SANITIZER) && !defined(THREAD_SANITIZER)
// Free() assumes that aggressive decommit is enabled for TCMalloc.
size_t aggressive_decommit_enabled;
MallocExtension::instance()->GetNumericProperty(

2
be/src/runtime/exec-env.cc
View File

@@ -294,7 +294,7 @@ Status ExecEnv::StartServices() {
BufferPoolMetric::UNUSED_RESERVATION_BYTES));
obj_pool_->Add(new MemTracker(negated_unused_reservation, -1,
"Buffer Pool: Unused Reservation", mem_tracker_.get()));
#ifndef ADDRESS_SANITIZER
#if !defined(ADDRESS_SANITIZER) && !defined(THREAD_SANITIZER)
// Aggressive decommit is required so that unused pages in the TCMalloc page heap are
// not backed by physical pages and do not contribute towards memory consumption.
size_t aggressive_decommit_enabled = 0;

4
be/src/runtime/query-exec-mgr.cc
View File

@@ -111,8 +111,8 @@ QueryState* QueryExecMgr::GetOrCreateQueryState(
void QueryExecMgr::StartQueryHelper(QueryState* qs) {
qs->StartFInstances();
#ifndef ADDRESS_SANITIZER
// tcmalloc and address sanitizer cannot be used together
#if !defined(ADDRESS_SANITIZER) && !defined(THREAD_SANITIZER)
// tcmalloc and address or thread sanitizer cannot be used together
if (FLAGS_log_mem_usage_interval > 0) {
uint64_t num_complete = ImpaladMetrics::IMPALA_SERVER_NUM_FRAGMENTS->value();
if (num_complete % FLAGS_log_mem_usage_interval == 0) {

6
be/src/util/default-path-handlers.cc
View File

@@ -135,8 +135,8 @@ void MemUsageHandler(MemTracker* mem_tracker, MetricGroup* metric_group,
document->AddMember("consumption", consumption, document->GetAllocator());
stringstream ss;
#ifdef ADDRESS_SANITIZER
ss << "Memory tracking is not available with address sanitizer builds.";
#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER)
ss << "Memory tracking is not available with address or thread sanitizer builds.";
#else
char buf[2048];
MallocExtension::instance()->GetStats(buf, 2048);
@@ -238,7 +238,7 @@ void AddDefaultUrlCallbacks(
webserver->RegisterUrlCallback("/memz", "memz.tmpl", callback);
}
#ifndef ADDRESS_SANITIZER
#if !defined(ADDRESS_SANITIZER) && !defined(THREAD_SANITIZER)
// Remote (on-demand) profiling is disabled if the process is already being profiled.
if (!FLAGS_enable_process_lifetime_heap_profiling) {
AddPprofUrlCallbacks(webserver);

10
be/src/util/memory-metrics.cc
View File

@@ -47,7 +47,7 @@ TcmallocMetric* TcmallocMetric::TOTAL_BYTES_RESERVED = nullptr;
TcmallocMetric* TcmallocMetric::PAGEHEAP_UNMAPPED_BYTES = nullptr;
TcmallocMetric::PhysicalBytesMetric* TcmallocMetric::PHYSICAL_BYTES_RESERVED = nullptr;
AsanMallocMetric* AsanMallocMetric::BYTES_ALLOCATED = nullptr;
SanitizerMallocMetric* SanitizerMallocMetric::BYTES_ALLOCATED = nullptr;
BufferPoolMetric* BufferPoolMetric::LIMIT = nullptr;
BufferPoolMetric* BufferPoolMetric::SYSTEM_ALLOCATED = nullptr;
@@ -81,10 +81,10 @@ Status impala::RegisterMemoryMetrics(MetricGroup* metrics, bool register_jvm_met
used_metrics.push_back(BufferPoolMetric::SYSTEM_ALLOCATED);
}
#ifdef ADDRESS_SANITIZER
AsanMallocMetric::BYTES_ALLOCATED = metrics->RegisterMetric(
new AsanMallocMetric(MetricDefs::Get("asan-total-bytes-allocated")));
used_metrics.push_back(AsanMallocMetric::BYTES_ALLOCATED);
#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER)
SanitizerMallocMetric::BYTES_ALLOCATED = metrics->RegisterMetric(
new SanitizerMallocMetric(MetricDefs::Get("sanitizer-total-bytes-allocated")));
used_metrics.push_back(SanitizerMallocMetric::BYTES_ALLOCATED);
#else
MetricGroup* tcmalloc_metrics = metrics->GetOrCreateChildGroup("tcmalloc");
// We rely on TCMalloc for our global memory metrics, so skip setting them up

16
be/src/util/memory-metrics.h
View File

@@ -23,7 +23,7 @@
#include <boost/bind.hpp>
#include <boost/thread/mutex.hpp>
#include <gperftools/malloc_extension.h>
#ifdef ADDRESS_SANITIZER
#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER)
#include <sanitizer/allocator_interface.h>
#endif
@@ -125,7 +125,7 @@ class TcmallocMetric : public IntGauge {
: IntGauge(def, 0), tcmalloc_var_(tcmalloc_var) { }
virtual void CalculateValue() {
#ifndef ADDRESS_SANITIZER
#if !defined(ADDRESS_SANITIZER) && !defined(THREAD_SANITIZER)
DCHECK_EQ(sizeof(size_t), sizeof(value_));
MallocExtension::instance()->GetNumericProperty(tcmalloc_var_.c_str(),
reinterpret_cast<size_t*>(&value_));
@@ -133,15 +133,15 @@ class TcmallocMetric : public IntGauge {
}
};
/// Alternative to TCMallocMetric if we're running under Address Sanitizer, which
/// does not provide the same metrics.
class AsanMallocMetric : public IntGauge {
/// Alternative to TCMallocMetric if we're running under a sanitizer that replaces
/// malloc(), e.g. address or thread sanitizer.
class SanitizerMallocMetric : public IntGauge {
public:
AsanMallocMetric(const TMetricDef& def) : IntGauge(def, 0) {}
static AsanMallocMetric* BYTES_ALLOCATED;
SanitizerMallocMetric(const TMetricDef& def) : IntGauge(def, 0) {}
static SanitizerMallocMetric* BYTES_ALLOCATED;
private:
virtual void CalculateValue() override {
#ifdef ADDRESS_SANITIZER
#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER)
value_ = __sanitizer_get_current_allocated_bytes();
#endif
}

2
be/src/util/metrics-test.cc
View File

@@ -215,7 +215,7 @@ TEST_F(MetricsTest, StatsMetricsSingle) {
}
TEST_F(MetricsTest, MemMetric) {
#ifndef ADDRESS_SANITIZER
#if !defined(ADDRESS_SANITIZER) && !defined(THREAD_SANITIZER)
MetricGroup metrics("MemMetrics");
ASSERT_OK(RegisterMemoryMetrics(&metrics, false, nullptr, nullptr));
// Smoke test to confirm that tcmalloc metrics are returning reasonable values.

12
be/src/util/pprof-path-handlers.cc
View File

@@ -54,9 +54,9 @@ void PprofCmdLineHandler(const Webserver::ArgumentMap& args, stringstream* outpu
// by calling HeapProfileStart(filename), continue to do work, and then, some number of
// seconds later, call GetHeapProfile() followed by HeapProfilerStop().
void PprofHeapHandler(const Webserver::ArgumentMap& args, stringstream* output) {
#ifdef ADDRESS_SANITIZER
#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER)
(void)PPROF_DEFAULT_SAMPLE_SECS; // Avoid unused variable warning.
(*output) << "Heap profiling is not available with address sanitizer builds.";
(*output) << "Heap profiling is not available with address/thread sanitizer builds.";
#else
Webserver::ArgumentMap::const_iterator it = args.find("seconds");
int seconds = PPROF_DEFAULT_SAMPLE_SECS;
@@ -78,8 +78,8 @@ void PprofHeapHandler(const Webserver::ArgumentMap& args, stringstream* output)
// The server should respond by calling ProfilerStart(), continuing to do its work,
// and then, XX seconds later, calling ProfilerStop().
void PprofCpuProfileHandler(const Webserver::ArgumentMap& args, stringstream* output) {
#ifdef ADDRESS_SANITIZER
(*output) << "CPU profiling is not available with address sanitizer builds.";
#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER)
(*output) << "CPU profiling is not available with address/thread sanitizer builds.";
#else
Webserver::ArgumentMap::const_iterator it = args.find("seconds");
int seconds = PPROF_DEFAULT_SAMPLE_SECS;
@@ -106,8 +106,8 @@ void PprofCpuProfileHandler(const Webserver::ArgumentMap& args, stringstream* ou
// The server should respond by calling:
// MallocExtension::instance()->GetHeapGrowthStacks(&output);
void PprofGrowthHandler(const Webserver::ArgumentMap& args, stringstream* output) {
#ifdef ADDRESS_SANITIZER
(*output) << "Growth profiling is not available with address sanitizer builds.";
#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER)
(*output) << "Growth profiling is not available with address/thread sanitizer builds.";
#else
string heap_growth_stack;
MallocExtension::instance()->GetHeapGrowthStacks(&heap_growth_stack);

3
bin/make_impala.sh
View File

@@ -149,7 +149,8 @@ then
if [[ ("$TARGET_BUILD_TYPE" == "ADDRESS_SANITIZER") \
|| ("$TARGET_BUILD_TYPE" == "TIDY") \
|| ("$TARGET_BUILD_TYPE" == "UBSAN") ]]
|| ("$TARGET_BUILD_TYPE" == "UBSAN") \
|| ("$TARGET_BUILD_TYPE" == "TSAN") ]]
then
CMAKE_ARGS+=(-DCMAKE_TOOLCHAIN_FILE=$IMPALA_HOME/cmake_modules/clang_toolchain.cmake)
else

1
bin/run-backend-tests.sh
View File

@@ -40,5 +40,6 @@ export CTEST_OUTPUT_ON_FAILURE=1
export ASAN_OPTIONS="handle_segv=0 detect_leaks=0 allocator_may_return_null=1"
export UBSAN_OPTIONS="print_stacktrace=1"
UBSAN_OPTIONS="${UBSAN_OPTIONS} suppressions=${IMPALA_HOME}/bin/ubsan-suppressions.txt"
export TSAN_OPTIONS="halt_on_error=1 history_size=7"
export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}"
"${MAKE_CMD:-make}" test ARGS="${BE_TEST_ARGS}"

1
bin/start-catalogd.sh
View File

@@ -73,5 +73,6 @@ fi
export ASAN_OPTIONS="handle_segv=0 detect_leaks=0 allocator_may_return_null=1"
export UBSAN_OPTIONS="print_stacktrace=1"
UBSAN_OPTIONS="${UBSAN_OPTIONS} suppressions=${IMPALA_HOME}/bin/ubsan-suppressions.txt"
export TSAN_OPTIONS="halt_on_error=0 history_size=7"
export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}"
exec ${BINARY_BASE_DIR}/${BUILD_TYPE}/catalog/catalogd ${CATALOGD_ARGS}

1
bin/start-impalad.sh
View File

@@ -101,5 +101,6 @@ fi
export ASAN_OPTIONS="handle_segv=0 detect_leaks=0 allocator_may_return_null=1"
export UBSAN_OPTIONS="print_stacktrace=1"
UBSAN_OPTIONS="${UBSAN_OPTIONS} suppressions=${IMPALA_HOME}/bin/ubsan-suppressions.txt"
export TSAN_OPTIONS="halt_on_error=0 history_size=7"
export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}"
exec ${TOOL_PREFIX} ${IMPALA_CMD} ${IMPALAD_ARGS}

1
bin/start-statestored.sh
View File

@@ -62,5 +62,6 @@ fi
export ASAN_OPTIONS="handle_segv=0 detect_leaks=0 allocator_may_return_null=1"
export UBSAN_OPTIONS="print_stacktrace=1"
UBSAN_OPTIONS="${UBSAN_OPTIONS} suppressions=${IMPALA_HOME}/bin/ubsan-suppressions.txt"
export TSAN_OPTIONS="halt_on_error=0 history_size=7"
export PATH="${IMPALA_TOOLCHAIN}/llvm-${IMPALA_LLVM_VERSION}/bin:${PATH}"
exec ${BINARY_BASE_DIR}/${BUILD_TYPE}/statestore/statestored ${STATESTORED_ARGS}

7
buildall.sh
View File

@@ -60,6 +60,7 @@ BUILD_ASAN=0
BUILD_FE_ONLY=0
BUILD_TIDY=0
BUILD_UBSAN=0
BUILD_TSAN=0
# Export MAKE_CMD so it is visible in scripts that invoke make, e.g. copy-udfs-udas.sh
export MAKE_CMD=make
LZO_CMAKE_ARGS=
@@ -118,6 +119,9 @@ do
-ubsan)
BUILD_UBSAN=1
;;
-tsan)
BUILD_TSAN=1
;;
-testpairwise)
EXPLORATION_STRATEGY=pairwise
;;
@@ -268,6 +272,9 @@ fi
if [[ ${BUILD_UBSAN} -eq 1 ]]; then
CMAKE_BUILD_TYPE=UBSAN
fi
if [[ ${BUILD_TSAN} -eq 1 ]]; then
CMAKE_BUILD_TYPE=TSAN
fi
MAKE_IMPALA_ARGS+=" -build_type=${CMAKE_BUILD_TYPE}"

6
common/thrift/metrics.json
View File

@@ -1114,16 +1114,16 @@
"key": "tcmalloc.total-bytes-reserved"
},
{
"description": "Bytes allocated from Address Sanitizer's malloc (Address Sanitizer debug builds only)",
"description": "Bytes allocated from the sanitizer malloc (Sanitizer debug builds only)",
"contexts": [
"STATESTORE",
"CATALOGSERVER",
"IMPALAD"
],
"label": "Address Sanitizer Malloc Bytes Allocated",
"label": "Sanitizer Malloc Bytes Allocated",
"units": "BYTES",
"kind": "GAUGE",
"key": "asan-total-bytes-allocated"
"key": "sanitizer-total-bytes-allocated"
},
{
"description": "Maximum allowed bytes allocated by the buffer pool.",

28
tests/common/environ.py
View File

@@ -35,7 +35,6 @@ except ImportError as e:
LOG = logging.getLogger('tests.common.environ')
test_start_cluster_args = os.environ.get("TEST_START_CLUSTER_ARGS", "")
# Find the likely BuildType of the running Impala. Assume it's found through the path
@@ -58,8 +57,8 @@ IMPALAD_PATH = os.path.join(impalad_basedir, 'service', 'impalad')
class SpecificImpaladBuildTypes:
"""
Represent a specific build type. In reality, there 5 specific build types. These
specific build types are needed by Python test code.
Represent a specific build type. These specific build types are needed by Python test
code.
The specific build types and their *most distinguishing* compiler options are:
@@ -68,6 +67,7 @@ class SpecificImpaladBuildTypes:
3. DEBUG_CODE_COVERAGE (gcc -ggdb -ftest-coverage)
4. RELEASE (gcc)
5. RELEASE_CODE_COVERAGE (gcc -ftest-coverage)
6. THREAD_SANITIZER (clang -fsanitize=thread)
"""
# ./buildall.sh -asan
ADDRESS_SANITIZER = 'address_sanitizer'
@@ -79,6 +79,8 @@ class SpecificImpaladBuildTypes:
RELEASE = 'release'
# ./buildall.sh -release -codecoverage
RELEASE_CODE_COVERAGE = 'release_code_coverage'
# ./buildall.sh -tsan
THREAD_SANITIZER = 'thread_sanitizer'
class ImpaladBuild(object):
@@ -111,6 +113,12 @@ class ImpaladBuild(object):
"""
return self.specific_build_type == SpecificImpaladBuildTypes.ADDRESS_SANITIZER
def is_tsan(self):
"""
Return whether the Impala under test was compiled with TSAN.
"""
return self.specific_build_type == SpecificImpaladBuildTypes.THREAD_SANITIZER
def is_dev(self):
"""
Return whether the Impala under test is a development build (i.e., any debug or ASAN
@@ -118,14 +126,15 @@ class ImpaladBuild(object):
"""
return self.specific_build_type in (
SpecificImpaladBuildTypes.ADDRESS_SANITIZER, SpecificImpaladBuildTypes.DEBUG,
SpecificImpaladBuildTypes.DEBUG_CODE_COVERAGE)
SpecificImpaladBuildTypes.DEBUG_CODE_COVERAGE,
SpecificImpaladBuildTypes.THREAD_SANITIZER)
def runs_slowly(self):
"""
Return whether the Impala under test "runs slowly". For our purposes this means
either compiled with code coverage enabled or ASAN.
either compiled with code coverage enabled, ASAN or TSAN.
"""
return self.has_code_coverage() or self.is_asan()
return self.has_code_coverage() or self.is_asan() or self.is_tsan()
def _get_impalad_dwarf_info(self):
"""
@@ -170,7 +179,8 @@ class ImpaladBuild(object):
assuming a debug build and log a warning.
"""
ASAN_CU_NAME = 'asan_preinit.cc'
NON_ASAN_CU_NAME = 'daemon-main.cc'
TSAN_CU_NAME = 'tsan_clock.cc'
DEFAULT_CU_NAME = 'daemon-main.cc'
GDB_FLAG = '-ggdb'
CODE_COVERAGE_FLAG = '-ftest-coverage'
@@ -185,7 +195,9 @@ class ImpaladBuild(object):
if die_name.endswith(ASAN_CU_NAME):
specific_build_type = SpecificImpaladBuildTypes.ADDRESS_SANITIZER
elif not die_name.endswith(NON_ASAN_CU_NAME):
if die_name.endswith(TSAN_CU_NAME):
specific_build_type = SpecificImpaladBuildTypes.THREAD_SANITIZER
elif not die_name.endswith(DEFAULT_CU_NAME):
LOG.warn('Unexpected DW_AT_name in first CU: {0}; choosing '
'DEBUG'.format(die_name))
specific_build_type = SpecificImpaladBuildTypes.DEBUG