IMPALA-5031: make codegen ubsan available by environment variable

bin/jenkins/all-tests.sh does not support any flags when calling
bootstrap_development.sh, which eventually calls buildall.sh. Since
Jenkins scripts are called non-interactively, the type of build is
usually controlled by an environment variable, but that was not
supported for codegen ubsan. This patch makes that possible under the
name "UBSAN_FULL".

Change-Id: Ifd108f8a56158566d95f4769048bc9ab45bd3514
Reviewed-on: http://gerrit.cloudera.org:8080/11742
Reviewed-by: Jim Apple <jbapple-impala@apache.org>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

CMakeLists.txt

@@ -213,6 +213,7 @@ if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG"
     OR "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER"
     OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY"
     OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN"
+    OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN_FULL"
     OR "${CMAKE_BUILD_TYPE}" STREQUAL "TSAN")
   # Use the LLVM libaries with assertions for debug builds.
   set(LLVM_ROOT ${LLVM_DEBUG_ROOT})

be/CMakeLists.txt

@@ -158,6 +158,8 @@ elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "TIDY")
   SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_TIDY}")
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
   SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_UBSAN}")
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN_FULL")
+  SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_UBSAN}")
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "TSAN")
   SET(CMAKE_CXX_FLAGS "${CXX_FLAGS_TSAN}")
 else()
@@ -184,6 +186,7 @@ if (CCACHE AND NOT DEFINED ENV{DISABLE_CCACHE})
   if ("${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER"
       OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY"
       OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN"
+      OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN_FULL"
       OR "${CMAKE_BUILD_TYPE}" STREQUAL "TSAN")
     # Need to set CCACHE_CPP so that ccache calls clang with the original source file for
     # both preprocessing and compilation. Otherwise, ccache will use clang to preprocess
@@ -234,7 +237,7 @@ elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
   SET(CLANG_IR_CXX_FLAGS "${CLANG_IR_CXX_FLAGS}" "-DNDEBUG")
 endif()
 
-if (UBSAN_CODEGEN)
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN_FULL")
   set(CLANG_IR_CXX_FLAGS "${CLANG_IR_CXX_FLAGS}" "-DUNDEFINED_SANITIZER"
     "-fno-omit-frame-pointer" "-fsanitize=undefined" "-fno-wrapv" "-ggdb3"
     "-fno-sanitize=alignment,function,vptr,float-divide-by-zero,float-cast-overflow"
@@ -330,6 +333,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR
     "${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER" OR
     "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN" OR
+    "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN_FULL" OR
     "${CMAKE_BUILD_TYPE}" STREQUAL "TSAN")
   set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/debug/")
 else()

bin/make_impala.sh

@@ -32,7 +32,6 @@ CLEAN=0
 TARGET_BUILD_TYPE=${TARGET_BUILD_TYPE:-""}
 BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS:-""}
 CMAKE_ONLY=0
-UBSAN_CODEGEN=0
 MAKE_CMD=make
 MAKE_ARGS="-j${IMPALA_BUILD_THREADS:-4} ${IMPALA_MAKE_FLAGS}"
 
@@ -81,9 +80,6 @@ do
     -tarballs)
       MAKE_TARGETS+=" tarballs"
       ;;
-    -ubsan_codegen)
-      UBSAN_CODEGEN=1
-      ;;
     -help|*)
       echo "make_impala.sh [-build_type=<build type> -notests -clean]"
       echo "[-build_type] : Target build type. Examples: Debug, Release, Address_sanitizer."
@@ -99,8 +95,6 @@ do
       echo "[-cscope] : Builds cscope metadata."
       echo "[-impala-lzo] : Builds Impala LZO."
       echo "[-tarballs] : Builds additional tarballs like the shell tarball."
-      echo "[-ubsan_codegen] : apply undefined behavior sanitizer to code generated by "\
-      echo "                   cross-compilation to LLVM IT."
       echo ""
       echo "If either -build_type or -build_*_libs is set, cmake will be re-run for the "
       echo "project. Otherwise the last cmake configuration will continue to take effect."
@@ -144,9 +138,6 @@ then
       CMAKE_ARGS+=(-DCMAKE_BUILD_TYPE=${TARGET_BUILD_TYPE})
     fi
 
-    if [ $UBSAN_CODEGEN -eq 1 ]; then
-      CMAKE_ARGS+=(-DUBSAN_CODEGEN=1)
-    fi
     if [ "x${BUILD_SHARED_LIBS}" != "x" ]; then
       CMAKE_ARGS+=(-DBUILD_SHARED_LIBS=${BUILD_SHARED_LIBS})
     fi
@@ -160,6 +151,7 @@ then
       if [[ ("$TARGET_BUILD_TYPE" == "ADDRESS_SANITIZER") \
                 || ("$TARGET_BUILD_TYPE" == "TIDY") \
                 || ("$TARGET_BUILD_TYPE" == "UBSAN") \
+                || ("$TARGET_BUILD_TYPE" == "UBSAN_FULL") \
                 || ("$TARGET_BUILD_TYPE" == "TSAN") ]]
       then
         CMAKE_ARGS+=(-DCMAKE_TOOLCHAIN_FILE=$IMPALA_HOME/cmake_modules/clang_toolchain.cmake)

buildall.sh

@@ -66,6 +66,7 @@ BUILD_ASAN=0
 BUILD_FE_ONLY=0
 BUILD_TIDY=0
 BUILD_UBSAN=0
+BUILD_UBSAN_FULL=0
 BUILD_TSAN=0
 # Export MAKE_CMD so it is visible in scripts that invoke make, e.g. copy-udfs-udas.sh
 export MAKE_CMD=make
@@ -126,8 +127,7 @@ do
       BUILD_UBSAN=1
       ;;
     -full_ubsan)
-      BUILD_UBSAN=1
-      MAKE_IMPALA_ARGS="${MAKE_IMPALA_ARGS} -ubsan_codegen"
+      BUILD_UBSAN_FULL=1
       ;;
     -tsan)
       BUILD_TSAN=1
@@ -290,6 +290,9 @@ fi
 if [[ ${BUILD_UBSAN} -eq 1 ]]; then
   CMAKE_BUILD_TYPE=UBSAN
 fi
+if [[ ${BUILD_UBSAN_FULL} -eq 1 ]]; then
+  CMAKE_BUILD_TYPE=UBSAN_FULL
+fi
 if [[ ${BUILD_TSAN} -eq 1 ]]; then
   CMAKE_BUILD_TYPE=TSAN
 fi

tests/common/environ.py

@@ -56,9 +56,11 @@ class SpecificImpaladBuildTypes:
   TSAN = 'tsan'
   # ./buildall.sh -ubsan
   UBSAN = 'ubsan'
+  # ./buildall.sh -full_ubsan
+  UBSAN_FULL = 'ubsan_full'
 
   VALID_BUILD_TYPES = [ADDRESS_SANITIZER, DEBUG, CODE_COVERAGE_DEBUG, RELEASE,
-      CODE_COVERAGE_RELEASE, TSAN, UBSAN]
+      CODE_COVERAGE_RELEASE, TSAN, UBSAN, UBSAN_FULL]
 
   @classmethod
   def detect(cls, impala_build_root):