IMPALA-12288: Add BUILD_WITH_NO_TESTS option to remove test targets

This patch adds a new option 'BUILD_WITH_NO_TESTS' to tell CMake not
to generate test targets. In order to be consistent with the previous
test workflow, this option is only set ON when building impala using
the 'buildall.sh' script with '-notest' and '-package' flags. This
is useful for a packaging build which do not need to build all test
binaries.

Testing:
  - Ran 'buildall.sh -release -package' with and without '-notests'
flag and verified generated executables.

Change-Id: I575ce76176c9f6a05fd2db0f420ebe6926d0272a
Reviewed-on: http://gerrit.cloudera.org:8080/20294
Reviewed-by: Michael Smith <michael.smith@cloudera.com>
Reviewed-by: Quanlong Huang <huangquanlong@gmail.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

be/CMakeLists.txt

@@ -25,6 +25,8 @@ enable_testing()
 #      to CMakeFiles/<currentdir>.dir/<srcfile>.s
 PROJECT(ASSEMBLER)
 
+option(BUILD_WITH_NO_TESTS "Do not generate test and benchmark targets" OFF)
+
 # compiler flags that are common across debug/release builds
 #  -Wall: Enable all warnings.
 #  -Wno-sign-compare: suppress warnings for comparison between signed and unsigned
@@ -504,7 +506,6 @@ set (IMPALA_LIBS
   security
   Service
   Statestore
-  TestUtil
   ThriftSaslTransport
   token_proto
   Udf
@@ -514,6 +515,10 @@ set (IMPALA_LIBS
   UtilCache
 )
 
+if (NOT BUILD_WITH_NO_TESTS)
+  set(IMPALA_LIBS ${IMPALA_LIBS} TestUtil)
+endif()
+
 set (IMPALA_LINK_LIBS
   ${WL_START_GROUP}
   ${IMPALA_LIBS}
@@ -702,17 +707,19 @@ endif()
 set(LLVM_IR_OUTPUT_DIRECTORY "${CMAKE_SOURCE_DIR}/llvm-ir")
 file(MAKE_DIRECTORY ${LLVM_IR_OUTPUT_DIRECTORY})
 
-# Add custom target to only build the backend tests
-# Note: this specifies "ALL" so it builds if running "make" with no arguments. This is
-# necessary due to the non-executable targets (i.e. generating backend test scripts)
-# that run for the unified backend tests.
-add_custom_target(be-test ALL)
+if (NOT BUILD_WITH_NO_TESTS)
+  # Add custom target to only build the backend tests
+  # Note: this specifies "ALL" so it builds if running "make" with no arguments. This is
+  # necessary due to the non-executable targets (i.e. generating backend test scripts)
+  # that run for the unified backend tests.
+  add_custom_target(be-test ALL)
 
-# Add custom target to build unified backend tests
-add_custom_target(unified-be-test)
+  # Add custom target to build unified backend tests
+  add_custom_target(unified-be-test)
 
-# Add custom target to build the unified backend test executable
-add_custom_target(unified-be-test-executable)
+  # Add custom target to build the unified backend test executable
+  add_custom_target(unified-be-test-executable)
+endif()
 
 # Variable to use to aggregate all of the filter patterns, joined by ":"
 set_property(GLOBAL PROPERTY AGG_UNIFIED_FILTER_PATTERN)
@@ -858,18 +865,20 @@ link_directories(
   ${CMAKE_CURRENT_SOURCE_DIR}/build/transport
 )
 
-# Add custom target to validate the unified backend test executable and test match
-# patterns. At this point, all filter patterns have been aggregated from the individual
-# ADD_UNIFIED_BE_TEST calls into AGG_UNIFIED_FILTER_PATTERN.
-get_property(TOTAL_UNIFIED_FILTER_PATTERN GLOBAL PROPERTY AGG_UNIFIED_FILTER_PATTERN)
-add_custom_target(unified-be-test-validated-executable
-  "${CMAKE_CURRENT_SOURCE_DIR}/../bin/validate-unified-backend-test-filters.py"
-  "-f" "${TOTAL_UNIFIED_FILTER_PATTERN}"
-  "-b" "${BUILD_OUTPUT_ROOT_DIRECTORY}/service/unifiedbetests")
+if (NOT BUILD_WITH_NO_TESTS)
+  # Add custom target to validate the unified backend test executable and test match
+  # patterns. At this point, all filter patterns have been aggregated from the individual
+  # ADD_UNIFIED_BE_TEST calls into AGG_UNIFIED_FILTER_PATTERN.
+  get_property(TOTAL_UNIFIED_FILTER_PATTERN GLOBAL PROPERTY AGG_UNIFIED_FILTER_PATTERN)
+  add_custom_target(unified-be-test-validated-executable
+    "${CMAKE_CURRENT_SOURCE_DIR}/../bin/validate-unified-backend-test-filters.py"
+    "-f" "${TOTAL_UNIFIED_FILTER_PATTERN}"
+    "-b" "${BUILD_OUTPUT_ROOT_DIRECTORY}/service/unifiedbetests")
 
-ADD_DEPENDENCIES(be-test unified-be-test)
-ADD_DEPENDENCIES(unified-be-test unified-be-test-validated-executable)
-ADD_DEPENDENCIES(unified-be-test-validated-executable unified-be-test-executable)
+  ADD_DEPENDENCIES(be-test unified-be-test)
+  ADD_DEPENDENCIES(unified-be-test unified-be-test-validated-executable)
+  ADD_DEPENDENCIES(unified-be-test-validated-executable unified-be-test-executable)
+endif()
 
 # only generate statically linked libs and executables
 set(BUILD_SHARED_LIBS OFF)

be/src/benchmarks/CMakeLists.txt

@@ -15,6 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
 
 # where to put generated libraries
 set(LIBRARY_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/benchmarks")

be/src/catalog/CMakeLists.txt

@@ -26,6 +26,10 @@ add_library(Catalog
 )
 add_dependencies(Catalog gen-deps)
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 add_library(CatalogTests STATIC
   catalog-util-test.cc
 )

be/src/codegen/CMakeLists.txt

@@ -44,11 +44,6 @@ add_library(CodeGen
 )
 add_dependencies(CodeGen gen-deps)
 
-add_library(CodeGenTests STATIC
-  instruction-counter-test.cc
-)
-add_dependencies(CodeGenTests gen-deps)
-
 # output cross compile to ir metadata
 set(IR_DESC_GEN_OUTPUT
   $ENV{IMPALA_HOME}/be/generated-sources/impala-ir/impala-ir-names.h
@@ -102,6 +97,14 @@ COMPILE_TO_IR_C_ARRAY(${IR_O2_C_FILE} impala_llvm_o2_ir -O2 ${PLATFORM_SPECIFIC_
 COMPILE_TO_IR_C_ARRAY(${IR_Os_C_FILE} impala_llvm_os_ir -Os ${PLATFORM_SPECIFIC_FLAGS})
 COMPILE_TO_IR_C_ARRAY(${LEGACY_AVX_IR_C_FILE} impala_legacy_avx_llvm_ir -O1 ${LEGACY_AVX_SPECIFIC_FLAGS})
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
+add_library(CodeGenTests STATIC
+  instruction-counter-test.cc
+)
+add_dependencies(CodeGenTests gen-deps)
 
 # Run the clang compiler to generate BC for llvm-codegen-test
 add_custom_target(test-loop.bc

be/src/common/CMakeLists.txt

@@ -38,12 +38,6 @@ add_library(Common
 )
 add_dependencies(Common gen-deps)
 
-add_library(CommonTests STATIC
-  atomic-test.cc
-  thread-debug-info-test.cc
-)
-add_dependencies(CommonTests gen-deps)
-
 # Command to generate the build version file if not present. We don't automatically
 # regenerate the file if present, which speeds up incremental builds but can lead
 # to the version being stale.
@@ -59,9 +53,19 @@ add_library(GlobalFlags
 )
 add_dependencies(GlobalFlags gen-deps)
 
-ADD_UNIFIED_BE_LSAN_TEST(atomic-test AtomicTest.*)
-ADD_UNIFIED_BE_LSAN_TEST(thread-debug-info-test ThreadDebugInfo.*)
-
 # Generate config.h from config.h.in, filling in variables from CMake
 CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in
     ${CMAKE_CURRENT_BINARY_DIR}/config.h)
+
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
+add_library(CommonTests STATIC
+  atomic-test.cc
+  thread-debug-info-test.cc
+)
+add_dependencies(CommonTests gen-deps)
+
+ADD_UNIFIED_BE_LSAN_TEST(atomic-test AtomicTest.*)
+ADD_UNIFIED_BE_LSAN_TEST(thread-debug-info-test ThreadDebugInfo.*)

be/src/exec/CMakeLists.txt

@@ -105,6 +105,10 @@ add_library(Exec
 
 add_dependencies(Exec gen-deps)
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 add_library(ExecTests STATIC
   acid-metadata-utils-test.cc
   delimited-text-parser-test.cc

be/src/exec/avro/CMakeLists.txt

@@ -33,6 +33,10 @@ add_library(ExecAvro
 
 add_dependencies(ExecAvro gen-deps)
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 add_library(ExecAvroTests STATIC
   hdfs-avro-scanner-test.cc
 )

be/src/exec/parquet/CMakeLists.txt

@@ -43,6 +43,10 @@ add_library(ExecParquet
 
 add_dependencies(ExecParquet gen-deps)
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 add_library(ExecParquetTests STATIC
   hdfs-parquet-scanner-test.cc
   parquet-bool-decoder-test.cc

be/src/experiments/CMakeLists.txt

@@ -26,6 +26,10 @@ add_library(Experiments
 )
 add_dependencies(Experiments gen-deps)
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 add_executable(data-provider-test data-provider-test.cc)
 add_executable(tuple-splitter-test tuple-splitter-test.cc)
 add_executable(hash-partition-test hash-partition-test.cc)

be/src/exprs/CMakeLists.txt

@@ -85,6 +85,10 @@ add_library(Exprs
 )
 add_dependencies(Exprs gen-deps)
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 add_library(ExprsTests STATIC
   datasketches-test.cc
   expr-test.cc

be/src/gutil/CMakeLists.txt

@@ -69,6 +69,10 @@ ADD_EXPORTABLE_LIBRARY(gutil
   # Disable warnings which trigger a lot in the Google code:
   COMPILE_FLAGS "-funsigned-char -Wno-deprecated -Wno-char-subscripts")
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 add_kudu_test(strings/string_util-test)
 
 add_library(GUtilTests STATIC

be/src/rpc/CMakeLists.txt

@@ -42,6 +42,13 @@ add_library(Rpc
 )
 add_dependencies(Rpc gen-deps)
 
+if (BUILD_WITH_NO_TESTS)
+  # thrift-util.cc uses the EXPECT_NO_THROW macro, explicitly linking gtest library
+  # to avoid linker errors.
+  target_link_libraries(Rpc gtest)
+  return()
+endif()
+
 add_library(RpcTests STATIC
   thrift-util-test.cc
 )

be/src/runtime/CMakeLists.txt

@@ -94,6 +94,10 @@ add_library(Runtime
 )
 add_dependencies(Runtime gen-deps)
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 add_library(RuntimeTests STATIC
   coordinator-backend-state-test.cc
   date-test.cc

be/src/runtime/bufferpool/CMakeLists.txt

@@ -31,6 +31,10 @@ add_library(BufferPool
 )
 add_dependencies(BufferPool gen-deps)
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 add_library(BufferPoolTests STATIC
   free-list-test.cc
   suballocator-test.cc

be/src/runtime/io/CMakeLists.txt

@@ -47,6 +47,10 @@ add_library(IoTests STATIC
 )
 add_dependencies(IoTests gen-deps)
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 # This test runs forever so should not be part of 'make test'
 add_executable(disk-io-mgr-stress-test disk-io-mgr-stress-test.cc)
 target_link_libraries(disk-io-mgr-stress-test ${IMPALA_TEST_LINK_LIBS})

be/src/scheduling/CMakeLists.txt

@@ -47,6 +47,10 @@ add_library(Scheduling STATIC
 )
 add_dependencies(Scheduling gen-deps)
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 add_library(SchedulingTests STATIC
   admission-controller-test.cc
   cluster-membership-mgr-test.cc

be/src/service/CMakeLists.txt

@@ -45,13 +45,6 @@ add_library(Service
 )
 add_dependencies(Service gen-deps)
 
-add_library(ServiceTests STATIC
-  hs2-util-test.cc
-  impala-server-test.cc
-  query-options-test.cc
-)
-add_dependencies(ServiceTests gen-deps)
-
 # this shared library provides Impala executor functionality to FE test.
 add_library(fesupport SHARED
   fe-support.cc
@@ -84,10 +77,6 @@ add_executable(impalad
   daemon-main.cc
 )
 
-add_executable(unifiedbetests
-  unified-betest-main.cc
-)
-
 # All Impala daemons run from the same binary. The code that is run is determined by the
 # name (i.e. argv[0]) of the command that executes the binary, so we create symlinks for
 # statestored and catalogd. The symlinks are relative so they can be installed along with
@@ -122,13 +111,28 @@ target_link_libraries(impalad
   ${IMPALA_LINK_LIBS}
 )
 
+install(FILES ${STATESTORED_SYMLINK} DESTINATION ${IMPALA_INSTALLDIR}/bin)
+install(FILES ${CATALOGD_SYMLINK} DESTINATION ${IMPALA_INSTALLDIR}/bin)
+install(TARGETS impalad DESTINATION ${IMPALA_INSTALLDIR}/bin)
+
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
+add_executable(unifiedbetests
+  unified-betest-main.cc
+)
+
 target_link_libraries(unifiedbetests
   ${JAVA_JSIG_LIBRARY} ${UNIFIED_TEST_LINK_LIBS})
 ADD_DEPENDENCIES(unified-be-test-executable unifiedbetests)
 
-install(FILES ${STATESTORED_SYMLINK} DESTINATION ${IMPALA_INSTALLDIR}/bin)
-install(FILES ${CATALOGD_SYMLINK} DESTINATION ${IMPALA_INSTALLDIR}/bin)
-install(TARGETS impalad DESTINATION ${IMPALA_INSTALLDIR}/bin)
+add_library(ServiceTests STATIC
+  hs2-util-test.cc
+  impala-server-test.cc
+  query-options-test.cc
+)
+add_dependencies(ServiceTests gen-deps)
 
 # Exception to unified be tests: Custom main() due to leak
 ADD_BE_TEST(session-expiry-test session-expiry-test.cc) # TODO: this leaks thrift server

be/src/statestore/CMakeLists.txt

@@ -36,4 +36,8 @@ add_library(Statestore
 )
 add_dependencies(Statestore gen-deps)
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 ADD_BE_LSAN_TEST(statestore-test)

be/src/testutil/CMakeLists.txt

@@ -15,6 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 # where to put generated libraries
 set(LIBRARY_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/testutil")
 

be/src/udf/CMakeLists.txt

@@ -40,5 +40,9 @@ add_library(ImpalaUdf ${UDF_SRC_FILES})
 add_dependencies(ImpalaUdf gen-deps)
 set_target_properties(ImpalaUdf PROPERTIES COMPILE_FLAGS "-DIMPALA_UDF_SDK_BUILD")
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 ADD_UDF_TEST(udf-test)
 ADD_UDF_TEST(uda-test)

be/src/udf_samples/CMakeLists.txt

@@ -54,6 +54,10 @@ add_dependencies(udf-sample-ir gen-deps)
 COMPILE_TO_IR(uda-sample.cc )
 add_dependencies(uda-sample-ir gen-deps)
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 # This is an example of how to use the test harness to help develop UDF and UDAs.
 add_executable(udf-sample-test udf-sample-test.cc)
 target_link_libraries(udf-sample-test ImpalaUdf udfsample)

be/src/util/CMakeLists.txt

@@ -145,6 +145,27 @@ endif()
 add_library(Util ${UTIL_SRCS})
 add_dependencies(Util gen-deps)
 
+# Squeasel requires C99 compatibility to build.
+SET_SOURCE_FILES_PROPERTIES(${SQUEASEL_SRC_DIR}/squeasel.c
+  PROPERTIES COMPILE_FLAGS -std=c99)
+
+# shared library which provides native logging support to JVMs over JNI.
+add_library(loggingsupport SHARED
+  logging-support.cc
+)
+
+add_executable(parquet-reader parquet-reader.cc)
+add_executable(impala-profile-tool impala-profile-tool.cc)
+
+target_link_libraries(parquet-reader ${IMPALA_LINK_LIBS})
+target_link_libraries(impala-profile-tool ${IMPALA_LINK_LIBS})
+
+target_link_libraries(loggingsupport ${IMPALA_LINK_LIBS_DYNAMIC_TARGETS})
+
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 add_library(UtilTests STATIC
   benchmark-test.cc
   bitmap-test.cc
@@ -197,23 +218,6 @@ add_library(UtilTests STATIC
 )
 add_dependencies(UtilTests gen-deps)
 
-# Squeasel requires C99 compatibility to build.
-SET_SOURCE_FILES_PROPERTIES(${SQUEASEL_SRC_DIR}/squeasel.c
-  PROPERTIES COMPILE_FLAGS -std=c99)
-
-# shared library which provides native logging support to JVMs over JNI.
-add_library(loggingsupport SHARED
-  logging-support.cc
-)
-
-add_executable(parquet-reader parquet-reader.cc)
-add_executable(impala-profile-tool impala-profile-tool.cc)
-
-target_link_libraries(parquet-reader ${IMPALA_LINK_LIBS})
-target_link_libraries(impala-profile-tool ${IMPALA_LINK_LIBS})
-
-target_link_libraries(loggingsupport ${IMPALA_LINK_LIBS_DYNAMIC_TARGETS})
-
 ADD_UNIFIED_BE_LSAN_TEST(benchmark-test "BenchmarkTest.*")
 ADD_UNIFIED_BE_LSAN_TEST(bitmap-test "Bitmap.*")
 ADD_UNIFIED_BE_LSAN_TEST(bit-packing-test "BitPackingTest.*")

be/src/util/cache/CMakeLists.txt

@@ -28,6 +28,10 @@ add_library(UtilCache
 )
 add_dependencies(UtilCache gen-deps)
 
+if (BUILD_WITH_NO_TESTS)
+  return()
+endif()
+
 add_library(UtilCacheTests STATIC
   cache-test.cc
   lirs-cache-test.cc

buildall.sh

@@ -513,6 +513,9 @@ generate_cmake_files() {
   if [[ $BUILD_SHARED_LIBS -eq 1 ]]; then
     CMAKE_ARGS+=(-DBUILD_SHARED_LIBS=ON)
   fi
+  if [[ "$BUILD_TESTS" -eq 0 && "$GEN_PACKAGE" -eq 1 ]]; then
+    CMAKE_ARGS+=(-DBUILD_WITH_NO_TESTS=ON)
+  fi
   if [[ "${MAKE_CMD}" = "ninja" ]]; then
     CMAKE_ARGS+=(-GNinja)
   fi