Change the build to allow debug and release builds to coexist.

2025-12-19 18:12:08 -05:00 · 2012-02-17 18:14:04 -08:00
parent e120233fb4
commit 88237350f0
16 changed files with 84 additions and 42 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@ thirdparty
 cscope.files
 cscope.out
 org.eclipse.jdt.core.prefs
+hive_benchmark_results.txt

 pprof.out

--- a/be/.gitignore
+++ b/be/.gitignore
@@ -2,5 +2,4 @@

 build
 generated-sources
-bin/results.txt

--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -5,6 +5,13 @@ cmake_minimum_required(VERSION 2.6)
 # generate CTest input files
 enable_testing()

+# Setting this enables compiling for assembly output.  To compile to assembly:
+#  1. cd into the directory containing the source file
+#  2. 'make help' will list the assembly file targets (i.e. <srcfile.s>
+#  3. 'make <srcfile>.s' to build the assembly for that file.  The file is built 
+#      to CMakeFiles/<currentdir>.dir/<srcfile>.s
+PROJECT(ASSEMBLER)
+
 # compiler flags for different build types (run 'cmake -DCMAKE_BUILD_TYPE=<type> .')
 # For all builds:
 #   -msse4.2: Enable sse4.2 compiler intrinsics.  
@@ -48,6 +55,11 @@ else()
  message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
 endif ()

+# Adding -fverboase-asm creates better annotated assembly.  This
+# doesn't seem to affect when building the binary.
+# TODO: is it possible to have specify different build flags for assembly?
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fverbose-asm")
+
 Message(${CMAKE_CXX_FLAGS})

 # setup doc generation with Doxygen
@@ -88,6 +100,13 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/generated-sources)
 # allow linking of static libs into dynamic lib
 add_definitions(-fPIC)

+# set compile output directory
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+  set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/debug/")
+else()
+  set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/release/")
+endif()
+
 # compile these subdirs using their own CMakeLists.txt
 add_subdirectory(src/common)
 add_subdirectory(src/exec)
@@ -115,8 +134,8 @@ link_directories(
 set(BUILD_SHARED_LIBS OFF)

 # where to put generated libraries
-set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/")
-set(ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/")
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+set(ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")

 # where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/build/")
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
--- a/be/bin/run_hive_benchmark.py
+++ b/be/bin/run_hive_benchmark.py
@@ -8,8 +8,8 @@ import os
 import re
 import sys

-query_cmd = 'build/service/runquery -profile_output_file=""'
-reference_result_file = 'bin/results.txt'
+query_cmd = 'build/release/service/runquery -profile_output_file=""'
+reference_result_file = 'hive_benchmark_results.txt'
 result_single_regex = 'returned (\d*) rows? in (\d*).(\d*) s'
 result_multiple_regex = 'returned (\d*) rows? in (\d*).(\d*) s with stddev (\d*).(\d*)'

@@ -39,7 +39,8 @@ def parse_reference_results():
  try:
    results = {}
    current_result = {}
-    f = open(reference_result_file)
+    full_path = os.environ['IMPALA_HOME'] + "/" + reference_result_file
+    f = open(full_path)
    for line in f:
      line = line.strip()
      if line.startswith("Query:"):
@@ -128,10 +129,10 @@ def run_query(reference_results, output, query, prime_buffer_cache, iterations):
  if reference_avg != 0:
    if avg_time < reference_avg:
      diff = (reference_avg - avg_time) / reference_avg * 100
-      print "  Avg Time: %fs (+%s%.2f%%%s)" % (avg_time, GREEN, diff, END)
+      print "  Avg Time: %fs (-%s%.2f%%%s)" % (avg_time, GREEN, diff, END)
    else:
      diff = (avg_time - reference_avg) / reference_avg * 100
-      print "  Avg Time: %fs (-%s%.2f%%%s)" % (avg_time, RED, diff, END)
+      print "  Avg Time: %fs (+%s%.2f%%%s)" % (avg_time, RED, diff, END)
  else:
    print "  Avg Time: %fs" % (avg_time)

@@ -151,9 +152,9 @@ reference_results = parse_reference_results()
 # flag regressions.  How do we reconcile the fact we are running on different machines?
 queries = [
  ["select count(field) from grep1gb where field like '%xyz%'", 5, 5],
-  ["select pageRank, pageURL from rankings where pageRank > 10 order by pageRank limit 100", 5, 5],
  ["select uv.sourceip, avg(r.pagerank), sum(uv.adrevenue) as totalrevenue from uservisits uv join rankings r on (r.pageurl = uv.desturl) where uv.visitdate > '1999-01-01' and uv.visitdate < '2000-01-01' group by uv.sourceip order by totalrevenue desc limit 1", 5, 5],
  ["select sourceIP, SUM(adRevenue) FROM uservisits GROUP by sourceIP order by SUM(adRevenue) desc limit 10", 5, 5],
+  ["select pageRank, pageURL from rankings where pageRank > 10 order by pageRank limit 100", 0, 5],
  ["select count(field) from grep10gb where field like '%xyz%'", 0, 1]
 ]

@@ -161,5 +162,5 @@ output = ""
 for query in queries:
  output = run_query(reference_results, output, query[0], query[1], query[2])

-print "\nCopy and paste below to %s/%s to update the reference results:" % (os.environ['IMPALA_BE_DIR'], reference_result_file)
+print "\nCopy and paste below to %s/%s to update the reference results:" % (os.environ['IMPALA_HOME'], reference_result_file)
 print output
--- a/be/generated-sources/gen-cpp/CMakeLists.txt
+++ b/be/generated-sources/gen-cpp/CMakeLists.txt
@@ -1,10 +1,10 @@
 # Copyright (c) 2011 Cloudera, Inc. All rights reserved.

 # where to put generated libraries
-set(LIBRARY_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/thrift")
+set(LIBRARY_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/thrift")

 # where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/thrift")
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/thrift")

 set(SRC_FILES
  ImpalaBackendService.cpp
--- a/be/generated-sources/opcode/CMakeLists.txt
+++ b/be/generated-sources/opcode/CMakeLists.txt
@@ -1,10 +1,10 @@
 # Copyright (c) 2011 Cloudera, Inc. All rights reserved.

 # where to put generated libraries
-set(LIBRARY_OUTPUT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/build/opcode")
+set(LIBRARY_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/opcode")

 # where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/build/opcode")
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/opcode")

 set(SRC_FILES
  functions.cc
--- a/be/src/common/CMakeLists.txt
+++ b/be/src/common/CMakeLists.txt
@@ -1,10 +1,11 @@
 # Copyright (c) 2011 Cloudera, Inc. All rights reserved.

+message (STATUS "Common ${BUILD_OUTPUT_ROOT_DIRECTORY}")
 # where to put generated libraries
-set(LIBRARY_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/common")
+set(LIBRARY_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/common")

 # where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/common")
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/common")

 add_library(Common
  status.cc
--- a/be/src/exec/CMakeLists.txt
+++ b/be/src/exec/CMakeLists.txt
@@ -3,10 +3,10 @@
 cmake_minimum_required(VERSION 2.6)

 # where to put generated libraries
-set(LIBRARY_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/exec")
+set(LIBRARY_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/exec")

 # where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/exec")
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/exec")

 add_library(Exec STATIC
  aggregation-node.cc
--- a/be/src/exprs/CMakeLists.txt
+++ b/be/src/exprs/CMakeLists.txt
@@ -1,10 +1,10 @@
 # Copyright (c) 2011 Cloudera, Inc. All rights reserved.

 # where to put generated libraries
-set(LIBRARY_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/exprs")
+set(LIBRARY_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/exprs")

 # where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/exprs")
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/exprs")

 add_library(Exprs
  agg-expr.cc
@@ -58,4 +58,4 @@ target_link_libraries(expr-test
  ${Boost_LIBRARIES}
 )

-add_test(expr-test ${CMAKE_SOURCE_DIR}/be/build/exprs/expr-test)
+add_test(expr-test ${BUILD_OUTPUT_ROOT_DIRECTORY}/exprs/expr-test)
--- a/be/src/runtime/CMakeLists.txt
+++ b/be/src/runtime/CMakeLists.txt
@@ -1,10 +1,10 @@
 # Copyright (c) 2011 Cloudera, Inc. All rights reserved.

 # where to put generated libraries
-set(LIBRARY_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/runtime")
+set(LIBRARY_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/runtime")

 # where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/runtime")
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/runtime")

 # add .cc files and uncomment line
 add_library(Runtime STATIC
@@ -91,7 +91,7 @@ target_link_libraries(data-stream-test
  ${Boost_LIBRARIES}
 )

-add_test(mem-pool-test ${CMAKE_SOURCE_DIR}/be/build/runtime/mem-pool-test)
-add_test(free-list-test ${CMAKE_SOURCE_DIR}/be/build/runtime/free-list-test)
-add_test(string-buffer-test ${CMAKE_SOURCE_DIR}/be/build/runtime/string-buffer-test)
-add_test(data-stream-test ${CMAKE_SOURCE_DIR}/be/build/runtime/data-stream-test)
+add_test(mem-pool-test ${BUILD_OUTPUT_ROOT_DIRECTORY}/runtime/mem-pool-test)
+add_test(free-list-test ${BUILD_OUTPUT_ROOT_DIRECTORY}/runtime/free-list-test)
+add_test(string-buffer-test ${BUILD_OUTPUT_ROOT_DIRECTORY}/runtime/string-buffer-test)
+add_test(data-stream-test ${BUILD_OUTPUT_ROOT_DIRECTORY}/runtime/data-stream-test)
--- a/be/src/service/CMakeLists.txt
+++ b/be/src/service/CMakeLists.txt
@@ -1,10 +1,10 @@
 # Copyright (c) 2011 Cloudera, Inc. All rights reserved.

 # where to put generated libraries
-set(LIBRARY_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/service")
+set(LIBRARY_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/service")

 # where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/service")
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/service")

 add_library(Service
  backend-service.cc
--- a/be/src/testutil/CMakeLists.txt
+++ b/be/src/testutil/CMakeLists.txt
@@ -3,10 +3,10 @@
 cmake_minimum_required(VERSION 2.6)

 # where to put generated libraries
-set(LIBRARY_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/testutil")
+set(LIBRARY_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/testutil")

 # where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/testutil")
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/testutil")

 add_library(TestUtil STATIC
  query-executor.cc  
--- a/be/src/util/CMakeLists.txt
+++ b/be/src/util/CMakeLists.txt
@@ -1,10 +1,10 @@
 # Copyright (c) 2011 Cloudera, Inc. All rights reserved.

 # where to put generated libraries
-set(LIBRARY_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/util")
+set(LIBRARY_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/util")

 # where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/be/build/util")
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/util")

 add_library(Util
  cpu-info.cc
@@ -31,5 +31,5 @@ target_link_libraries(perf-counters-test
  ${Boost_LIBRARIES}
 )

-add_test(perf-counters-test ${CMAKE_SOURCE_DIR}/be/build/util/perf-counters-test)
+add_test(perf-counters-test ${BUILD_OUTPUT_ROOT_DIRECTORY}/util/perf-counters-test)

--- a/bin/make_debug.sh
+++ b/bin/make_debug.sh
@@ -1,3 +1,10 @@
 cd $IMPALA_HOME
 cmake -DCMAKE_BUILD_TYPE=Debug .
-make clean && make
+make clean 
+
+cd $IMPALA_HOME/common/function-registry
+make
+cd $IMPALA_HOME/common/thrift
+make
+cd $IMPALA_BE_DIR
+make -j
--- a/bin/make_release.sh
+++ b/bin/make_release.sh
@@ -3,17 +3,31 @@ cd $IMPALA_HOME

 # build with profile gen enabled
 cmake -DCMAKE_BUILD_TYPE=PROFILE_GEN .
-make clean && make
+make clean 
+cd $IMPALA_HOME/common/function-registry
+make
+cd $IMPALA_HOME/common/thrift
+make
+cd $IMPALA_BE_DIR
+make -j
+cd $IMPALA_HOME

 # Run sample queries - outputs .gcda files 
 # TODO: determine if the PGO output is cumulative.  I couldn't find any documentation about this
-be/build/service/runquery -query="select count(field) from grep1gb where field like '%xyz%'" -profile_output_file=""
-be/build/service/runquery -query="select sourceIP, SUM(adRevenue) FROM uservisits GROUP by sourceIP order by SUM(adRevenue) desc limit 10" -profile_output_file=""
-be/build/service/runquery -query="select uv.sourceip, avg(r.pagerank), sum(uv.adrevenue) as totalrevenue from uservisits uv join rankings r on (r.pageurl = uv.desturl) where uv.visitdate > '1999-01-01' and uv.visitdate < '2000-01-01' group by uv.sourceip order by totalrevenue desc limit 1" -profile_output_file=""
+be/build/release/service/runquery -query="select count(field) from grep1gb where field like '%xyz%'" -profile_output_file=""
+be/build/release/service/runquery -query="select sourceIP, SUM(adRevenue) FROM uservisits GROUP by sourceIP order by SUM(adRevenue) desc limit 10" -profile_output_file=""
+be/build/release/service/runquery -query="select uv.sourceip, avg(r.pagerank), sum(uv.adrevenue) as totalrevenue from uservisits uv join rankings r on (r.pageurl = uv.desturl) where uv.visitdate > '1999-01-01' and uv.visitdate < '2000-01-01' group by uv.sourceip order by totalrevenue desc limit 1" -profile_output_file=""

 # Build again using the PGO data
 cmake -DCMAKE_BUILD_TYPE=PROFILE_BUILD .
-make clean && make
+make clean 
+cd $IMPALA_HOME/common/function-registry
+make
+cd $IMPALA_HOME/common/thrift
+make
+cd $IMPALA_BE_DIR
+make -j
+cd $IMPALA_HOME

 # Remove all the PGO intermediates and set build to release.  This seems to be the most 
 # convenient workflow.  Otherwise, changing one src file after a PGO build causes many warnings 
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -249,7 +249,7 @@
        <configuration>
          <redirectTestOutputToFile>true</redirectTestOutputToFile>
          <systemPropertyVariables>
-	    <java.library.path>${java.library.path}:${project.basedir}/../be/build/service</java.library.path>
+	    <java.library.path>${java.library.path}:${project.basedir}/../be/build/debug/service:${project.basedir}/../be/build/release/service</java.library.path>
          </systemPropertyVariables>
        </configuration>
      </plugin>