IMPALA-10090: Create aarch64 development environment on ubuntu 18.04

Including following changes:
1 build native-toolchain local by script on aarch64 platform
2 change some native-toolchain's lib version number
3 split SKIP_TOOLCHAIN_BOOTSTRAP and DOWNLOAD_CDH_COMPONETS to two things,
  because on aarch64, just need to download cdp components ,
  but not need to download toolchain.
4 download hadoop aarch64 nativelibs , impala building needs these libs.

With this commit,  on ubuntu 18.04 aarch64 version,
just need to run bin/bootstrap_development.sh, just like x86.

Change-Id: I769668c834ab0dd504a822ed9153186778275d59
Reviewed-on: http://gerrit.cloudera.org:8080/16065
Reviewed-by: Tim Armstrong <tarmstrong@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

bin/bootstrap_system.sh

@@ -216,6 +216,12 @@ ubuntu apt-get --yes install ccache curl gawk g++ gcc libffi-dev \
         python-dev python-setuptools postgresql ssh wget vim-common psmisc \
         lsof openjdk-8-jdk openjdk-8-source openjdk-8-dbg apt-utils git ant
 
+ARCH_NAME=$(uname -p)
+if [[ $ARCH_NAME == 'aarch64' ]]; then
+  ubuntu apt-get --yes install unzip pkg-config flex maven python3-pip build-essential \
+          texinfo bison autoconf automake libtool libz-dev libncurses-dev
+fi
+
 if [[ "$UBUNTU" == true ]]; then
   # Don't use openjdk-8-jdk 8u181-b13-1ubuntu0.16.04.1 which is known to break the
   # surefire tests. If we detect that version, we downgrade to the last known good one.
@@ -240,7 +246,6 @@ fi
 
 # Ubuntu 18.04 or 20.04 install OpenJDK 11 and configure it as the default Java version.
 # Impala is currently tested with OpenJDK 8, so configure that version as the default.
-ARCH_NAME=$(uname -p)
 if [[ $ARCH_NAME == 'aarch64' ]]; then
     ubuntu20 sudo update-java-alternatives -s java-1.8.0-openjdk-arm64
     ubuntu18 sudo update-java-alternatives -s java-1.8.0-openjdk-arm64
@@ -378,7 +383,14 @@ then
   sudo -u postgres psql -c "CREATE ROLE hiveuser LOGIN PASSWORD 'password';"
 fi
 sudo -u postgres psql -c "ALTER ROLE hiveuser WITH CREATEDB;"
-sudo -u postgres psql -c "SELECT * FROM pg_roles WHERE rolname = 'hiveuser';"
+# On Ubuntu 18.04 aarch64 version, the sql 'select * from pg_roles' blocked,
+# because output of 'select *' is too long to display in 1 line.
+# So here just change it to 'select count(*)' as a work around.
+if [[ $ARCH_NAME == 'aarch64' ]]; then
+  sudo -u postgres psql -c "SELECT count(*) FROM pg_roles WHERE rolname = 'hiveuser';"
+else
+  sudo -u postgres psql -c "SELECT * FROM pg_roles WHERE rolname = 'hiveuser';"
+fi
 
 # Setup ssh to ssh to localhost
 mkdir -p ~/.ssh
@@ -461,10 +473,28 @@ fi
 
 echo -e "\n$SET_JAVA_HOME" >> "${IMPALA_HOME}/bin/impala-config-local.sh"
 eval "$SET_JAVA_HOME"
-
 # Assert that we have a java available
 test -f $JAVA_HOME/bin/java
 
+if [[ $ARCH_NAME == 'aarch64' ]]; then
+  echo -e "\nexport SKIP_TOOLCHAIN_BOOTSTRAP=true" >> \
+    "${IMPALA_HOME}/bin/impala-config-local.sh"
+  SET_TOOLCHAIN_HOME="export NATIVE_TOOLCHAIN_HOME=${IMPALA_HOME}/../native-toolchain"
+  echo -e "\n$SET_TOOLCHAIN_HOME" >> ~/.bashrc
+  echo -e "\n$SET_TOOLCHAIN_HOME" >> "${IMPALA_HOME}/bin/impala-config-local.sh"
+  eval "$SET_TOOLCHAIN_HOME"
+  if ! [[ -d "$NATIVE_TOOLCHAIN_HOME" ]]; then
+    time -p git clone https://github.com/cloudera/native-toolchain/ \
+      "$NATIVE_TOOLCHAIN_HOME"
+  fi
+  cd "$NATIVE_TOOLCHAIN_HOME"
+  echo "Begin build tool chain, may need several hours, please be patient...."
+  sudo chmod 755 ~/.cache
+  ./buildall.sh
+  cd -
+  mkdir -p ${IMPALA_HOME}/toolchain
+fi
+
 # Try to prepopulate the m2 directory to save time
 if ! bin/jenkins/populate_m2_directory.py ; then
   echo "Failed to prepopulate the m2 directory. Continuing..."

bin/bootstrap_toolchain.py

@@ -57,6 +57,7 @@ import logging
 import glob
 import multiprocessing.pool
 import os
+import platform
 import random
 import re
 import shutil
@@ -505,11 +506,13 @@ def main():
   create_directory_from_env_var("IMPALA_TOOLCHAIN_PACKAGES_HOME")
 
   downloads = []
-  downloads += get_toolchain_downloads()
+  if os.getenv("SKIP_TOOLCHAIN_BOOTSTRAP", "false") != "true":
+    downloads += get_toolchain_downloads()
   kudu_download = None
   if os.getenv("DOWNLOAD_CDH_COMPONENTS", "false") == "true":
     create_directory_from_env_var("CDP_COMPONENTS_HOME")
-    downloads += get_kudu_downloads()
+    if platform.processor() != "aarch64":
+      downloads += get_kudu_downloads()
     downloads += get_hadoop_downloads()
 
   components_needing_download = [d for d in downloads if d.needs_download()]

bin/impala-config.sh

@@ -68,7 +68,7 @@ fi
 # moving to a different build of the toolchain, e.g. when a version is bumped or a
 # compile option is changed. The build id can be found in the output of the toolchain
 # build jobs, it is constructed from the build number and toolchain git hash prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID=52-c3fa626d9b
+export IMPALA_TOOLCHAIN_BUILD_ID=55-dcf54c8601
 # Versions of toolchain dependencies.
 # -----------------------------------
 export IMPALA_AVRO_VERSION=1.7.4-p5
@@ -99,7 +99,7 @@ export IMPALA_GFLAGS_VERSION=2.2.0-p2
 unset IMPALA_GFLAGS_URL
 export IMPALA_GLOG_VERSION=0.3.4-p3
 unset IMPALA_GLOG_URL
-export IMPALA_GPERFTOOLS_VERSION=2.5-p1
+export IMPALA_GPERFTOOLS_VERSION=2.5-p2
 unset IMPALA_GPERFTOOLS_URL
 export IMPALA_GTEST_VERSION=1.6.0
 unset IMPALA_GTEST_URL
@@ -107,9 +107,9 @@ export IMPALA_LIBEV_VERSION=4.20
 unset IMPALA_LIBEV_URL
 export IMPALA_LIBUNWIND_VERSION=1.3-rc1-p3
 unset IMPALA_LIBUNWIND_URL
-export IMPALA_LLVM_VERSION=5.0.1-p2
+export IMPALA_LLVM_VERSION=5.0.1-p3
 unset IMPALA_LLVM_URL
-export IMPALA_LLVM_ASAN_VERSION=5.0.1-p2
+export IMPALA_LLVM_ASAN_VERSION=5.0.1-p3
 unset IMPALA_LLVM_ASAN_URL
 
 # LLVM stores some files in subdirectories that are named after what
@@ -119,7 +119,7 @@ export IMPALA_LLVM_UBSAN_BASE_VERSION=5.0.1
 
 # Debug builds should use the release+asserts build to get additional coverage.
 # Don't use the LLVM debug build because the binaries are too large to distribute.
-export IMPALA_LLVM_DEBUG_VERSION=5.0.1-asserts-p2
+export IMPALA_LLVM_DEBUG_VERSION=5.0.1-asserts-p3
 unset IMPALA_LLVM_DEBUG_URL
 export IMPALA_LZ4_VERSION=1.7.5
 unset IMPALA_LZ4_URL
@@ -185,6 +185,8 @@ export CDP_PARQUET_VERSION=1.10.99.7.2.1.0-287
 export CDP_RANGER_VERSION=2.0.0.7.2.1.0-287
 export CDP_TEZ_VERSION=0.9.1.7.2.1.0-287
 
+export ARCH_NAME=$(uname -p)
+
 export IMPALA_HUDI_VERSION=0.5.0-incubating
 export IMPALA_KITE_VERSION=1.0.0-cdh6.x-SNAPSHOT
 export IMPALA_ORC_JAVA_VERSION=1.6.2

buildall.sh

@@ -397,6 +397,23 @@ bootstrap_dependencies() {
   # Populate necessary thirdparty components unless it's set to be skipped.
   if [[ "${SKIP_TOOLCHAIN_BOOTSTRAP}" = true ]]; then
     echo "SKIP_TOOLCHAIN_BOOTSTRAP is true, skipping toolchain bootstrap."
+    if [[ "${DOWNLOAD_CDH_COMPONENTS}" = true ]]; then
+      echo ">>> Downloading and extracting cdh components."
+      "$IMPALA_HOME/bin/bootstrap_toolchain.py"
+    fi
+    # Create soft link to locally builded native-toolchain on aarch64
+    if [[ "$(uname -p)" = "aarch64" ]]; then
+      mkdir -p $IMPALA_TOOLCHAIN_PACKAGES_HOME
+      cd "$IMPALA_TOOLCHAIN_PACKAGES_HOME"
+      ln -f -s ${NATIVE_TOOLCHAIN_HOME}/build/* .
+      cd -
+      if ! [[ -d "$IMPALA_HOME/../hadoopAarch64NativeLibs" ]]; then
+        git clone https://github.com/zhaorenhai/hadoopAarch64NativeLibs \
+          "$IMPALA_HOME/../hadoopAarch64NativeLibs"
+      fi
+      cp $IMPALA_HOME/../hadoopAarch64NativeLibs/lib*  $HADOOP_HOME/lib/native/
+    fi
+
   else
     echo ">>> Downloading and extracting toolchain dependencies."
     "$IMPALA_HOME/bin/bootstrap_toolchain.py"
@@ -466,7 +483,7 @@ generate_cmake_files() {
   fi
 
   # ARM64's L3 cacheline size is different according to CPU vendor's implementations of
-  # architecture. so here we will let use decide this value.
+  # architecture. so here we will let user decide this value.
   # If user defined CACHELINESIZE_AARCH64 in impala-config-local.sh, then we will use that
   # value, if user did not define it, then we will get the value from OS, if fail, then
   # we will use the default value 64.