From 0098113d9582c3b93044eed4078a7f0724dda26f Mon Sep 17 00:00:00 2001 From: zhaorenhai Date: Wed, 3 Jun 2020 09:16:56 +0800 Subject: [PATCH] IMPALA-10090: Create aarch64 development environment on ubuntu 18.04 Including following changes: 1 build native-toolchain local by script on aarch64 platform 2 change some native-toolchain's lib version number 3 split SKIP_TOOLCHAIN_BOOTSTRAP and DOWNLOAD_CDH_COMPONETS to two things, because on aarch64, just need to download cdp components , but not need to download toolchain. 4 download hadoop aarch64 nativelibs , impala building needs these libs. With this commit, on ubuntu 18.04 aarch64 version, just need to run bin/bootstrap_development.sh, just like x86. Change-Id: I769668c834ab0dd504a822ed9153186778275d59 Reviewed-on: http://gerrit.cloudera.org:8080/16065 Reviewed-by: Tim Armstrong Tested-by: Impala Public Jenkins --- bin/bootstrap_system.sh | 36 +++++++++++++++++++++++++++++++++--- bin/bootstrap_toolchain.py | 7 +++++-- bin/impala-config.sh | 12 +++++++----- buildall.sh | 19 ++++++++++++++++++- 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh index 305c60b13..2eca95f81 100755 --- a/bin/bootstrap_system.sh +++ b/bin/bootstrap_system.sh @@ -216,6 +216,12 @@ ubuntu apt-get --yes install ccache curl gawk g++ gcc libffi-dev \ python-dev python-setuptools postgresql ssh wget vim-common psmisc \ lsof openjdk-8-jdk openjdk-8-source openjdk-8-dbg apt-utils git ant +ARCH_NAME=$(uname -p) +if [[ $ARCH_NAME == 'aarch64' ]]; then + ubuntu apt-get --yes install unzip pkg-config flex maven python3-pip build-essential \ + texinfo bison autoconf automake libtool libz-dev libncurses-dev +fi + if [[ "$UBUNTU" == true ]]; then # Don't use openjdk-8-jdk 8u181-b13-1ubuntu0.16.04.1 which is known to break the # surefire tests. If we detect that version, we downgrade to the last known good one. @@ -240,7 +246,6 @@ fi # Ubuntu 18.04 or 20.04 install OpenJDK 11 and configure it as the default Java version. # Impala is currently tested with OpenJDK 8, so configure that version as the default. -ARCH_NAME=$(uname -p) if [[ $ARCH_NAME == 'aarch64' ]]; then ubuntu20 sudo update-java-alternatives -s java-1.8.0-openjdk-arm64 ubuntu18 sudo update-java-alternatives -s java-1.8.0-openjdk-arm64 @@ -378,7 +383,14 @@ then sudo -u postgres psql -c "CREATE ROLE hiveuser LOGIN PASSWORD 'password';" fi sudo -u postgres psql -c "ALTER ROLE hiveuser WITH CREATEDB;" -sudo -u postgres psql -c "SELECT * FROM pg_roles WHERE rolname = 'hiveuser';" +# On Ubuntu 18.04 aarch64 version, the sql 'select * from pg_roles' blocked, +# because output of 'select *' is too long to display in 1 line. +# So here just change it to 'select count(*)' as a work around. +if [[ $ARCH_NAME == 'aarch64' ]]; then + sudo -u postgres psql -c "SELECT count(*) FROM pg_roles WHERE rolname = 'hiveuser';" +else + sudo -u postgres psql -c "SELECT * FROM pg_roles WHERE rolname = 'hiveuser';" +fi # Setup ssh to ssh to localhost mkdir -p ~/.ssh @@ -461,10 +473,28 @@ fi echo -e "\n$SET_JAVA_HOME" >> "${IMPALA_HOME}/bin/impala-config-local.sh" eval "$SET_JAVA_HOME" - # Assert that we have a java available test -f $JAVA_HOME/bin/java +if [[ $ARCH_NAME == 'aarch64' ]]; then + echo -e "\nexport SKIP_TOOLCHAIN_BOOTSTRAP=true" >> \ + "${IMPALA_HOME}/bin/impala-config-local.sh" + SET_TOOLCHAIN_HOME="export NATIVE_TOOLCHAIN_HOME=${IMPALA_HOME}/../native-toolchain" + echo -e "\n$SET_TOOLCHAIN_HOME" >> ~/.bashrc + echo -e "\n$SET_TOOLCHAIN_HOME" >> "${IMPALA_HOME}/bin/impala-config-local.sh" + eval "$SET_TOOLCHAIN_HOME" + if ! [[ -d "$NATIVE_TOOLCHAIN_HOME" ]]; then + time -p git clone https://github.com/cloudera/native-toolchain/ \ + "$NATIVE_TOOLCHAIN_HOME" + fi + cd "$NATIVE_TOOLCHAIN_HOME" + echo "Begin build tool chain, may need several hours, please be patient...." + sudo chmod 755 ~/.cache + ./buildall.sh + cd - + mkdir -p ${IMPALA_HOME}/toolchain +fi + # Try to prepopulate the m2 directory to save time if ! bin/jenkins/populate_m2_directory.py ; then echo "Failed to prepopulate the m2 directory. Continuing..." diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py index 5d59da10c..71e81d6a4 100755 --- a/bin/bootstrap_toolchain.py +++ b/bin/bootstrap_toolchain.py @@ -57,6 +57,7 @@ import logging import glob import multiprocessing.pool import os +import platform import random import re import shutil @@ -505,11 +506,13 @@ def main(): create_directory_from_env_var("IMPALA_TOOLCHAIN_PACKAGES_HOME") downloads = [] - downloads += get_toolchain_downloads() + if os.getenv("SKIP_TOOLCHAIN_BOOTSTRAP", "false") != "true": + downloads += get_toolchain_downloads() kudu_download = None if os.getenv("DOWNLOAD_CDH_COMPONENTS", "false") == "true": create_directory_from_env_var("CDP_COMPONENTS_HOME") - downloads += get_kudu_downloads() + if platform.processor() != "aarch64": + downloads += get_kudu_downloads() downloads += get_hadoop_downloads() components_needing_download = [d for d in downloads if d.needs_download()] diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 749f0e5ca..e0998c135 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -68,7 +68,7 @@ fi # moving to a different build of the toolchain, e.g. when a version is bumped or a # compile option is changed. The build id can be found in the output of the toolchain # build jobs, it is constructed from the build number and toolchain git hash prefix. -export IMPALA_TOOLCHAIN_BUILD_ID=52-c3fa626d9b +export IMPALA_TOOLCHAIN_BUILD_ID=55-dcf54c8601 # Versions of toolchain dependencies. # ----------------------------------- export IMPALA_AVRO_VERSION=1.7.4-p5 @@ -99,7 +99,7 @@ export IMPALA_GFLAGS_VERSION=2.2.0-p2 unset IMPALA_GFLAGS_URL export IMPALA_GLOG_VERSION=0.3.4-p3 unset IMPALA_GLOG_URL -export IMPALA_GPERFTOOLS_VERSION=2.5-p1 +export IMPALA_GPERFTOOLS_VERSION=2.5-p2 unset IMPALA_GPERFTOOLS_URL export IMPALA_GTEST_VERSION=1.6.0 unset IMPALA_GTEST_URL @@ -107,9 +107,9 @@ export IMPALA_LIBEV_VERSION=4.20 unset IMPALA_LIBEV_URL export IMPALA_LIBUNWIND_VERSION=1.3-rc1-p3 unset IMPALA_LIBUNWIND_URL -export IMPALA_LLVM_VERSION=5.0.1-p2 +export IMPALA_LLVM_VERSION=5.0.1-p3 unset IMPALA_LLVM_URL -export IMPALA_LLVM_ASAN_VERSION=5.0.1-p2 +export IMPALA_LLVM_ASAN_VERSION=5.0.1-p3 unset IMPALA_LLVM_ASAN_URL # LLVM stores some files in subdirectories that are named after what @@ -119,7 +119,7 @@ export IMPALA_LLVM_UBSAN_BASE_VERSION=5.0.1 # Debug builds should use the release+asserts build to get additional coverage. # Don't use the LLVM debug build because the binaries are too large to distribute. -export IMPALA_LLVM_DEBUG_VERSION=5.0.1-asserts-p2 +export IMPALA_LLVM_DEBUG_VERSION=5.0.1-asserts-p3 unset IMPALA_LLVM_DEBUG_URL export IMPALA_LZ4_VERSION=1.7.5 unset IMPALA_LZ4_URL @@ -185,6 +185,8 @@ export CDP_PARQUET_VERSION=1.10.99.7.2.1.0-287 export CDP_RANGER_VERSION=2.0.0.7.2.1.0-287 export CDP_TEZ_VERSION=0.9.1.7.2.1.0-287 +export ARCH_NAME=$(uname -p) + export IMPALA_HUDI_VERSION=0.5.0-incubating export IMPALA_KITE_VERSION=1.0.0-cdh6.x-SNAPSHOT export IMPALA_ORC_JAVA_VERSION=1.6.2 diff --git a/buildall.sh b/buildall.sh index 1a6f69b4a..c97d564c1 100755 --- a/buildall.sh +++ b/buildall.sh @@ -397,6 +397,23 @@ bootstrap_dependencies() { # Populate necessary thirdparty components unless it's set to be skipped. if [[ "${SKIP_TOOLCHAIN_BOOTSTRAP}" = true ]]; then echo "SKIP_TOOLCHAIN_BOOTSTRAP is true, skipping toolchain bootstrap." + if [[ "${DOWNLOAD_CDH_COMPONENTS}" = true ]]; then + echo ">>> Downloading and extracting cdh components." + "$IMPALA_HOME/bin/bootstrap_toolchain.py" + fi + # Create soft link to locally builded native-toolchain on aarch64 + if [[ "$(uname -p)" = "aarch64" ]]; then + mkdir -p $IMPALA_TOOLCHAIN_PACKAGES_HOME + cd "$IMPALA_TOOLCHAIN_PACKAGES_HOME" + ln -f -s ${NATIVE_TOOLCHAIN_HOME}/build/* . + cd - + if ! [[ -d "$IMPALA_HOME/../hadoopAarch64NativeLibs" ]]; then + git clone https://github.com/zhaorenhai/hadoopAarch64NativeLibs \ + "$IMPALA_HOME/../hadoopAarch64NativeLibs" + fi + cp $IMPALA_HOME/../hadoopAarch64NativeLibs/lib* $HADOOP_HOME/lib/native/ + fi + else echo ">>> Downloading and extracting toolchain dependencies." "$IMPALA_HOME/bin/bootstrap_toolchain.py" @@ -466,7 +483,7 @@ generate_cmake_files() { fi # ARM64's L3 cacheline size is different according to CPU vendor's implementations of - # architecture. so here we will let use decide this value. + # architecture. so here we will let user decide this value. # If user defined CACHELINESIZE_AARCH64 in impala-config-local.sh, then we will use that # value, if user did not define it, then we will get the value from OS, if fail, then # we will use the default value 64.