IMPALA-11157: Use native-toolchain hadoop for aarch64

Uses hadoop binaries from native-toolchain for aarch64 rather than
pre-built binaries from an individual's GitHub repo.

Change-Id: Ide5ad327d6ce7c2a6b7d0ec4cf1dd53fef987720
Reviewed-on: http://gerrit.cloudera.org:8080/20737
Reviewed-by: Laszlo Gaal <laszlo.gaal@cloudera.com>
Tested-by: Michael Smith <michael.smith@cloudera.com>

bin/bootstrap_toolchain.py

@@ -485,6 +485,9 @@ def get_toolchain_downloads():
   protobuf_package_clang = ToolchainPackage(
       "protobuf", explicit_version=os.environ.get("IMPALA_PROTOBUF_CLANG_VERSION"))
   toolchain_packages += [protobuf_package_clang]
+  if platform.machine() == 'aarch64':
+    toolchain_packages.append(ToolchainPackage("hadoop",
+        explicit_version=os.environ.get("IMPALA_HADOOP_CLIENT_BINARY_VERSION")))
   # Check whether this platform is supported (or whether a valid custom toolchain
   # has been provided).
   if not try_get_platform_release_label() \

bin/impala-config.sh

@@ -85,13 +85,13 @@ export USE_AVRO_CPP=${USE_AVRO_CPP:=false}
 # moving to a different build of the toolchain, e.g. when a version is bumped or a
 # compile option is changed. The build id can be found in the output of the toolchain
 # build jobs, it is constructed from the build number and toolchain git hash prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID_AARCH64=3-c22e6eb3b9
-export IMPALA_TOOLCHAIN_BUILD_ID_X86_64=377-c22e6eb3b9
+export IMPALA_TOOLCHAIN_BUILD_ID_AARCH64=8-62067ab072
+export IMPALA_TOOLCHAIN_BUILD_ID_X86_64=378-62067ab072
 export IMPALA_TOOLCHAIN_REPO=\
 ${IMPALA_TOOLCHAIN_REPO:-https://github.com/cloudera/native-toolchain.git}
 export IMPALA_TOOLCHAIN_BRANCH=${IMPALA_TOOLCHAIN_BRANCH:-master}
 export IMPALA_TOOLCHAIN_COMMIT_HASH=\
-${IMPALA_TOOLCHAIN_COMMIT_HASH-c22e6eb3b95979bdd7a2ec8f86abb751b439f7a4}
+${IMPALA_TOOLCHAIN_COMMIT_HASH-62067ab072750a6395307499caf9ebaa65cb79b7}
 # Compare the build ref in build IDs by removing everything up-to-and-including the
 # first hyphen.
 if [ "${IMPALA_TOOLCHAIN_BUILD_ID_AARCH64#*-}" \
@@ -101,6 +101,8 @@ if [ "${IMPALA_TOOLCHAIN_BUILD_ID_AARCH64#*-}" \
   exit 1
 fi
 
+export ARCH_NAME=$(uname -p)
+
 # Versions of toolchain dependencies.
 # -----------------------------------
 if $USE_AVRO_CPP; then
@@ -201,6 +203,10 @@ export IMPALA_CLOUDFLAREZLIB_VERSION=9e601a3f37
 unset IMPALA_CLOUDFLAREZLIB_URL
 export IMPALA_CALLONCEHACK_VERSION=1.0.0
 unset IMPALA_CALLONCEHACK_URL
+if [[ $ARCH_NAME == 'aarch64' ]]; then
+  export IMPALA_HADOOP_CLIENT_BINARY_VERSION=3.3.6
+  unset IMPALA_HADOOP_CLIENT_BINARY_URL
+fi
 # Thrift related environment variables.
 # IMPALA_THRIFT_POM_VERSION is used to populate IMPALA_THRIFT_JAVA_VERSION and
 # thrift.version in java/pom.xml.
@@ -261,8 +267,6 @@ export APACHE_HIVE_VERSION=3.1.3
 export APACHE_HIVE_STORAGE_API_VERSION=2.7.0
 export APACHE_OZONE_VERSION=1.3.0
 
-export ARCH_NAME=$(uname -p)
-
 # Java dependencies that are not also runtime components. Declaring versions here allows
 # other branches to override them in impala-config-branch.sh for cleaner patches.
 export IMPALA_BOUNCY_CASTLE_VERSION=1.68

buildall.sh

@@ -443,13 +443,10 @@ bootstrap_dependencies() {
     "$IMPALA_HOME/bin/bootstrap_toolchain.py"
     echo "Toolchain bootstrap complete."
   fi
-  # Download prebuilt Hadoop native binaries for aarch64
+  # Use prebuilt Hadoop native binaries for aarch64
   if [[ "$(uname -p)" = "aarch64" ]]; then
-    if ! [[ -d "$IMPALA_HOME/../hadoopAarch64NativeLibs" ]]; then
-      git clone https://github.com/zhaorenhai/hadoopAarch64NativeLibs \
-        "$IMPALA_HOME/../hadoopAarch64NativeLibs"
-    fi
-    cp $IMPALA_HOME/../hadoopAarch64NativeLibs/lib*  $HADOOP_HOME/lib/native/
+    cp $IMPALA_TOOLCHAIN_PACKAGES_HOME/hadoop-$IMPALA_HADOOP_CLIENT_BINARY_VERSION/lib/* \
+        $HADOOP_HOME/lib/native/
   fi
   if [[ "${USE_APACHE_HIVE}" = true ]]; then
     "$IMPALA_HOME/testdata/bin/patch_hive.sh"