IMPALA-12354: Add aarch64 native-toolchain build

Pre-built toolchains are identified by a TOOLCHAIN_BUILD_ID. This commit
adds an aarch64 (64-bit ARM) native-toolchain build, separate from the
x86_64 native-toolchain build, with its own environment variable set in
impala-config.sh. bootstrap_toolchain.py selects which version to use
based on 'uname -m'.

impala-config.sh also verifies that IMPALA_TOOLCHAIN_BUILD_ID_AARCH64
and IMPALA_TOOLCHAIN_BUILD_ID_X86_64 were produced from the same
native-toolchain ref by checking the 2nd token of the build ID.

Updates package version to include the architecture tag to match how
native-toolchain now names them.

Testing:
- successfully built on ARM, and tests passed (exceptions noted in
  IMPALA-12490)

Change-Id: I9bfa7125dbc647b33041c5572d97b7f7ccad6258
Reviewed-on: http://gerrit.cloudera.org:8080/20519
Reviewed-by: Joe McDonnell <joemcdonnell@cloudera.com>
Tested-by: Michael Smith <michael.smith@cloudera.com>

bin/bootstrap_system.sh

@@ -480,15 +480,6 @@ SET_IMPALA_HOME="export IMPALA_HOME=$(pwd)"
 echo -e "\n$SET_IMPALA_HOME" >> ~/.bashrc
 eval "$SET_IMPALA_HOME"
 
-if [[ $ARCH_NAME == 'aarch64' ]]; then
-  SET_TOOLCHAIN_HOME="export NATIVE_TOOLCHAIN_HOME=${IMPALA_HOME}/../native-toolchain"
-  echo -e "\n$SET_TOOLCHAIN_HOME" >> ~/.bashrc
-  echo -e "\n$SET_TOOLCHAIN_HOME" >> "${IMPALA_HOME}/bin/impala-config-local.sh"
-  eval "$SET_TOOLCHAIN_HOME"
-  # Provide access to ~/.cache on build machines so we can use ccache.
-  sudo chmod 755 ~/.cache
-fi
-
 # Try to prepopulate the m2 directory to save time
 if [[ "${PREPOPULATE_M2_REPOSITORY:-true}" == true ]] ; then
   echo ">>> Populating m2 directory..."

bin/bootstrap_toolchain.py

@@ -253,12 +253,17 @@ class ToolchainPackage(EnvVersionedPackage):
       target_comp = parts[1]
     compiler = get_toolchain_compiler()
     label = get_platform_release_label(release=platform_release).toolchain
-    toolchain_build_id = os.environ["IMPALA_TOOLCHAIN_BUILD_ID"]
+    # Most common return values for machine are x86_64 or aarch64
+    arch = platform.machine()
+    if arch not in ['aarch64', 'x86_64']:
+      raise Exception("Unsupported architecture '{}' for pre-built native-toolchain. "
+          "Fetch and build it locally by setting NATIVE_TOOLCHAIN_HOME".format(arch))
+    toolchain_build_id = os.environ["IMPALA_TOOLCHAIN_BUILD_ID_{}".format(arch.upper())]
     toolchain_host = os.environ["IMPALA_TOOLCHAIN_HOST"]
-    template_subs = {'compiler': compiler, 'label': label,
+    template_subs = {'compiler': compiler, 'label': label, 'arch': arch,
                      'toolchain_build_id': toolchain_build_id,
                      'toolchain_host': toolchain_host}
-    archive_basename_tmpl = "${name}-${version}-${compiler}-${label}"
+    archive_basename_tmpl = "${name}-${version}-${compiler}-${label}-${arch}"
     url_prefix_tmpl = "https://${toolchain_host}/build/${toolchain_build_id}/" + \
         "${name}/${version}-${compiler}/"
     unpack_directory_tmpl = "${name}-${version}"

bin/impala-config.sh

@@ -85,12 +85,22 @@ export USE_AVRO_CPP=${USE_AVRO_CPP:=false}
 # moving to a different build of the toolchain, e.g. when a version is bumped or a
 # compile option is changed. The build id can be found in the output of the toolchain
 # build jobs, it is constructed from the build number and toolchain git hash prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID=358-e7cfab15d3
+export IMPALA_TOOLCHAIN_BUILD_ID_AARCH64=1-244b88b440
+export IMPALA_TOOLCHAIN_BUILD_ID_X86_64=375-244b88b440
 export IMPALA_TOOLCHAIN_REPO=\
 ${IMPALA_TOOLCHAIN_REPO:-https://github.com/cloudera/native-toolchain.git}
 export IMPALA_TOOLCHAIN_BRANCH=${IMPALA_TOOLCHAIN_BRANCH:-master}
 export IMPALA_TOOLCHAIN_COMMIT_HASH=\
-${IMPALA_TOOLCHAIN_COMMIT_HASH-e7cfab15d36ae051747252b676f0a11a9c58fe05}
+${IMPALA_TOOLCHAIN_COMMIT_HASH-244b88b440d10d28510312caaa8d134a916d3ce5}
+# Compare the build ref in build IDs by removing everything up-to-and-including the
+# first hyphen.
+if [ "${IMPALA_TOOLCHAIN_BUILD_ID_AARCH64#*-}" \
+    != "${IMPALA_TOOLCHAIN_BUILD_ID_X86_64#*-}" ]; then
+  echo "IMPALA_TOOLCHAIN_BUILD_ID_AARCH64 and IMPALA_TOOLCHAIN_BUILD_ID_X86_64 must" \
+    "come from the same commit hash."
+  exit 1
+fi
+
 # Versions of toolchain dependencies.
 # -----------------------------------
 if $USE_AVRO_CPP; then

buildall.sh

@@ -424,7 +424,7 @@ bootstrap_dependencies() {
         git remote add toolchain "${IMPALA_TOOLCHAIN_REPO}"
         git fetch toolchain "${IMPALA_TOOLCHAIN_BRANCH}"
         # Specifying a branch avoids a large message from git about detached HEADs.
-        git checkout "${IMPALA_TOOLCHAIN_COMMIT_HASH}" -b "${IMPALA_TOOLCHAIN_BUILD_ID}"
+        git checkout "${IMPALA_TOOLCHAIN_COMMIT_HASH}" -b "${IMPALA_TOOLCHAIN_BRANCH}"
       else
         pushd "${NATIVE_TOOLCHAIN_HOME}"
       fi