From 9bc4d38eb9a746a379bd650256487ecef3e8eeaa Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Tue, 12 Dec 2023 19:37:51 +0000 Subject: [PATCH] IMPALA-11157: Switch to hadoop-client build The hadoop build only produces client binaries, not a full hadoop build. The name was therefore misleading, and could not replace the full build of hadoop required by Impala. Impala's toolchain bootstrap process would then fail if we tried to include two packages named "hadoop" when overriding the download URL via IMPALA_HADOOP_URL. Renames hadoop to hadoop-client to clarify its contents and avoid conflicts with a full hadoop build. Change-Id: Ia50b5151e5339b06ae2b623a4b2090ae6708491f Reviewed-on: http://gerrit.cloudera.org:8080/20779 Tested-by: Impala Public Jenkins Reviewed-by: Joe McDonnell Reviewed-by: Quanlong Huang --- bin/bootstrap_toolchain.py | 3 +-- bin/impala-config.sh | 10 +++++----- buildall.sh | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py index 211036001..2162bc1db 100755 --- a/bin/bootstrap_toolchain.py +++ b/bin/bootstrap_toolchain.py @@ -486,8 +486,7 @@ def get_toolchain_downloads(): "protobuf", explicit_version=os.environ.get("IMPALA_PROTOBUF_CLANG_VERSION")) toolchain_packages += [protobuf_package_clang] if platform.machine() == 'aarch64': - toolchain_packages.append(ToolchainPackage("hadoop", - explicit_version=os.environ.get("IMPALA_HADOOP_CLIENT_BINARY_VERSION"))) + toolchain_packages.append(ToolchainPackage("hadoop-client")) # Check whether this platform is supported (or whether a valid custom toolchain # has been provided). if not try_get_platform_release_label() \ diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 817e4977e..7bc23770b 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -85,13 +85,13 @@ export USE_AVRO_CPP=${USE_AVRO_CPP:=false} # moving to a different build of the toolchain, e.g. when a version is bumped or a # compile option is changed. The build id can be found in the output of the toolchain # build jobs, it is constructed from the build number and toolchain git hash prefix. -export IMPALA_TOOLCHAIN_BUILD_ID_AARCH64=8-62067ab072 -export IMPALA_TOOLCHAIN_BUILD_ID_X86_64=378-62067ab072 +export IMPALA_TOOLCHAIN_BUILD_ID_AARCH64=11-8dbe785e9e +export IMPALA_TOOLCHAIN_BUILD_ID_X86_64=380-8dbe785e9e export IMPALA_TOOLCHAIN_REPO=\ ${IMPALA_TOOLCHAIN_REPO:-https://github.com/cloudera/native-toolchain.git} export IMPALA_TOOLCHAIN_BRANCH=${IMPALA_TOOLCHAIN_BRANCH:-master} export IMPALA_TOOLCHAIN_COMMIT_HASH=\ -${IMPALA_TOOLCHAIN_COMMIT_HASH-62067ab072750a6395307499caf9ebaa65cb79b7} +${IMPALA_TOOLCHAIN_COMMIT_HASH-8dbe785e9e000c665722a11068f9a4a6b454cfeb} # Compare the build ref in build IDs by removing everything up-to-and-including the # first hyphen. if [ "${IMPALA_TOOLCHAIN_BUILD_ID_AARCH64#*-}" \ @@ -204,8 +204,8 @@ unset IMPALA_CLOUDFLAREZLIB_URL export IMPALA_CALLONCEHACK_VERSION=1.0.0 unset IMPALA_CALLONCEHACK_URL if [[ $ARCH_NAME == 'aarch64' ]]; then - export IMPALA_HADOOP_CLIENT_BINARY_VERSION=3.3.6 - unset IMPALA_HADOOP_CLIENT_BINARY_URL + export IMPALA_HADOOP_CLIENT_VERSION=3.3.6 + unset IMPALA_HADOOP_CLIENT_URL fi # Thrift related environment variables. # IMPALA_THRIFT_POM_VERSION is used to populate IMPALA_THRIFT_JAVA_VERSION and diff --git a/buildall.sh b/buildall.sh index e39eaa922..41e05faf7 100755 --- a/buildall.sh +++ b/buildall.sh @@ -445,7 +445,7 @@ bootstrap_dependencies() { fi # Use prebuilt Hadoop native binaries for aarch64 if [[ "$(uname -p)" = "aarch64" ]]; then - cp $IMPALA_TOOLCHAIN_PACKAGES_HOME/hadoop-$IMPALA_HADOOP_CLIENT_BINARY_VERSION/lib/* \ + cp $IMPALA_TOOLCHAIN_PACKAGES_HOME/hadoop-client-$IMPALA_HADOOP_CLIENT_VERSION/lib/* \ $HADOOP_HOME/lib/native/ fi if [[ "${USE_APACHE_HIVE}" = true ]]; then