jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-19 09:58:28 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-11157: Switch to hadoop-client build

Browse Source 
The hadoop build only produces client binaries, not a full hadoop build.
The name was therefore misleading, and could not replace the full build
of hadoop required by Impala. Impala's toolchain bootstrap process would
then fail if we tried to include two packages named "hadoop" when
overriding the download URL via IMPALA_HADOOP_URL.

Renames hadoop to hadoop-client to clarify its contents and avoid
conflicts with a full hadoop build.

Change-Id: Ia50b5151e5339b06ae2b623a4b2090ae6708491f
Reviewed-on: http://gerrit.cloudera.org:8080/20779
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Reviewed-by: Joe McDonnell <joemcdonnell@cloudera.com>
Reviewed-by: Quanlong Huang <huangquanlong@gmail.com>
This commit is contained in:
Michael Smith
2023-12-12 19:37:51 +00:00
parent 45682c132f
commit 9bc4d38eb9
3 changed files with 7 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
bin/bootstrap_toolchain.py
View File

@@ -486,8 +486,7 @@ def get_toolchain_downloads():
"protobuf", explicit_version=os.environ.get("IMPALA_PROTOBUF_CLANG_VERSION"))
toolchain_packages += [protobuf_package_clang]
if platform.machine() == 'aarch64':
toolchain_packages.append(ToolchainPackage("hadoop",
explicit_version=os.environ.get("IMPALA_HADOOP_CLIENT_BINARY_VERSION")))
toolchain_packages.append(ToolchainPackage("hadoop-client"))
# Check whether this platform is supported (or whether a valid custom toolchain
# has been provided).
if not try_get_platform_release_label() \

10
bin/impala-config.sh
View File

@@ -85,13 +85,13 @@ export USE_AVRO_CPP=${USE_AVRO_CPP:=false}
# moving to a different build of the toolchain, e.g. when a version is bumped or a
# compile option is changed. The build id can be found in the output of the toolchain
# build jobs, it is constructed from the build number and toolchain git hash prefix.
export IMPALA_TOOLCHAIN_BUILD_ID_AARCH64=8-62067ab072
export IMPALA_TOOLCHAIN_BUILD_ID_X86_64=378-62067ab072
export IMPALA_TOOLCHAIN_BUILD_ID_AARCH64=11-8dbe785e9e
export IMPALA_TOOLCHAIN_BUILD_ID_X86_64=380-8dbe785e9e
export IMPALA_TOOLCHAIN_REPO=\
${IMPALA_TOOLCHAIN_REPO:-https://github.com/cloudera/native-toolchain.git}
export IMPALA_TOOLCHAIN_BRANCH=${IMPALA_TOOLCHAIN_BRANCH:-master}
export IMPALA_TOOLCHAIN_COMMIT_HASH=\
${IMPALA_TOOLCHAIN_COMMIT_HASH-62067ab072750a6395307499caf9ebaa65cb79b7}
${IMPALA_TOOLCHAIN_COMMIT_HASH-8dbe785e9e000c665722a11068f9a4a6b454cfeb}
# Compare the build ref in build IDs by removing everything up-to-and-including the
# first hyphen.
if [ "${IMPALA_TOOLCHAIN_BUILD_ID_AARCH64#*-}" \
@@ -204,8 +204,8 @@ unset IMPALA_CLOUDFLAREZLIB_URL
export IMPALA_CALLONCEHACK_VERSION=1.0.0
unset IMPALA_CALLONCEHACK_URL
if [[ $ARCH_NAME == 'aarch64' ]]; then
export IMPALA_HADOOP_CLIENT_BINARY_VERSION=3.3.6
unset IMPALA_HADOOP_CLIENT_BINARY_URL
export IMPALA_HADOOP_CLIENT_VERSION=3.3.6
unset IMPALA_HADOOP_CLIENT_URL
fi
# Thrift related environment variables.
# IMPALA_THRIFT_POM_VERSION is used to populate IMPALA_THRIFT_JAVA_VERSION and

2
buildall.sh
View File

@@ -445,7 +445,7 @@ bootstrap_dependencies() {
fi
# Use prebuilt Hadoop native binaries for aarch64
if [[ "$(uname -p)" = "aarch64" ]]; then
cp $IMPALA_TOOLCHAIN_PACKAGES_HOME/hadoop-$IMPALA_HADOOP_CLIENT_BINARY_VERSION/lib/* \
cp $IMPALA_TOOLCHAIN_PACKAGES_HOME/hadoop-client-$IMPALA_HADOOP_CLIENT_VERSION/lib/* \
$HADOOP_HOME/lib/native/
fi
if [[ "${USE_APACHE_HIVE}" = true ]]; then