From d5ada970c66ddb22aae5fc7282384e62b729031e Mon Sep 17 00:00:00 2001 From: Fredy Wijaya Date: Tue, 31 Jul 2018 20:06:29 -0700 Subject: [PATCH] IMPALA-7381: Prevent build failure after switching to new CDH_BUILD_NUMBER Switching to a new CDH_BUILD_NUMBER requires downloading new CDH components as well as forcing Maven to update its local repository. This patch updates the CDH_COMPONENTS_HOME to include the CDH_BUILD_NUMBER which will automatically download the new CDH components after switching to a new CDH_BUILD_NUMBER. When running a build if it detects that a new CDH_BUILD_NUMBER has changed, the build will force an update to the local Maven repository. This helps to prevent build failure even on a fresh Git clone due to stale local Maven repository. Testing: - Manually tested by running buildall.sh with different CDH_BUILD_NUMBER Change-Id: Ib0ad9c2258663d3bd7470e6df921041d1ca0c0be Reviewed-on: http://gerrit.cloudera.org:8080/11099 Reviewed-by: Impala Public Jenkins Tested-by: Impala Public Jenkins --- .gitignore | 1 + README.md | 3 ++- bin/impala-config.sh | 2 +- buildall.sh | 12 ++++++++++++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 8d8e4e4ae..a39b39ecc 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ org.eclipse.jdt.ui.prefs load-*-generated.sql bin/version.info bin/impala-config-local.sh +.cdh # distcc options .impala_compiler_opts diff --git a/README.md b/README.md index c0565ab31..0d700d7a3 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,8 @@ can do so through the environment variables and scripts listed below. | IMPALA_HOME | | Top level Impala directory | | IMPALA_TOOLCHAIN | "${IMPALA_HOME}/toolchain" | Native toolchain directory (for compilers, libraries, etc.) | | SKIP_TOOLCHAIN_BOOTSTRAP | "false" | Skips downloading the toolchain any python dependencies if "true" | -| CDH_COMPONENTS_HOME | "${IMPALA_HOME}/toolchain/cdh_components" OR "${IMPALA_HOME}/thirdparty" (if detected) | If a thirdparty directory is present, components found here will override anything in IMPALA_TOOLCHAIN. | +| CDH_BUILD_NUMBER | | Identifier to indicate the CDH build number +| CDH_COMPONENTS_HOME | "${IMPALA_HOME}/toolchain/cdh_components-${CDH_BUILD_NUMBER}" OR "${IMPALA_HOME}/thirdparty" (if detected) | If a thirdparty directory is present, components found here will override anything in IMPALA_TOOLCHAIN. | | CDH_MAJOR_VERSION | "5" | Identifier used to uniqueify paths for potentially incompatible component builds. | | IMPALA_CONFIG_SOURCED | "1" | Set by ${IMPALA_HOME}/bin/impala-config.sh (internal use) | | JAVA_HOME | "/usr/lib/jvm/${JAVA_VERSION}" | Used to locate Java | diff --git a/bin/impala-config.sh b/bin/impala-config.sh index c627a0e5b..fe656ac1b 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -475,7 +475,7 @@ export PATH="$IMPALA_HOME/bin:$IMPALA_TOOLCHAIN/cmake-$IMPALA_CMAKE_VERSION/bin/ # The directory in which all the thirdparty CDH components live. if [ "${DOWNLOAD_CDH_COMPONENTS}" = true ]; then - export CDH_COMPONENTS_HOME="$IMPALA_TOOLCHAIN/cdh_components" + export CDH_COMPONENTS_HOME="$IMPALA_TOOLCHAIN/cdh_components-$CDH_BUILD_NUMBER" else export CDH_COMPONENTS_HOME="$IMPALA_HOME/thirdparty" fi diff --git a/buildall.sh b/buildall.sh index a86581a8c..59e27ee7b 100755 --- a/buildall.sh +++ b/buildall.sh @@ -465,6 +465,18 @@ create_log_dirs bootstrap_dependencies +# Create .cdh file that contains the CDH_BUILD_NUMBER. If the content +# of the file is different than the one in the environment variable, +# append -U into IMPALA_MAVEN_OPTION to force Maven to update its local +# cache. +CDH_FILE="${IMPALA_HOME}/.cdh" +if [[ -f ${CDH_FILE} ]]; then + if [[ $(cat ${CDH_FILE}) != ${CDH_BUILD_NUMBER} ]]; then + export IMPALA_MAVEN_OPTIONS="${IMPALA_MAVEN_OPTIONS} -U" + fi +fi +echo "${CDH_BUILD_NUMBER}" > ${CDH_FILE} + if [[ "$BUILD_FE_ONLY" -eq 1 ]]; then build_fe exit 0