diff --git a/README-build.md b/README-build.md index 5ee8b357c..0db523f9b 100644 --- a/README-build.md +++ b/README-build.md @@ -31,8 +31,8 @@ can do so through the environment variables and scripts listed below. | CDP_COMPONENTS_HOME | "${IMPALA_HOME}/toolchain/cdp_components-${CDP_BUILD_NUMBER}" | Location of the CDP components within the toolchain. | | CDH_MAJOR_VERSION | "7" | Identifier used to uniqueify paths for potentially incompatible component builds. | | IMPALA_CONFIG_SOURCED | "1" | Set by ${IMPALA_HOME}/bin/impala-config.sh (internal use) | -| IMPALA_JAVA_HOME_OVERRIDE | | Specify a non-system Java version. Overrides IMPALA_JDK_VERSION behavior. | -| IMPALA_JDK_VERSION | "system" | Set to 8 or 11 to select a system Java version. Default will set JAVA_HOME based on the javac symlink in PATH. | +| IMPALA_JDK_VERSION | "" | Set to 8+ to select a system Java version. Empty value uses JAVA_HOME, or sets it based on system defaults. | +| JAVA_HOME | "" | Uses Java from JAVA_HOME unless IMPALA_JDK_VERSION is set. | | JAVA | "${JAVA_HOME}/bin/java" | Java binary location. | | CLASSPATH | | See bin/set-classpath.sh for details. | | PYTHONPATH | | See bin/set-pythonpath.sh for details. | diff --git a/bin/bootstrap_build.sh b/bin/bootstrap_build.sh index 38b3b8d0a..6ef7d6ce1 100755 --- a/bin/bootstrap_build.sh +++ b/bin/bootstrap_build.sh @@ -66,8 +66,13 @@ if [[ $DISTRIB_ID == Ubuntu && $DISTRIB_RELEASE == 20.04 ]]; then fi JDK_VERSION=17 +if [[ "$(uname -p)" == 'aarch64' ]]; then + PACKAGE_ARCH='arm64' +else + PACKAGE_ARCH='amd64' +fi sudo apt-get --yes --quiet install openjdk-${JDK_VERSION}-jdk openjdk-${JDK_VERSION}-source -export JAVA_HOME=/usr/lib/jvm/java-${JDK_VERSION}-openjdk-amd64 +export JAVA_HOME=/usr/lib/jvm/java-${JDK_VERSION}-openjdk-${PACKAGE_ARCH} # Download Maven since the packaged version is pretty old. : ${IMPALA_TOOLCHAIN_HOST:=native-toolchain.s3.amazonaws.com} diff --git a/bin/impala-config-java.sh b/bin/impala-config-java.sh index 2a4f7f8cd..002f1dbbb 100644 --- a/bin/impala-config-java.sh +++ b/bin/impala-config-java.sh @@ -15,19 +15,16 @@ # specific language governing permissions and limitations # under the License. -IMPALA_JDK_VERSION=${IMPALA_JDK_VERSION:-system} +IMPALA_JDK_VERSION=${IMPALA_JDK_VERSION:-} -# Set OS Java package variables for bootstrap_system and Docker builds -if [[ "${IMPALA_JDK_VERSION}" == "system" || "${IMPALA_JDK_VERSION}" == "8" ]]; then +# Set OS Java package variables for bootstrap_system and Docker builds. +# Defaults to installing Java 8. +if [[ "${IMPALA_JDK_VERSION}" == "" || "${IMPALA_JDK_VERSION}" == "8" ]]; then UBUNTU_JAVA_VERSION=8 REDHAT_JAVA_VERSION=1.8.0 - export IMPALA_JDK_VERSION_NUM=8 - export IMPALA_JAVA_TARGET=1.8 else UBUNTU_JAVA_VERSION="${IMPALA_JDK_VERSION}" REDHAT_JAVA_VERSION="${IMPALA_JDK_VERSION}" - export IMPALA_JDK_VERSION_NUM="${IMPALA_JDK_VERSION}" - export IMPALA_JAVA_TARGET="${IMPALA_JDK_VERSION}" fi if [[ "$(uname -p)" == 'aarch64' ]]; then @@ -35,8 +32,3 @@ if [[ "$(uname -p)" == 'aarch64' ]]; then else UBUNTU_PACKAGE_ARCH='amd64' fi - -echo "JAVA_HOME: ${JAVA_HOME:-}" -echo "IMPALA_JDK_VERSION: $IMPALA_JDK_VERSION" -echo "IMPALA_JDK_VERSION_NUM: $IMPALA_JDK_VERSION_NUM" -echo "IMPALA_JAVA_TARGET: $IMPALA_JAVA_TARGET" diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 36df07267..b4c4ff01f 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -498,27 +498,26 @@ fi # the JDK version as part of bin/impala-config-local.sh # Decision tree: -# if IMPALA_JAVA_HOME_OVERRIDE is set, respect it -# else if IMPALA_JDK_VERSION == system, look for system JDK -# else if IMPALA_JDK_VERSION == 8+, look for Java 8+ JDK +# if IMPALA_JDK_VERSION is set, look for that version based on known locations +# else if JAVA_HOME is set, use it +# else look for system JDK -# Initialize IMPALA_JDK_VERSION and set package variables for Docker builds +# Set package variables for Docker builds and OS-specific detection. . "$IMPALA_HOME/bin/impala-config-java.sh" -if [[ -n "${IMPALA_JAVA_HOME_OVERRIDE-}" ]]; then - IMPALA_JDK_VERSION=override -fi - -DETECTED_JAVA_HOME=Invalid -if [[ "${IMPALA_JDK_VERSION}" == "system" ]]; then - # Try to detect the system's JAVA_HOME - # If javac exists, then the system has a Java SDK (JRE does not have javac). - # Follow the symbolic links and use this to determine the system's JAVA_HOME. - DETECTED_JAVA_HOME="/usr/java/default" - if [ -n "$(which javac)" ]; then - DETECTED_JAVA_HOME=$(dirname $(dirname $(readlink -f $(which javac)))) +DETECTED_JAVA_HOME=${JAVA_HOME:-} +if [[ -z "${IMPALA_JDK_VERSION:-}" ]]; then + # IMPALA_JDK_VERSION is empty or unset. Use JAVA_HOME or detect system default. + if [[ -z "${DETECTED_JAVA_HOME:-}" ]]; then + # Try to detect the system's JAVA_HOME + # If javac exists, then the system has a Java SDK (JRE does not have javac). + # Follow the symbolic links and use this to determine the system's JAVA_HOME. + DETECTED_JAVA_HOME="/usr/java/default" + if [ -n "$(which javac)" ]; then + DETECTED_JAVA_HOME=$(dirname $(dirname $(readlink -f $(which javac)))) + fi fi -elif [[ "${IMPALA_JDK_VERSION}" != "override" ]]; then +else # Now, we are looking for a specific version, and that will depend on the # distribution. Currently, this is implemented for Redhat and Ubuntu. DISTRIBUTION=Unknown @@ -534,40 +533,42 @@ elif [[ "${IMPALA_JDK_VERSION}" != "override" ]]; then fi if [[ "${DISTRIBUTION}" == "Unknown" ]]; then echo "ERROR: auto-detection of JAVA_HOME only supported for Ubuntu and RedHat." - echo "Use IMPALA_JAVA_HOME_OVERRIDE to configure JAVA_HOME." + echo "Set JAVA_HOME to use a specific location." return 1 fi JVMS_PATH=/usr/lib/jvm if [[ "${DISTRIBUTION}" == "Ubuntu" ]]; then - JAVA_PACKAGE_NAME="java-${IMPALA_JDK_VERSION}-openjdk-${UBUNTU_PACKAGE_ARCH}" + JAVA_PACKAGE_NAME="java-${UBUNTU_JAVA_VERSION}-openjdk-${UBUNTU_PACKAGE_ARCH}" DETECTED_JAVA_HOME="${JVMS_PATH}/${JAVA_PACKAGE_NAME}" elif [[ "${DISTRIBUTION}" == "Redhat" ]]; then - if [[ "${IMPALA_JDK_VERSION}" == "8" ]]; then - DETECTED_JAVA_HOME="${JVMS_PATH}/java-1.8.0" - else - DETECTED_JAVA_HOME="${JVMS_PATH}/java-${IMPALA_JDK_VERSION}" - fi + DETECTED_JAVA_HOME="${JVMS_PATH}/java-${REDHAT_JAVA_VERSION}" fi if [[ ! -d "${DETECTED_JAVA_HOME}" ]]; then echo "ERROR: Could not detect Java ${IMPALA_JDK_VERSION}."\ - "${DETECTED_JAVA_HOME} is not a directory." + "${DETECTED_JAVA_HOME} is not a directory." return 1 fi fi -# Prefer the JAVA_HOME set in the environment, but use the system's JAVA_HOME otherwise -export JAVA_HOME="${IMPALA_JAVA_HOME_OVERRIDE:-${DETECTED_JAVA_HOME}}" -if [ ! -d "$JAVA_HOME" ]; then +# Update JAVA_HOME to the detected JAVA_HOME if it exists. +if [ ! -d "${DETECTED_JAVA_HOME}" ]; then echo "JAVA_HOME must be set to the location of your JDK!" return 1 fi +export JAVA_HOME="${DETECTED_JAVA_HOME}" export JAVA="$JAVA_HOME/bin/java" if [[ ! -e "$JAVA" ]]; then echo "Could not find java binary at $JAVA" >&2 return 1 fi +# Target the Java version matching the JDK. +export IMPALA_JAVA_TARGET=$("$JAVA" -version 2>&1 | awk -F'[\".]' '/version/ {print $2}') +if [[ $IMPALA_JAVA_TARGET -eq 1 ]]; then + # Capture 1.x. + IMPALA_JAVA_TARGET=$("$JAVA" -version 2>&1 | awk -F'[\".]' '/version/ {print $2"."$3}') +fi # Java libraries required by executables and java tests. export LIB_JAVA=$(find "${JAVA_HOME}/" -name libjava.so | head -1) @@ -1218,6 +1219,7 @@ echo "THRIFT_PY_HOME = $THRIFT_PY_HOME" echo "CLASSPATH = $CLASSPATH" echo "LIBHDFS_OPTS = $LIBHDFS_OPTS" echo "JAVA_HOME = $JAVA_HOME" +echo "IMPALA_JAVA_TARGET = $IMPALA_JAVA_TARGET" echo "POSTGRES_JDBC_DRIVER = $POSTGRES_JDBC_DRIVER" echo "IMPALA_TOOLCHAIN = $IMPALA_TOOLCHAIN" echo "IMPALA_TOOLCHAIN_PACKAGES_HOME = $IMPALA_TOOLCHAIN_PACKAGES_HOME" @@ -1278,7 +1280,7 @@ fi # Check for minimum required Java version # Only issue Java version warning when running Java 7. -if $JAVA -version 2>&1 | grep -q 'java version "1.7'; then +if [[ $IMPALA_JAVA_TARGET -eq 1.7 ]]; then cat << EOF WARNING: Your development environment is configured for Hadoop 3 and Java 7. Hadoop 3 diff --git a/bin/set-impala-java-tool-options.sh b/bin/set-impala-java-tool-options.sh index de1d16947..346bb15fb 100755 --- a/bin/set-impala-java-tool-options.sh +++ b/bin/set-impala-java-tool-options.sh @@ -22,7 +22,7 @@ export IMPALA_JAVA_TOOL_OPTIONS="${IMPALA_JAVA_TOOL_OPTIONS:-}" -if (( IMPALA_JDK_VERSION_NUM > 8 )); then +if (( IMPALA_JAVA_TARGET > 8 )); then echo "JDK >= 9 detected, adding --add-opens to IMPALA_JAVA_TOOL_OPTIONS" ADD_OPENS_OPTS=" --add-opens=java.base/java.io=ALL-UNNAMED" ADD_OPENS_OPTS+=" --add-opens=java.base/java.lang.invoke=ALL-UNNAMED"