IMPALA-14450: Simplify Java version selection

Removes IMPALA_JAVA_HOME_OVERRIDE and updates version selection. In
order of priority
1. If IMPALA_JDK_VERSION is set, use the OS JDK version from a known
   location. This is primarily used when also installing the JDK as part
   of automated builds.
2. If JAVA_HOME is set, use it.
3. Look for the system default JDK.

The IMPALA_JDK_VERSION variable is no longer modified to avoid issues
when sourcing impala-config.sh multiple times. JAVA_HOME will be
modified if IMPALA_JDK_VERSION is set; both must be unset to restore
using the system default Java.

If switching between JDKs, now prefer setting JAVA_HOME. If relying on
system Java, unset JAVA_HOME after e.g. update-java-alternatives.

The detected Java version is set in IMPALA_JAVA_TARGET, which is used to
add Java 9+ options and configure the Java compilation target.

Eliminates IMPALA_JDK_VERSION_NUM as it's value was always identical to
IMPALA_JAVA_TARGET.

Stops printing from impala-config-java.sh. It made the output from
impala-config.sh look strange, and the decisions can all be clearly
determined from impala-config.sh printed variables later or the packages
installed in bootstrap_system.sh.

Fixes JAVA_HOME in bootstrap_build.sh on ARM64 systems.

Change-Id: I68435ca69522f8310221a0f3050f13d86568b9da
Reviewed-on: http://gerrit.cloudera.org:8080/23434
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

README-build.md

@@ -31,8 +31,8 @@ can do so through the environment variables and scripts listed below.
 | CDP_COMPONENTS_HOME | "${IMPALA_HOME}/toolchain/cdp_components-${CDP_BUILD_NUMBER}" | Location of the CDP components within the toolchain. |
 | CDH_MAJOR_VERSION | "7" | Identifier used to uniqueify paths for potentially incompatible component builds. |
 | IMPALA_CONFIG_SOURCED | "1" |  Set by ${IMPALA_HOME}/bin/impala-config.sh (internal use) |
-| IMPALA_JAVA_HOME_OVERRIDE | | Specify a non-system Java version. Overrides IMPALA_JDK_VERSION behavior. |
+| IMPALA_JDK_VERSION | "" | Set to 8+ to select a system Java version. Empty value uses JAVA_HOME, or sets it based on system defaults. |
-| IMPALA_JDK_VERSION | "system" | Set to 8 or 11 to select a system Java version. Default will set JAVA_HOME based on the javac symlink in PATH. |
+| JAVA_HOME | "" | Uses Java from JAVA_HOME unless IMPALA_JDK_VERSION is set. |
 | JAVA | "${JAVA_HOME}/bin/java" | Java binary location. |
 | CLASSPATH | | See bin/set-classpath.sh for details. |
 | PYTHONPATH | | See bin/set-pythonpath.sh for details. |

bin/bootstrap_build.sh

@@ -66,8 +66,13 @@ if [[ $DISTRIB_ID == Ubuntu && $DISTRIB_RELEASE == 20.04 ]]; then
 fi
 
 JDK_VERSION=17
+if [[ "$(uname -p)" == 'aarch64' ]]; then
+  PACKAGE_ARCH='arm64'
+else
+  PACKAGE_ARCH='amd64'
+fi
 sudo apt-get --yes --quiet install openjdk-${JDK_VERSION}-jdk openjdk-${JDK_VERSION}-source
-export JAVA_HOME=/usr/lib/jvm/java-${JDK_VERSION}-openjdk-amd64
+export JAVA_HOME=/usr/lib/jvm/java-${JDK_VERSION}-openjdk-${PACKAGE_ARCH}
 
 # Download Maven since the packaged version is pretty old.
 : ${IMPALA_TOOLCHAIN_HOST:=native-toolchain.s3.amazonaws.com}

bin/impala-config-java.sh

@@ -15,19 +15,16 @@
 # specific language governing permissions and limitations
 # under the License.
 
-IMPALA_JDK_VERSION=${IMPALA_JDK_VERSION:-system}
+IMPALA_JDK_VERSION=${IMPALA_JDK_VERSION:-}
 
-# Set OS Java package variables for bootstrap_system and Docker builds
+# Set OS Java package variables for bootstrap_system and Docker builds.
-if [[ "${IMPALA_JDK_VERSION}" == "system" || "${IMPALA_JDK_VERSION}" == "8" ]]; then
+# Defaults to installing Java 8.
+if [[ "${IMPALA_JDK_VERSION}" == "" || "${IMPALA_JDK_VERSION}" == "8" ]]; then
   UBUNTU_JAVA_VERSION=8
   REDHAT_JAVA_VERSION=1.8.0
-  export IMPALA_JDK_VERSION_NUM=8
-  export IMPALA_JAVA_TARGET=1.8
 else
   UBUNTU_JAVA_VERSION="${IMPALA_JDK_VERSION}"
   REDHAT_JAVA_VERSION="${IMPALA_JDK_VERSION}"
-  export IMPALA_JDK_VERSION_NUM="${IMPALA_JDK_VERSION}"
-  export IMPALA_JAVA_TARGET="${IMPALA_JDK_VERSION}"
 fi
 
 if [[ "$(uname -p)" == 'aarch64' ]]; then
@@ -35,8 +32,3 @@ if [[ "$(uname -p)" == 'aarch64' ]]; then
 else
   UBUNTU_PACKAGE_ARCH='amd64'
 fi
-
-echo "JAVA_HOME: ${JAVA_HOME:-}"
-echo "IMPALA_JDK_VERSION: $IMPALA_JDK_VERSION"
-echo "IMPALA_JDK_VERSION_NUM: $IMPALA_JDK_VERSION_NUM"
-echo "IMPALA_JAVA_TARGET: $IMPALA_JAVA_TARGET"

bin/impala-config.sh

@@ -498,27 +498,26 @@ fi
 # the JDK version as part of bin/impala-config-local.sh
 
 # Decision tree:
-# if IMPALA_JAVA_HOME_OVERRIDE is set, respect it
+# if IMPALA_JDK_VERSION is set, look for that version based on known locations
-# else if IMPALA_JDK_VERSION == system, look for system JDK
+# else if JAVA_HOME is set, use it
-# else if IMPALA_JDK_VERSION == 8+, look for Java 8+ JDK
+# else look for system JDK
 
-# Initialize IMPALA_JDK_VERSION and set package variables for Docker builds
+# Set package variables for Docker builds and OS-specific detection.
 . "$IMPALA_HOME/bin/impala-config-java.sh"
 
-if [[ -n "${IMPALA_JAVA_HOME_OVERRIDE-}" ]]; then
+DETECTED_JAVA_HOME=${JAVA_HOME:-}
-  IMPALA_JDK_VERSION=override
+if [[ -z "${IMPALA_JDK_VERSION:-}" ]]; then
-fi
+  # IMPALA_JDK_VERSION is empty or unset. Use JAVA_HOME or detect system default.
-
+  if [[ -z "${DETECTED_JAVA_HOME:-}" ]]; then
-DETECTED_JAVA_HOME=Invalid
+    # Try to detect the system's JAVA_HOME
-if [[ "${IMPALA_JDK_VERSION}" == "system" ]]; then
+    # If javac exists, then the system has a Java SDK (JRE does not have javac).
-  # Try to detect the system's JAVA_HOME
+    # Follow the symbolic links and use this to determine the system's JAVA_HOME.
-  # If javac exists, then the system has a Java SDK (JRE does not have javac).
+    DETECTED_JAVA_HOME="/usr/java/default"
-  # Follow the symbolic links and use this to determine the system's JAVA_HOME.
+    if [ -n "$(which javac)" ]; then
-  DETECTED_JAVA_HOME="/usr/java/default"
+      DETECTED_JAVA_HOME=$(dirname $(dirname $(readlink -f $(which javac))))
-  if [ -n "$(which javac)" ]; then
+    fi
-    DETECTED_JAVA_HOME=$(dirname $(dirname $(readlink -f $(which javac))))
   fi
-elif [[ "${IMPALA_JDK_VERSION}" != "override" ]]; then
+else
   # Now, we are looking for a specific version, and that will depend on the
   # distribution. Currently, this is implemented for Redhat and Ubuntu.
   DISTRIBUTION=Unknown
@@ -534,40 +533,42 @@ elif [[ "${IMPALA_JDK_VERSION}" != "override" ]]; then
   fi
   if [[ "${DISTRIBUTION}" == "Unknown" ]]; then
     echo "ERROR: auto-detection of JAVA_HOME only supported for Ubuntu and RedHat."
-    echo "Use IMPALA_JAVA_HOME_OVERRIDE to configure JAVA_HOME."
+    echo "Set JAVA_HOME to use a specific location."
     return 1
   fi
 
   JVMS_PATH=/usr/lib/jvm
   if [[ "${DISTRIBUTION}" == "Ubuntu" ]]; then
-    JAVA_PACKAGE_NAME="java-${IMPALA_JDK_VERSION}-openjdk-${UBUNTU_PACKAGE_ARCH}"
+    JAVA_PACKAGE_NAME="java-${UBUNTU_JAVA_VERSION}-openjdk-${UBUNTU_PACKAGE_ARCH}"
     DETECTED_JAVA_HOME="${JVMS_PATH}/${JAVA_PACKAGE_NAME}"
   elif [[ "${DISTRIBUTION}" == "Redhat" ]]; then
-    if [[ "${IMPALA_JDK_VERSION}" == "8" ]]; then
+    DETECTED_JAVA_HOME="${JVMS_PATH}/java-${REDHAT_JAVA_VERSION}"
-      DETECTED_JAVA_HOME="${JVMS_PATH}/java-1.8.0"
-    else
-      DETECTED_JAVA_HOME="${JVMS_PATH}/java-${IMPALA_JDK_VERSION}"
-    fi
   fi
 
   if [[ ! -d "${DETECTED_JAVA_HOME}" ]]; then
     echo "ERROR: Could not detect Java ${IMPALA_JDK_VERSION}."\
-         "${DETECTED_JAVA_HOME} is not a directory."
+        "${DETECTED_JAVA_HOME} is not a directory."
     return 1
   fi
 fi
 
-# Prefer the JAVA_HOME set in the environment, but use the system's JAVA_HOME otherwise
+# Update JAVA_HOME to the detected JAVA_HOME if it exists.
-export JAVA_HOME="${IMPALA_JAVA_HOME_OVERRIDE:-${DETECTED_JAVA_HOME}}"
+if [ ! -d "${DETECTED_JAVA_HOME}" ]; then
-if [ ! -d "$JAVA_HOME" ]; then
   echo "JAVA_HOME must be set to the location of your JDK!"
   return 1
 fi
+export JAVA_HOME="${DETECTED_JAVA_HOME}"
 export JAVA="$JAVA_HOME/bin/java"
 if [[ ! -e "$JAVA" ]]; then
   echo "Could not find java binary at $JAVA" >&2
   return 1
 fi
+# Target the Java version matching the JDK.
+export IMPALA_JAVA_TARGET=$("$JAVA" -version 2>&1 | awk -F'[\".]' '/version/ {print $2}')
+if [[ $IMPALA_JAVA_TARGET -eq 1 ]]; then
+  # Capture 1.x.
+  IMPALA_JAVA_TARGET=$("$JAVA" -version 2>&1 | awk -F'[\".]' '/version/ {print $2"."$3}')
+fi
 
 # Java libraries required by executables and java tests.
 export LIB_JAVA=$(find "${JAVA_HOME}/" -name libjava.so | head -1)
@@ -1218,6 +1219,7 @@ echo "THRIFT_PY_HOME          = $THRIFT_PY_HOME"
 echo "CLASSPATH               = $CLASSPATH"
 echo "LIBHDFS_OPTS            = $LIBHDFS_OPTS"
 echo "JAVA_HOME               = $JAVA_HOME"
+echo "IMPALA_JAVA_TARGET      = $IMPALA_JAVA_TARGET"
 echo "POSTGRES_JDBC_DRIVER    = $POSTGRES_JDBC_DRIVER"
 echo "IMPALA_TOOLCHAIN        = $IMPALA_TOOLCHAIN"
 echo "IMPALA_TOOLCHAIN_PACKAGES_HOME = $IMPALA_TOOLCHAIN_PACKAGES_HOME"
@@ -1278,7 +1280,7 @@ fi
 
 # Check for minimum required Java version
 # Only issue Java version warning when running Java 7.
-if $JAVA -version 2>&1 | grep -q 'java version "1.7'; then
+if [[ $IMPALA_JAVA_TARGET -eq 1.7 ]]; then
   cat << EOF
 
 WARNING: Your development environment is configured for Hadoop 3 and Java 7. Hadoop 3

bin/set-impala-java-tool-options.sh

@@ -22,7 +22,7 @@
 
 export IMPALA_JAVA_TOOL_OPTIONS="${IMPALA_JAVA_TOOL_OPTIONS:-}"
 
-if (( IMPALA_JDK_VERSION_NUM > 8 )); then
+if (( IMPALA_JAVA_TARGET > 8 )); then
   echo "JDK >= 9 detected, adding --add-opens to IMPALA_JAVA_TOOL_OPTIONS"
   ADD_OPENS_OPTS=" --add-opens=java.base/java.io=ALL-UNNAMED"
   ADD_OPENS_OPTS+=" --add-opens=java.base/java.lang.invoke=ALL-UNNAMED"