IMPALA-14450: Simplify Java version selection

Removes IMPALA_JAVA_HOME_OVERRIDE and updates version selection. In order of priority 1. If IMPALA_JDK_VERSION is set, use the OS JDK version from a known location. This is primarily used when also installing the JDK as part of automated builds. 2. If JAVA_HOME is set, use it. 3. Look for the system default JDK. The IMPALA_JDK_VERSION variable is no longer modified to avoid issues when sourcing impala-config.sh multiple times. JAVA_HOME will be modified if IMPALA_JDK_VERSION is set; both must be unset to restore using the system default Java. If switching between JDKs, now prefer setting JAVA_HOME. If relying on system Java, unset JAVA_HOME after e.g. update-java-alternatives. The detected Java version is set in IMPALA_JAVA_TARGET, which is used to add Java 9+ options and configure the Java compilation target. Eliminates IMPALA_JDK_VERSION_NUM as it's value was always identical to IMPALA_JAVA_TARGET. Stops printing from impala-config-java.sh. It made the output from impala-config.sh look strange, and the decisions can all be clearly determined from impala-config.sh printed variables later or the packages installed in bootstrap_system.sh. Fixes JAVA_HOME in bootstrap_build.sh on ARM64 systems. Change-Id: I68435ca69522f8310221a0f3050f13d86568b9da Reviewed-on: http://gerrit.cloudera.org:8080/23434 Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com> Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
2025-12-19 18:12:08 -05:00 · 2025-09-16 13:47:34 -07:00
parent ca356a8df5
commit d217b9ecc6
5 changed files with 44 additions and 45 deletions
--- a/README-build.md
+++ b/README-build.md
@@ -31,8 +31,8 @@ can do so through the environment variables and scripts listed below.
 | CDP_COMPONENTS_HOME | "${IMPALA_HOME}/toolchain/cdp_components-${CDP_BUILD_NUMBER}" | Location of the CDP components within the toolchain. |
 | CDH_MAJOR_VERSION | "7" | Identifier used to uniqueify paths for potentially incompatible component builds. |
 | IMPALA_CONFIG_SOURCED | "1" |  Set by ${IMPALA_HOME}/bin/impala-config.sh (internal use) |
-| IMPALA_JAVA_HOME_OVERRIDE | | Specify a non-system Java version. Overrides IMPALA_JDK_VERSION behavior. |
-| IMPALA_JDK_VERSION | "system" | Set to 8 or 11 to select a system Java version. Default will set JAVA_HOME based on the javac symlink in PATH. |
+| IMPALA_JDK_VERSION | "" | Set to 8+ to select a system Java version. Empty value uses JAVA_HOME, or sets it based on system defaults. |
+| JAVA_HOME | "" | Uses Java from JAVA_HOME unless IMPALA_JDK_VERSION is set. |
 | JAVA | "${JAVA_HOME}/bin/java" | Java binary location. |
 | CLASSPATH | | See bin/set-classpath.sh for details. |
 | PYTHONPATH | | See bin/set-pythonpath.sh for details. |
--- a/bin/bootstrap_build.sh
+++ b/bin/bootstrap_build.sh
@@ -66,8 +66,13 @@ if [[ $DISTRIB_ID == Ubuntu && $DISTRIB_RELEASE == 20.04 ]]; then
 fi

 JDK_VERSION=17
+if [[ "$(uname -p)" == 'aarch64' ]]; then
+  PACKAGE_ARCH='arm64'
+else
+  PACKAGE_ARCH='amd64'
+fi
 sudo apt-get --yes --quiet install openjdk-${JDK_VERSION}-jdk openjdk-${JDK_VERSION}-source
-export JAVA_HOME=/usr/lib/jvm/java-${JDK_VERSION}-openjdk-amd64
+export JAVA_HOME=/usr/lib/jvm/java-${JDK_VERSION}-openjdk-${PACKAGE_ARCH}

 # Download Maven since the packaged version is pretty old.
 : ${IMPALA_TOOLCHAIN_HOST:=native-toolchain.s3.amazonaws.com}
--- a/bin/impala-config-java.sh
+++ b/bin/impala-config-java.sh
@@ -15,19 +15,16 @@
 # specific language governing permissions and limitations
 # under the License.

-IMPALA_JDK_VERSION=${IMPALA_JDK_VERSION:-system}
+IMPALA_JDK_VERSION=${IMPALA_JDK_VERSION:-}

-# Set OS Java package variables for bootstrap_system and Docker builds
-if [[ "${IMPALA_JDK_VERSION}" == "system" || "${IMPALA_JDK_VERSION}" == "8" ]]; then
+# Set OS Java package variables for bootstrap_system and Docker builds.
+# Defaults to installing Java 8.
+if [[ "${IMPALA_JDK_VERSION}" == "" || "${IMPALA_JDK_VERSION}" == "8" ]]; then
  UBUNTU_JAVA_VERSION=8
  REDHAT_JAVA_VERSION=1.8.0
-  export IMPALA_JDK_VERSION_NUM=8
-  export IMPALA_JAVA_TARGET=1.8
 else
  UBUNTU_JAVA_VERSION="${IMPALA_JDK_VERSION}"
  REDHAT_JAVA_VERSION="${IMPALA_JDK_VERSION}"
-  export IMPALA_JDK_VERSION_NUM="${IMPALA_JDK_VERSION}"
-  export IMPALA_JAVA_TARGET="${IMPALA_JDK_VERSION}"
 fi

 if [[ "$(uname -p)" == 'aarch64' ]]; then
@@ -35,8 +32,3 @@ if [[ "$(uname -p)" == 'aarch64' ]]; then
 else
  UBUNTU_PACKAGE_ARCH='amd64'
 fi
-
-echo "JAVA_HOME: ${JAVA_HOME:-}"
-echo "IMPALA_JDK_VERSION: $IMPALA_JDK_VERSION"
-echo "IMPALA_JDK_VERSION_NUM: $IMPALA_JDK_VERSION_NUM"
-echo "IMPALA_JAVA_TARGET: $IMPALA_JAVA_TARGET"
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -498,19 +498,17 @@ fi
 # the JDK version as part of bin/impala-config-local.sh

 # Decision tree:
-# if IMPALA_JAVA_HOME_OVERRIDE is set, respect it
-# else if IMPALA_JDK_VERSION == system, look for system JDK
-# else if IMPALA_JDK_VERSION == 8+, look for Java 8+ JDK
+# if IMPALA_JDK_VERSION is set, look for that version based on known locations
+# else if JAVA_HOME is set, use it
+# else look for system JDK

-# Initialize IMPALA_JDK_VERSION and set package variables for Docker builds
+# Set package variables for Docker builds and OS-specific detection.
 . "$IMPALA_HOME/bin/impala-config-java.sh"

-if [[ -n "${IMPALA_JAVA_HOME_OVERRIDE-}" ]]; then
-  IMPALA_JDK_VERSION=override
-fi
-
-DETECTED_JAVA_HOME=Invalid
-if [[ "${IMPALA_JDK_VERSION}" == "system" ]]; then
+DETECTED_JAVA_HOME=${JAVA_HOME:-}
+if [[ -z "${IMPALA_JDK_VERSION:-}" ]]; then
+  # IMPALA_JDK_VERSION is empty or unset. Use JAVA_HOME or detect system default.
+  if [[ -z "${DETECTED_JAVA_HOME:-}" ]]; then
    # Try to detect the system's JAVA_HOME
    # If javac exists, then the system has a Java SDK (JRE does not have javac).
    # Follow the symbolic links and use this to determine the system's JAVA_HOME.
@@ -518,7 +516,8 @@ if [[ "${IMPALA_JDK_VERSION}" == "system" ]]; then
    if [ -n "$(which javac)" ]; then
      DETECTED_JAVA_HOME=$(dirname $(dirname $(readlink -f $(which javac))))
    fi
-elif [[ "${IMPALA_JDK_VERSION}" != "override" ]]; then
+  fi
+else
  # Now, we are looking for a specific version, and that will depend on the
  # distribution. Currently, this is implemented for Redhat and Ubuntu.
  DISTRIBUTION=Unknown
@@ -534,20 +533,16 @@ elif [[ "${IMPALA_JDK_VERSION}" != "override" ]]; then
  fi
  if [[ "${DISTRIBUTION}" == "Unknown" ]]; then
    echo "ERROR: auto-detection of JAVA_HOME only supported for Ubuntu and RedHat."
-    echo "Use IMPALA_JAVA_HOME_OVERRIDE to configure JAVA_HOME."
+    echo "Set JAVA_HOME to use a specific location."
    return 1
  fi

  JVMS_PATH=/usr/lib/jvm
  if [[ "${DISTRIBUTION}" == "Ubuntu" ]]; then
-    JAVA_PACKAGE_NAME="java-${IMPALA_JDK_VERSION}-openjdk-${UBUNTU_PACKAGE_ARCH}"
+    JAVA_PACKAGE_NAME="java-${UBUNTU_JAVA_VERSION}-openjdk-${UBUNTU_PACKAGE_ARCH}"
    DETECTED_JAVA_HOME="${JVMS_PATH}/${JAVA_PACKAGE_NAME}"
  elif [[ "${DISTRIBUTION}" == "Redhat" ]]; then
-    if [[ "${IMPALA_JDK_VERSION}" == "8" ]]; then
-      DETECTED_JAVA_HOME="${JVMS_PATH}/java-1.8.0"
-    else
-      DETECTED_JAVA_HOME="${JVMS_PATH}/java-${IMPALA_JDK_VERSION}"
-    fi
+    DETECTED_JAVA_HOME="${JVMS_PATH}/java-${REDHAT_JAVA_VERSION}"
  fi

  if [[ ! -d "${DETECTED_JAVA_HOME}" ]]; then
@@ -557,17 +552,23 @@ elif [[ "${IMPALA_JDK_VERSION}" != "override" ]]; then
  fi
 fi

-# Prefer the JAVA_HOME set in the environment, but use the system's JAVA_HOME otherwise
-export JAVA_HOME="${IMPALA_JAVA_HOME_OVERRIDE:-${DETECTED_JAVA_HOME}}"
-if [ ! -d "$JAVA_HOME" ]; then
+# Update JAVA_HOME to the detected JAVA_HOME if it exists.
+if [ ! -d "${DETECTED_JAVA_HOME}" ]; then
  echo "JAVA_HOME must be set to the location of your JDK!"
  return 1
 fi
+export JAVA_HOME="${DETECTED_JAVA_HOME}"
 export JAVA="$JAVA_HOME/bin/java"
 if [[ ! -e "$JAVA" ]]; then
  echo "Could not find java binary at $JAVA" >&2
  return 1
 fi
+# Target the Java version matching the JDK.
+export IMPALA_JAVA_TARGET=$("$JAVA" -version 2>&1 | awk -F'[\".]' '/version/ {print $2}')
+if [[ $IMPALA_JAVA_TARGET -eq 1 ]]; then
+  # Capture 1.x.
+  IMPALA_JAVA_TARGET=$("$JAVA" -version 2>&1 | awk -F'[\".]' '/version/ {print $2"."$3}')
+fi

 # Java libraries required by executables and java tests.
 export LIB_JAVA=$(find "${JAVA_HOME}/" -name libjava.so | head -1)
@@ -1218,6 +1219,7 @@ echo "THRIFT_PY_HOME          = $THRIFT_PY_HOME"
 echo "CLASSPATH               = $CLASSPATH"
 echo "LIBHDFS_OPTS            = $LIBHDFS_OPTS"
 echo "JAVA_HOME               = $JAVA_HOME"
+echo "IMPALA_JAVA_TARGET      = $IMPALA_JAVA_TARGET"
 echo "POSTGRES_JDBC_DRIVER    = $POSTGRES_JDBC_DRIVER"
 echo "IMPALA_TOOLCHAIN        = $IMPALA_TOOLCHAIN"
 echo "IMPALA_TOOLCHAIN_PACKAGES_HOME = $IMPALA_TOOLCHAIN_PACKAGES_HOME"
@@ -1278,7 +1280,7 @@ fi

 # Check for minimum required Java version
 # Only issue Java version warning when running Java 7.
-if $JAVA -version 2>&1 | grep -q 'java version "1.7'; then
+if [[ $IMPALA_JAVA_TARGET -eq 1.7 ]]; then
  cat << EOF

 WARNING: Your development environment is configured for Hadoop 3 and Java 7. Hadoop 3
--- a/bin/set-impala-java-tool-options.sh
+++ b/bin/set-impala-java-tool-options.sh
@@ -22,7 +22,7 @@

 export IMPALA_JAVA_TOOL_OPTIONS="${IMPALA_JAVA_TOOL_OPTIONS:-}"

-if (( IMPALA_JDK_VERSION_NUM > 8 )); then
+if (( IMPALA_JAVA_TARGET > 8 )); then
  echo "JDK >= 9 detected, adding --add-opens to IMPALA_JAVA_TOOL_OPTIONS"
  ADD_OPENS_OPTS=" --add-opens=java.base/java.io=ALL-UNNAMED"
  ADD_OPENS_OPTS+=" --add-opens=java.base/java.lang.invoke=ALL-UNNAMED"