Revert "IMPALA-11253: Support testing with Java 11"

This reverts commit ee6395db76 as it is
not flexible enough at detecting Java automatically in likely build
environments.

Change-Id: I836c9f7fd10740b15f7e40b2e7f889ac7ee61fc3
Reviewed-on: http://gerrit.cloudera.org:8080/19908
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Reviewed-by: Michael Smith <michael.smith@cloudera.com>

CMakeLists.txt

@@ -384,8 +384,6 @@ IMPALA_ADD_THIRDPARTY_LIB(crcutil ${CRCUTIL_INCLUDE_DIR} ${CRCUTIL_STATIC_LIB}
   ${CRCUTIL_SHARED_LIB})
 
 # find jni headers and libs
-set(JAVA_AWT_LIBRARY NotNeeded)
-set(JAVA_AWT_INCLUDE_PATH NotNeeded)
 find_package(JNI REQUIRED)
 IMPALA_ADD_THIRDPARTY_LIB(java_jvm "${JNI_INCLUDE_DIRS}" "" ${JAVA_JVM_LIBRARY})
 

LICENSE.txt

@@ -701,6 +701,40 @@ cmake_modules/FindGTest.cmake: MIT license
 
 --------------------------------------------------------------------------------
 
+cmake_modules/FindJNI.cmake: 3-clause BSD
+
+ Copyright 2001-2009 Kitware, Inc.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+
+ * Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+ * Neither the names of Kitware, Inc., the Insight Software Consortium,
+   nor the names of their contributors may be used to endorse or promote
+   products derived from this software without specific prior written
+   permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
 be/src/kudu/util (some portions): 3-clause BSD license
 
 Some portions of this module are derived from code from LevelDB

bin/bootstrap_system.sh

@@ -214,19 +214,13 @@ if [[ "$UBUNTU" == true ]]; then
     sleep 1
   done
 fi
-
-# Set UBUNTU_JAVA_VERSION, UBUNTU_PACKAGE_ARCH, REDHAT_JAVA_VERSION
-source "$IMPALA_HOME/bin/impala-config-java.sh"
-
 ubuntu apt-get update
 ubuntu apt-get --yes install ccache curl gawk g++ gcc apt-utils git libffi-dev \
         libkrb5-dev krb5-admin-server krb5-kdc krb5-user libsasl2-dev \
         libsasl2-modules libsasl2-modules-gssapi-mit libssl-dev make ninja-build \
         python-dev python-setuptools python3-dev python3-setuptools postgresql \
-        ssh wget vim-common psmisc lsof net-tools language-pack-en libxml2-dev \
+        ssh wget vim-common psmisc lsof openjdk-8-jdk openjdk-8-source openjdk-8-dbg \
-        libxslt-dev openjdk-${UBUNTU_JAVA_VERSION}-jdk \
+        net-tools language-pack-en libxml2-dev libxslt-dev
-        openjdk-${UBUNTU_JAVA_VERSION}-source openjdk-${UBUNTU_JAVA_VERSION}-dbg
-
 # Required by Kudu in the minicluster
 ubuntu20 apt-get --yes install libtinfo5
 ARCH_NAME=$(uname -p)
@@ -236,18 +230,46 @@ if [[ $ARCH_NAME == 'aarch64' ]]; then
           libncurses5-dev libreadline-dev
 fi
 
-# Configure the default Java version to be the version we selected.
+if [[ "$UBUNTU" == true ]]; then
-ubuntu sudo update-java-alternatives -s \
+  # Don't use openjdk-8-jdk 8u181-b13-1ubuntu0.16.04.1 which is known to break the
-    java-1.${UBUNTU_JAVA_VERSION}.0-openjdk-${UBUNTU_PACKAGE_ARCH}
+  # surefire tests. If we detect that version, we downgrade to the last known good one.
+  # See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=911925 for details.
+  JDK_BAD_VERSION="8u181-b13-1ubuntu0.16.04.1"
+  if dpkg -l openjdk-8-jdk | grep -q $JDK_BAD_VERSION; then
+    JDK_TARGET_VERSION="8u181-b13-0ubuntu0.16.04.1"
+    DEB_DIR=$(mktemp -d)
+    pushd $DEB_DIR
+    wget --no-verbose \
+        "https://launchpadlibrarian.net/380913637/openjdk-8-jdk_8u181-b13-0ubuntu0.16.04.1_amd64.deb" \
+        "https://launchpadlibrarian.net/380913636/openjdk-8-jdk-headless_8u181-b13-0ubuntu0.16.04.1_amd64.deb" \
+        "https://launchpadlibrarian.net/380913641/openjdk-8-jre_8u181-b13-0ubuntu0.16.04.1_amd64.deb" \
+        "https://launchpadlibrarian.net/380913638/openjdk-8-jre-headless_8u181-b13-0ubuntu0.16.04.1_amd64.deb" \
+        "https://launchpadlibrarian.net/380913642/openjdk-8-source_8u181-b13-0ubuntu0.16.04.1_all.deb" \
+        "https://launchpadlibrarian.net/380913633/openjdk-8-dbg_8u181-b13-0ubuntu0.16.04.1_amd64.deb"
+    sudo dpkg -i *.deb
+    popd
+    rm -rf $DEB_DIR
+  fi
+fi
+
+# Ubuntu 18.04 and 20.04 install OpenJDK 11 and configure it as the default Java version.
+# Impala is currently tested with OpenJDK 8, so configure that version as the default.
+if [[ $ARCH_NAME == 'aarch64' ]]; then
+  ubuntu20 sudo update-java-alternatives -s java-1.8.0-openjdk-arm64
+  ubuntu18 sudo update-java-alternatives -s java-1.8.0-openjdk-arm64
+else
+  ubuntu18 sudo update-java-alternatives -s java-1.8.0-openjdk-amd64
+  ubuntu20 sudo update-java-alternatives -s java-1.8.0-openjdk-amd64
+fi
 
 redhat sudo yum install -y curl gawk gcc gcc-c++ git krb5-devel krb5-server \
         krb5-workstation libevent-devel libffi-devel make openssl-devel cyrus-sasl \
         cyrus-sasl-gssapi cyrus-sasl-devel cyrus-sasl-plain \
         postgresql postgresql-server \
         wget vim-common nscd cmake fuse-devel zlib-devel \
-        psmisc lsof openssh-server redhat-lsb python3-devel python3-setuptools \
+        psmisc lsof openssh-server redhat-lsb java-1.8.0-openjdk-devel \
-        net-tools langpacks-en glibc-langpack-en libxml2-devel libxslt-devel \
+        java-1.8.0-openjdk-src python3-devel python3-setuptools net-tools \
-        java-${REDHAT_JAVA_VERSION}-openjdk-src java-${REDHAT_JAVA_VERSION}-openjdk-devel
+        langpacks-en glibc-langpack-en libxml2-devel libxslt-devel
 
 # Enable the Powertools repo for snappy-devel on RedHat 8
 redhat8 sudo yum install -y dnf-plugins-core
@@ -443,6 +465,22 @@ SET_IMPALA_HOME="export IMPALA_HOME=$(pwd)"
 echo -e "\n$SET_IMPALA_HOME" >> ~/.bashrc
 eval "$SET_IMPALA_HOME"
 
+# Ubuntu and RH install JDK's in slightly different paths.
+if [[ $UBUNTU == true ]]; then
+  # Assert that there's only one glob match.
+  [ 1 == $(compgen -G "/usr/lib/jvm/java-8-openjdk-*" | wc -l) ]
+  SET_JAVA_HOME="export JAVA_HOME=$(compgen -G '/usr/lib/jvm/java-8-openjdk-*')"
+else
+  # Assert that there's only one glob match.
+  [ 1 == $(compgen -G "/usr/lib/jvm/java-1.8.0-openjdk-*" | wc -l) ]
+  SET_JAVA_HOME="export JAVA_HOME=$(compgen -G '/usr/lib/jvm/java-1.8.0-openjdk-*')"
+fi
+
+echo -e "\n$SET_JAVA_HOME" >> "${IMPALA_HOME}/bin/impala-config-local.sh"
+eval "$SET_JAVA_HOME"
+# Assert that we have a java available
+test -f $JAVA_HOME/bin/java
+
 if [[ $ARCH_NAME == 'aarch64' ]]; then
   echo -e "\nexport SKIP_TOOLCHAIN_BOOTSTRAP=true" >> \
     "${IMPALA_HOME}/bin/impala-config-local.sh"

bin/impala-config-java.sh

@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-IMPALA_JDK_VERSION=${IMPALA_JDK_VERSION:-system}
-
-# Set OS Java package variables for bootstrap_system and Docker builds
-if [[ "${IMPALA_JDK_VERSION}" == "system" || "${IMPALA_JDK_VERSION}" == "8" ]]; then
-  UBUNTU_JAVA_VERSION=8
-  REDHAT_JAVA_VERSION=1.8.0
-elif [[ "${IMPALA_JDK_VERSION}" == "11" ]]; then
-  UBUNTU_JAVA_VERSION=11
-  REDHAT_JAVA_VERSION=11
-else
-  echo "Unknown value for IMPALA_JDK_VERSION=${IMPALA_JDK_VERSION}"
-  exit 1
-fi
-
-if [[ "$(uname -p)" == 'aarch64' ]]; then
-  UBUNTU_PACKAGE_ARCH='arm64'
-else
-  UBUNTU_PACKAGE_ARCH='amd64'
-fi

bin/impala-config.sh

@@ -380,72 +380,20 @@ fi
 
 # It is important to have a coherent view of the JAVA_HOME and JAVA executable.
 # The JAVA_HOME should be determined first, then the JAVA executable should be
-# derived from JAVA_HOME. For development, it is useful to be able to specify
+# derived from JAVA_HOME. bin/bootstrap_development.sh adds code to
-# the JDK version as part of bin/impala-config-local.sh
+# bin/impala-config-local.sh to set JAVA_HOME, so it is important to pick up that
+# setting before deciding what JAVA_HOME to use.
 
-# Decision tree:
+# Try to detect the system's JAVA_HOME
-# if IMPALA_JAVA_HOME_OVERRIDE is set, respect it
+# If javac exists, then the system has a Java SDK (JRE does not have javac).
-# else if IMPALA_JDK_VERSION == system, look for system JDK
+# Follow the symbolic links and use this to determine the system's JAVA_HOME.
-# else if IMPALA_JDK_VERSION == 8, look for Java 8 JDK
+SYSTEM_JAVA_HOME="/usr/java/default"
-# else if IMPALA_JDK_VERSION == 11, look for Java 11 JDK
+if [ -n "$(which javac)" ]; then
-
+  SYSTEM_JAVA_HOME=$(which javac | xargs readlink -f | sed "s:/bin/javac::")
-# Initialize IMPALA_JDK_VERSION and set package variables for Docker builds
-. "$IMPALA_HOME/bin/impala-config-java.sh"
-
-if [[ -n "${IMPALA_JAVA_HOME_OVERRIDE-}" ]]; then
-  IMPALA_JDK_VERSION=override
-fi
-
-DETECTED_JAVA_HOME=Invalid
-if [[ "${IMPALA_JDK_VERSION}" == "system" ]]; then
-  # Try to detect the system's JAVA_HOME
-  # If javac exists, then the system has a Java SDK (JRE does not have javac).
-  # Follow the symbolic links and use this to determine the system's JAVA_HOME.
-  DETECTED_JAVA_HOME="/usr/java/default"
-  if [ -n "$(which javac)" ]; then
-    DETECTED_JAVA_HOME=$(dirname $(dirname $(readlink -f $(which javac))))
-  fi
-elif [[ "${IMPALA_JDK_VERSION}" != "override" ]]; then
-  # Now, we are looking for a specific version, and that will depend on the
-  # distribution. Currently, this is implemented for Redhat and Ubuntu.
-  DISTRIBUTION=Unknown
-  if [[ -f /etc/redhat-release ]]; then
-    echo "Identified Redhat image."
-    DISTRIBUTION=Redhat
-  else
-    source /etc/lsb-release
-    if [[ $DISTRIB_ID == Ubuntu ]]; then
-      echo "Identified Ubuntu image."
-      DISTRIBUTION=Ubuntu
-    fi
-  fi
-  if [[ "${DISTRIBUTION}" == "Unknown" ]]; then
-    echo "ERROR: auto-detection of JAVA_HOME only supported for Ubuntu and RedHat."
-    echo "Use IMPALA_JAVA_HOME_OVERRIDE to configure JAVA_HOME."
-    return 1
-  fi
-
-  JVMS_PATH=/usr/lib/jvm
-  if [[ "${DISTRIBUTION}" == "Ubuntu" ]]; then
-    JAVA_PACKAGE_NAME="java-${IMPALA_JDK_VERSION}-openjdk-${UBUNTU_PACKAGE_ARCH}"
-    DETECTED_JAVA_HOME="${JVMS_PATH}/${JAVA_PACKAGE_NAME}"
-  elif [[ "${DISTRIBUTION}" == "Redhat" ]]; then
-    if [[ "${IMPALA_JDK_VERSION}" == "8" ]]; then
-      DETECTED_JAVA_HOME="${JVMS_PATH}/java-1.8.0"
-    else
-      DETECTED_JAVA_HOME="${JVMS_PATH}/java-${IMPALA_JDK_VERSION}"
-    fi
-  fi
-
-  if [[ ! -d "${DETECTED_JAVA_HOME}" ]]; then
-    echo "ERROR: Could not detect Java ${IMPALA_JDK_VERSION}."\
-         "${DETECTED_JAVA_HOME} is not a directory."
-    return 1
-  fi
 fi
 
 # Prefer the JAVA_HOME set in the environment, but use the system's JAVA_HOME otherwise
-export JAVA_HOME="${IMPALA_JAVA_HOME_OVERRIDE:-${DETECTED_JAVA_HOME}}"
+export JAVA_HOME="${JAVA_HOME:-${SYSTEM_JAVA_HOME}}"
 if [ ! -d "$JAVA_HOME" ]; then
   echo "JAVA_HOME must be set to the location of your JDK!"
   return 1
@@ -456,16 +404,6 @@ if [[ ! -e "$JAVA" ]]; then
   return 1
 fi
 
-# Determine the JDK version for later checks
-IMPALA_JDK_VERSION=$($JAVA -version 2>&1 | head -n1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')
-IMPALA_JDK_MAJOR=$(echo $IMPALA_JDK_VERSION | cut -d'.' -f1)
-IMPALA_JDK_MINOR=$(echo $IMPALA_JDK_VERSION | cut -d'.' -f2)
-if [[ $IMPALA_JDK_MAJOR -gt 1 ]]; then
-  export IMPALA_JDK_VERSION=$IMPALA_JDK_MAJOR
-else
-  export IMPALA_JDK_VERSION=$IMPALA_JDK_MINOR
-fi
-
 # Java libraries required by executables and java tests.
 export LIB_JAVA=$(find "${JAVA_HOME}/" -name libjava.so | head -1)
 export LIB_JSIG=$(find "${JAVA_HOME}/" -name libjsig.so | head -1)
@@ -1083,7 +1021,7 @@ fi
 
 # Check for minimum required Java version
 # Only issue Java version warning when running Java 7.
-if [ $IMPALA_JDK_VERSION -le 7 ]; then
+if $JAVA -version 2>&1 | grep -q 'java version "1.7'; then
   cat << EOF
 
 WARNING: Your development environment is configured for Hadoop 3 and Java 7. Hadoop 3

bin/run-all-tests.sh

@@ -52,9 +52,6 @@ fi
 # Run Cluster Tests
 : ${CLUSTER_TEST:=true}
 : ${CLUSTER_TEST_FILES:=}
-# Verifiers to run after all tests. Skipped if empty.
-: ${TEST_SUITE_VERIFIERS:=verifiers/test_banned_log_messages.py}
-: ${TEST_SUITE_VERIFIERS_LOG_DIR:=${IMPALA_LOGS_DIR}/verifiers}
 # Extra arguments passed to start-impala-cluster for tests. These do not apply to custom
 # cluster tests.
 : ${TEST_START_CLUSTER_ARGS:=}
@@ -333,17 +330,6 @@ do
     export IMPALA_MAX_LOG_FILES="${IMPALA_MAX_LOG_FILES_SAVE}"
   fi
 
-  if [ ! -z "${TEST_SUITE_VERIFIERS}" ]; then
-    mkdir -p "${TEST_SUITE_VERIFIERS_LOG_DIR}"
-    pushd "${IMPALA_HOME}/tests"
-    if ! impala-py.test ${TEST_SUITE_VERIFIERS} \
-        --junitxml=${TEST_SUITE_VERIFIERS_LOG_DIR}/TEST-impala-verifiers.xml \
-        --resultlog=${TEST_SUITE_VERIFIERS_LOG_DIR}/TEST-impala-verifiers.log; then
-      TEST_RET_CODE=1
-    fi
-    popd
-  fi
-
   # Run the process failure tests.
   # Disabled temporarily until we figure out the proper timeouts required to make the test
   # succeed.

cmake_modules/FindJNI.cmake

@@ -0,0 +1,281 @@
+# - Find JNI java libraries.
+# This module finds if Java is installed and determines where the
+# include files and libraries are. It also determines what the name of
+# the library is. This code sets the following variables:
+#
+#  JNI_INCLUDE_DIRS      = the include dirs to use
+#  JNI_LIBRARIES         = the libraries to use
+#  JAVA_AWT_LIBRARY      = the path to the jawt library
+#  JAVA_JVM_LIBRARY      = the path to the jvm library
+#  JAVA_INCLUDE_PATH     = the include path to jni.h
+#  JAVA_INCLUDE_PATH2    = the include path to jni_md.h
+#  JAVA_AWT_INCLUDE_PATH = the include path to jawt.h
+#
+
+#=============================================================================
+# Copyright 2001-2009 Kitware, Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# * Redistributions of source code must retain the above copyright
+#   notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+#   notice, this list of conditions and the following disclaimer in the
+#   documentation and/or other materials provided with the distribution.
+#
+# * Neither the names of Kitware, Inc., the Insight Software Consortium,
+#   nor the names of their contributors may be used to endorse or promote
+#   products derived from this software without specific prior written
+#   permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#=============================================================================
+
+# Expand {libarch} occurences to java_libarch subdirectory(-ies) and set ${_var}
+macro(JAVA_APPEND_LIBRARY_DIRECTORIES _var)
+    # Determine java arch-specific library subdir
+    if (CMAKE_SYSTEM_NAME MATCHES "Linux")
+        # Based on openjdk/jdk/make/common/shared/Platform.gmk as of 6b16
+        # and kaffe as of 1.1.8 which uses the first part of the
+        # GNU config.guess platform triplet.
+        if (CMAKE_SYSTEM_PROCESSOR MATCHES "^i[3-9]86$")
+            set(_java_libarch "i386")
+        elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+            set(_java_libarch "amd64" "x86_64")
+        elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^ppc")
+            set(_java_libarch "ppc" "powerpc" "ppc64")
+        elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^sparc")
+            set(_java_libarch "sparc" "sparcv9")
+        else (CMAKE_SYSTEM_PROCESSOR MATCHES "^i[3-9]86$")
+            set(_java_libarch "${CMAKE_SYSTEM_PROCESSOR}")
+        endif(CMAKE_SYSTEM_PROCESSOR MATCHES "^i[3-9]86$")
+    else (CMAKE_SYSTEM_NAME MATCHES "Linux")
+        set(_java_libarch "i386" "amd64" "ppc") # previous default
+    endif (CMAKE_SYSTEM_NAME MATCHES "Linux")
+
+    foreach(_path ${ARGN})
+        if (_path MATCHES "{libarch}")
+            foreach(_libarch ${_java_libarch})
+                string(REPLACE "{libarch}" "${_libarch}" _newpath "${_path}")
+                list(APPEND ${_var} "${_newpath}")
+            endforeach(_libarch)
+        else (_path MATCHES "{libarch}")
+            list(APPEND ${_var} "${_path}")
+        endif (_path MATCHES "{libarch}")
+    endforeach(_path)
+endmacro(JAVA_APPEND_LIBRARY_DIRECTORIES)
+
+file(TO_CMAKE_PATH "$ENV{JAVA_HOME}" _JAVA_HOME)
+
+get_filename_component(java_install_version
+  "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit;CurrentVersion]" NAME
+)
+
+set(JAVA_AWT_LIBRARY_DIRECTORIES
+    "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.4;JavaHome]/lib"
+    "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.3;JavaHome]/lib"
+    "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\${java_install_version};JavaHome]/lib"
+)
+
+
+java_append_library_directories(JAVA_AWT_LIBRARY_DIRECTORIES
+    ${_JAVA_HOME}/jre/lib/{libarch}
+    ${_JAVA_HOME}/jre/lib
+    ${_JAVA_HOME}/lib/{libarch}
+    ${_JAVA_HOME}/lib
+    ${_JAVA_HOME}
+    /usr/lib
+    /usr/local/lib
+    /usr/lib/jvm/java/lib
+    /usr/lib/java/jre/lib/{libarch}
+    /usr/local/lib/java/jre/lib/{libarch}
+    /usr/local/share/java/jre/lib/{libarch}
+    /usr/lib/j2sdk1.4-sun/jre/lib/{libarch}
+    /usr/lib/j2sdk1.5-sun/jre/lib/{libarch}
+    /opt/sun-jdk-1.5.0.04/jre/lib/{libarch}
+    /usr/lib/jvm/java-6-sun/jre/lib/{libarch}
+    /usr/lib/jvm/java-1.5.0-sun/jre/lib/{libarch}
+    /usr/lib/jvm/java-6-sun-1.6.0.00/jre/lib/{libarch}       # can this one be removed according to #8821 ? Alex
+    /usr/lib/jvm/java-openjdk/jre/lib/{libarch}
+    /usr/lib/jvm/java-6-openjdk/jre/lib/{libarch}
+    /usr/lib/jvm/java-openjdk/jre/lib/{libarch}
+    # Debian specific paths for default JVM
+    /usr/lib/jvm/default-java/jre/lib/{libarch}
+    /usr/lib/jvm/default-java/jre/lib
+    /usr/lib/jvm/default-java/lib
+)
+
+set(JAVA_JVM_LIBRARY_DIRECTORIES)
+foreach (dir ${JAVA_AWT_LIBRARY_DIRECTORIES})
+    set(JAVA_JVM_LIBRARY_DIRECTORIES
+        ${JAVA_JVM_LIBRARY_DIRECTORIES}
+        "${dir}"
+        "${dir}/client"
+        "${dir}/server"
+    )
+endforeach (dir)
+
+
+set(JAVA_AWT_INCLUDE_DIRECTORIES
+    "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.4;JavaHome]/include"
+    "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.3;JavaHome]/include"
+    "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\${java_install_version};JavaHome]/include"
+    ${_JAVA_HOME}/include
+    /usr/include
+    /usr/local/include
+    /usr/lib/java/include
+    /usr/local/lib/java/include
+    /usr/lib/jvm/java/include
+    /usr/lib/jvm/java-6-sun/include
+    /usr/lib/jvm/java-1.5.0-sun/include
+    /usr/lib/jvm/java-6-sun-1.6.0.00/include       # can this one be removed according to #8821 ? Alex
+    /usr/lib/jvm/java-6-openjdk/include
+    /usr/local/share/java/include
+    /usr/lib/j2sdk1.4-sun/include
+    /usr/lib/j2sdk1.5-sun/include
+    /opt/sun-jdk-1.5.0.04/include
+    # Debian specific path for default JVM
+    /usr/lib/jvm/default-java/include
+)
+
+foreach(JAVA_PROG "${JAVA_RUNTIME}" "${JAVA_COMPILE}" "${JAVA_ARCHIVE}" "${JAVA_HEADER}")
+    get_filename_component(jpath "${JAVA_PROG}" PATH)
+    foreach (JAVA_INC_PATH ../include ../java/include ../share/java/include)
+        if (EXISTS ${jpath}/${JAVA_INC_PATH})
+            set(JAVA_AWT_INCLUDE_DIRECTORIES
+                ${JAVA_AWT_INCLUDE_DIRECTORIES}
+                "${jpath}/${JAVA_INC_PATH}"
+            )
+        endif (EXISTS ${jpath}/${JAVA_INC_PATH})
+    endforeach (JAVA_INC_PATH)
+
+    foreach (JAVA_LIB_PATH
+            ../lib ../jre/lib ../jre/lib/i386
+            ../java/lib ../java/jre/lib ../java/jre/lib/i386
+            ../share/java/lib ../share/java/jre/lib ../share/java/jre/lib/i386)
+        if (EXISTS ${jpath}/${JAVA_LIB_PATH})
+            set(JAVA_AWT_LIBRARY_DIRECTORIES
+                ${JAVA_AWT_LIBRARY_DIRECTORIES}
+                "${jpath}/${JAVA_LIB_PATH}"
+            )
+        endif (EXISTS ${jpath}/${JAVA_LIB_PATH})
+    endforeach (JAVA_LIB_PATH)
+endforeach (JAVA_PROG)
+
+if (APPLE)
+    if (EXISTS ~/Library/Frameworks/JavaVM.framework)
+        set(JAVA_HAVE_FRAMEWORK 1)
+    endif (EXISTS ~/Library/Frameworks/JavaVM.framework)
+
+    if (EXISTS /Library/Frameworks/JavaVM.framework)
+        SET(JAVA_HAVE_FRAMEWORK 1)
+    endif (EXISTS /Library/Frameworks/JavaVM.framework)
+
+    if (EXISTS /System/Library/Frameworks/JavaVM.framework)
+        set(JAVA_HAVE_FRAMEWORK 1)
+    endif(EXISTS /System/Library/Frameworks/JavaVM.framework)
+
+    if (JAVA_HAVE_FRAMEWORK)
+        if(NOT JAVA_AWT_LIBRARY)
+            set(JAVA_AWT_LIBRARY "-framework JavaVM" CACHE FILEPATH "Java Frameworks" FORCE)
+        endif(NOT JAVA_AWT_LIBRARY)
+
+        if (NOT JAVA_JVM_LIBRARY)
+            set (JAVA_JVM_LIBRARY "-framework JavaVM" CACHE FILEPATH "Java Frameworks" FORCE)
+        endif (NOT JAVA_JVM_LIBRARY)
+
+        if (NOT JAVA_AWT_INCLUDE_PATH)
+            if (EXISTS /System/Library/Frameworks/JavaVM.framework/Headers/jawt.h)
+                set(JAVA_AWT_INCLUDE_PATH "/System/Library/Frameworks/JavaVM.framework/Headers" CACHE FILEPATH "jawt.h location" FORCE)
+            endif(EXISTS /System/Library/Frameworks/JavaVM.framework/Headers/jawt.h)
+        endif (NOT JAVA_AWT_INCLUDE_PATH)
+
+        #
+        # If using "-framework JavaVM", prefer its headers *before* the others in
+        # JAVA_AWT_INCLUDE_DIRECTORIES... (*prepend* to the list here)
+        #
+        set(JAVA_AWT_INCLUDE_DIRECTORIES
+            ~/Library/Frameworks/JavaVM.framework/Headers
+            /Library/Frameworks/JavaVM.framework/Headers
+            /System/Library/Frameworks/JavaVM.framework/Headers
+            ${JAVA_AWT_INCLUDE_DIRECTORIES}
+        )
+    endif(JAVA_HAVE_FRAMEWORK)
+else (APPLE)
+    find_library(JAVA_AWT_LIBRARY
+        NAMES
+            jawt
+        PATHS
+            ${JAVA_AWT_LIBRARY_DIRECTORIES}
+    )
+
+    find_library(JAVA_JSIG_LIBRARY
+        NAMES
+            jsig
+        PATHS
+            ${JAVA_JVM_LIBRARY_DIRECTORIES}
+    )
+    find_library(JAVA_JVM_LIBRARY
+        NAMES
+            jvm
+            JavaVM
+        PATHS
+            ${JAVA_JVM_LIBRARY_DIRECTORIES}
+    )
+endif (APPLE)
+
+# add in the include path
+find_path(JAVA_INCLUDE_PATH
+    NAMES
+        jni.h
+    PATHS
+        ${JAVA_AWT_INCLUDE_DIRECTORIES}
+)
+
+find_path(JAVA_INCLUDE_PATH2
+    NAMES
+        jni_md.h
+    PATHS
+        ${JAVA_INCLUDE_PATH}
+        ${JAVA_INCLUDE_PATH}/win32
+        ${JAVA_INCLUDE_PATH}/linux
+        ${JAVA_INCLUDE_PATH}/freebsd
+        ${JAVA_INCLUDE_PATH}/solaris
+)
+
+find_path(JAVA_AWT_INCLUDE_PATH
+    NAMES
+        jawt.h
+    PATHS
+        ${JAVA_INCLUDE_PATH}
+)
+
+set(JNI_LIBRARIES
+    ${JAVA_AWT_LIBRARY}
+    ${JAVA_JSIG_LIBRARY}
+    ${JAVA_JVM_LIBRARY}
+)
+
+set(JNI_INCLUDE_DIRS
+    ${JAVA_INCLUDE_PATH}
+    ${JAVA_INCLUDE_PATH2}
+    ${JAVA_AWT_INCLUDE_PATH}
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(JNI DEFAULT_MSG JNI_LIBRARIES JNI_INCLUDE_DIRS JAVA_AWT_LIBRARY JAVA_JSIG_LIBRARY JAVA_JVM_LIBRARY)
+
+mark_as_advanced(JNI_LIBRARIES JNI_INCLUDE_DIRS JAVA_AWT_LIBRARY JAVA_JSIG_LIBRARY JAVA_JVM_LIBRARY)

fe/pom.xml

@@ -621,7 +621,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.11.0</version>
+        <version>3.3</version>
         <configuration>
           <source>1.8</source>
           <target>1.8</target>
@@ -632,7 +632,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-jar-plugin</artifactId>
-        <version>3.3.0</version>
+        <version>2.6</version>
       <executions>
         <execution>
           <goals>
@@ -685,7 +685,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-dependency-plugin</artifactId>
-        <version>3.5.0</version>
+        <version>3.1.1</version>
         <executions>
           <!-- TODO(todd): consider removing this execution or moving it to
                some kind of 'dist' profile. No need to copy all of these jars
@@ -734,7 +734,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>2.20</version>
         <configuration>
           <trimStackTrace>false</trimStackTrace>
           <reportsDirectory>${surefire.reports.dir}</reportsDirectory>
@@ -761,7 +761,7 @@ under the License.
       <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>build-helper-maven-plugin</artifactId>
-        <version>3.3.0</version>
+        <version>1.5</version>
         <executions>
           <!-- Tell maven about our generated files -->
           <execution>
@@ -822,7 +822,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-enforcer-plugin</artifactId>
-        <version>3.1.0</version>
+        <version>3.0.0-M1</version>
         <executions>
           <execution>
             <id>enforce-banned-dependencies</id>
@@ -1059,7 +1059,7 @@ under the License.
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-compiler-plugin</artifactId>
-            <version>3.11.0</version>
+            <version>3.3</version>
             <configuration>
               <showWarnings>true</showWarnings>
               <compilerId>javac-with-errorprone</compilerId>
@@ -1103,7 +1103,7 @@ under the License.
           <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-compiler-plugin</artifactId>
-            <version>3.11.0</version>
+            <version>3.3</version>
             <configuration>
               <excludes>
                 <exclude>**/org/apache/impala/catalog/metastore/*.java</exclude>

fe/src/test/java/org/apache/impala/util/JMXJsonUtilTest.java

@@ -50,8 +50,7 @@ public class JMXJsonUtilTest {
     assertTrue("Invalid JSON: "  + jmxJson, rootNode.hasNonNull("beans"));
     List<String> values = rootNode.get("beans").findValuesAsText("name");
     assertTrue("Invalid JSON: "  + jmxJson,
-        values.contains("java.lang:type=MemoryPool,name=Metaspace") ||
+        values.contains("java.lang:type=MemoryPool,name=Metaspace"));
-        values.contains("java.lang:name=Metaspace,type=MemoryPool"));
     assertTrue("Invalid JSON: "  + jmxJson, values.contains("java.lang:type=Runtime"));
   }
 }

java/TableFlattener/pom.xml

@@ -34,7 +34,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.11.0</version>
+        <version>3.3</version>
         <configuration>
           <source>1.8</source>
           <target>1.8</target>

java/datagenerator/pom.xml

@@ -81,7 +81,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.11.0</version>
+        <version>3.3</version>
         <configuration>
           <source>1.8</source>
           <target>1.8</target>
@@ -91,7 +91,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
-        <version>3.0.0</version>
+        <version>2.18</version>
         <configuration>
           <redirectTestOutputToFile>true</redirectTestOutputToFile>
         </configuration>

java/executor-deps/pom.xml

@@ -188,7 +188,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-dependency-plugin</artifactId>
-        <version>3.5.0</version>
+        <version>3.1.1</version>
         <executions>
           <execution>
             <id>write-executor-classpath</id>

java/ext-data-source/api/pom.xml

@@ -44,7 +44,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-source-plugin</artifactId>
-        <version>3.2.1</version>
+        <version>2.4</version>
         <executions>
           <execution>
             <id>attach-sources</id>
@@ -57,7 +57,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.11.0</version>
+        <version>3.3</version>
         <configuration>
           <source>1.8</source>
           <target>1.8</target>
@@ -67,7 +67,7 @@
       <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>build-helper-maven-plugin</artifactId>
-        <version>3.3.0</version>
+        <version>1.5</version>
         <executions>
           <!-- Tell maven about our generated files -->
           <execution>

java/ext-data-source/sample/pom.xml

@@ -48,7 +48,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-source-plugin</artifactId>
-        <version>3.2.1</version>
+        <version>2.4</version>
         <executions>
           <execution>
             <id>attach-sources</id>
@@ -61,7 +61,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.11.0</version>
+        <version>3.3</version>
         <configuration>
           <source>1.8</source>
           <target>1.8</target>

java/ext-data-source/test/pom.xml

@@ -48,7 +48,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.11.0</version>
+        <version>3.3</version>
         <configuration>
           <source>1.8</source>
           <target>1.8</target>

java/query-event-hook-api/pom.xml

@@ -35,7 +35,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-source-plugin</artifactId>
-        <version>3.2.1</version>
+        <version>2.4</version>
         <executions>
           <execution>
             <id>attach-sources</id>
@@ -48,7 +48,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.11.0</version>
+        <version>3.3</version>
         <configuration>
           <source>1.8</source>
           <target>1.8</target>

java/shaded-deps/hive-exec/pom.xml

@@ -55,7 +55,7 @@ the same dependencies
     <plugins>
       <plugin>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.4.1</version>
+        <version>3.2.1</version>
         <configuration>
           <artifactSet>
             <includes>

java/shaded-deps/s3a-aws-sdk/pom.xml

@@ -43,7 +43,7 @@ though some of them might not be necessary. The exclusions are sorted alphabetic
     <plugins>
       <plugin>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.4.1</version>
+        <version>3.2.1</version>
         <configuration>
           <artifactSet>
             <includes>

java/test-corrupt-hive-udfs/pom.xml

@@ -42,7 +42,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.11.0</version>
+        <version>3.3</version>
         <configuration>
           <source>1.8</source>
           <target>1.8</target>

java/test-hive-udfs/pom.xml

@@ -69,7 +69,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.11.0</version>
+        <version>3.3</version>
         <configuration>
           <source>1.8</source>
           <target>1.8</target>

java/yarn-extras/pom.xml

@@ -89,7 +89,7 @@
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
-        <version>3.11.0</version>
+        <version>3.3</version>
         <configuration>
           <source>1.8</source>
           <target>1.8</target>

testdata/bin/run-mini-dfs.sh

@@ -42,7 +42,7 @@ set +e
 $IMPALA_HOME/testdata/cluster/admin start_cluster
 if [[ $? != 0 ]]; then
   # Only issue Java version warning when running Java 7.
-  [ $IMPALA_JDK_VERSION -le 7 ] || exit $?
+  $JAVA -version 2>&1 | grep -q 'java version "1.7' || exit 1
 
   cat << EOF
 

testdata/bin/run-ranger-server.sh

@@ -31,11 +31,5 @@ fi
 unset CLASSPATH
 . $IMPALA_HOME/bin/impala-config.sh
 
-# Required to start Ranger with Java 11
+JAVA_OPTS="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=30130" \
-if [[ ! -d "${RANGER_HOME}"/ews/logs ]]; then
-  mkdir -p "${RANGER_HOME}"/ews/logs
-fi
-
-JAVA_DBG_SOCKET="-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=30130"
-JAVA_OPTS="-XX:+IgnoreUnrecognizedVMOptions -Xdebug ${JAVA_DBG_SOCKET}" \
     "${RANGER_HOME}"/ews/ranger-admin-services.sh restart

tests/verifiers/test_banned_log_messages.py

@@ -1,43 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# Test that impalad logs omit specific messages we shouldn't see.
-
-from __future__ import absolute_import, division, print_function
-import os
-import subprocess
-
-from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.skip import SkipIfDockerizedCluster
-
-
-class TestBannedLogMessages(ImpalaTestSuite):
-  """Verify that specific log messages are banned from Impala logs.
-
-  This test suite should be run after all the tests have been run.
-  """
-
-  @SkipIfDockerizedCluster.daemon_logs_not_exposed
-  def test_no_inaccessible_objects(self):
-    """Test that cluster logs do not contain InaccessibleObjectException"""
-    log_dir = os.environ["IMPALA_LOGS_DIR"]
-    message = 'InaccessibleObjectException'
-    for root, _, files in os.walk(log_dir):
-      for file in files:
-        log_file_path = os.path.join(root, file)
-        returncode = subprocess.call(['grep', message, log_file_path])
-        assert returncode == 1, "%s contains '%s'" % (log_file_path, message)