IMPALA-14559: Ship calcite-planner jar in Impala packages

This adds the java/impala-package Maven project to make it easier
to ship / test the Calcite planner. impala-package has a dependency
on impala-frontend and calcite-planner, so its classpath requires
no extra work when constructing the classpath.

An additional cleanup is that this no longer puts the
impala-frontend-*-tests.jar on the classpath by default. This requires
updating the query event hooks test, as it relies on that jar being
present.

This does not change the default value for the use_calcite_planner
query option, so there is no change in behavior.

Testing:
 - Ran a core job
 - Built docker images and OS packages locally

Change-Id: I81dec2a5b59e279229a735c8bb1a23c77111a793
Reviewed-on: http://gerrit.cloudera.org:8080/23497
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>

bin/set-classpath.sh

@@ -33,40 +33,22 @@ fi
 # frontend tests. This uses jars to avoid issues related to that
 # inconsistency.
 CLASSPATH=\
-"$IMPALA_HOME"/fe/src/test/resources:\
+"$IMPALA_HOME"/fe/src/test/resources:
-"$IMPALA_HOME"/fe/target/impala-frontend-${IMPALA_VERSION}.jar:\
-"$IMPALA_HOME"/fe/target/dependency:\
-"$IMPALA_HOME"/fe/target/impala-frontend-${IMPALA_VERSION}-tests.jar:
 
-FE_CP_FILE="$IMPALA_HOME/fe/target/build-classpath.txt"
+PACKAGE_CP_FILE="$IMPALA_HOME/java/impala-package/target/package-classpath.txt"
 
-if [ ! -s "$FE_CP_FILE" ]; then
+if [ ! -s "$PACKAGE_CP_FILE" ]; then
-  >&2 echo FE classpath file $FE_CP_FILE missing.
+  >&2 echo impala-package classpath file "$PACKAGE_CP_FILE" missing.
-  >&2 echo Build the front-end first.
+  >&2 echo Build the impala-package first.
   return 1
 fi
 
-CLASSPATH=$(cat $FE_CP_FILE):"$CLASSPATH"
+CLASSPATH=$(cat "$PACKAGE_CP_FILE"):"$CLASSPATH"
-
-# Currently the Calcite planner is experimental and not included by default.
-# If the USE_CALCITE_PLANNER is explicitly set, then the jar dependencies
-# are added to the CLASSPATH
-USE_CALCITE_PLANNER=${USE_CALCITE_PLANNER:-false}
-if [ "true" = "$USE_CALCITE_PLANNER" ]; then
-
- CALCITE_CP_FILE="$IMPALA_HOME/java/calcite-planner/target/calcite-build-classpath.txt"
- if [ ! -s "$CALCITE_CP_FILE" ]; then
-    >&2 echo Calcite front-end classpath file $CALCITE_CP_FILE missing.
-    >&2 echo Build the Calcite front-end first.
-    return 1
-  fi
-  CLASSPATH="$CLASSPATH":$(cat $CALCITE_CP_FILE):\
-"$IMPALA_HOME"/java/calcite-planner/target/calcite-planner-${IMPALA_VERSION}.jar:
-fi
 
 if [[ "${1:-notest}" = "test" ]]; then
   FE_TEST_CP_FILE="$IMPALA_HOME/fe/target/test-classpath.txt"
-  CLASSPATH=$(cat $FE_TEST_CP_FILE):"$CLASSPATH"
+  CLASSPATH=$(cat "$FE_TEST_CP_FILE"):\
+"$IMPALA_HOME"/fe/target/impala-frontend-${IMPALA_VERSION}-tests.jar:"$CLASSPATH"
 fi
 
 : ${CUSTOM_CLASSPATH=}

docker/setup_build_context.py

@@ -193,26 +193,28 @@ if args.utility_context:
       os.path.join(IMPALA_HOME, "docker/utility_entrypoint.sh"), BIN_DIR)
 else:
   # Impala Coordinator dependencies.
+  impala_package_classpath_file = "java/impala-package/target/package-classpath.txt"
   num_jars_on_classpath = 0
-  dep_classpath = open(os.path.join(IMPALA_HOME, "fe/target/build-classpath.txt")).read()
+  num_frontend_jars = 0
+  num_calcite_jars = 0
+  dep_classpath = open(os.path.join(IMPALA_HOME, impala_package_classpath_file)).read()
   for jar in dep_classpath.split(":"):
     num_jars_on_classpath += 1
     assert os.path.exists(jar), "missing jar from classpath: {0}".format(jar)
+    if jar.find("calcite-planner") != -1:
+      assert jar.find("tests") == -1
+      num_calcite_jars += 1
+    if jar.find("impala-frontend") != -1:
+      assert jar.find("tests") == -1
+      num_frontend_jars += 1
     symlink_file_into_dir(jar, LIB_DIR)
 
   if num_jars_on_classpath == 0:
     raise Exception("No jars listed in {0}".format(os.path.join(IMPALA_HOME,
-        "fe/target/build-classpath.txt")))
+        impala_package_classpath_file)))
 
-  # Impala Coordinator jars.
+  # There must be exactly one jar for each of impala-frontend and calcite-planner
-  num_frontend_jars = 0
+  assert num_calcite_jars == 1
-  for jar in glob.glob(os.path.join(IMPALA_HOME, "fe/target/impala-frontend-*.jar")):
-    # Ignore the tests jar
-    if jar.find("-tests") != -1:
-      continue
-    symlink_file_into_dir(jar, LIB_DIR)
-    num_frontend_jars += 1
-  # There must be exactly one impala-frontend jar.
   assert num_frontend_jars == 1
 
   # Impala Executor dependencies.

java/impala-package/pom.xml

@@ -0,0 +1,79 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <groupId>org.apache.impala</groupId>
+    <artifactId>impala-parent</artifactId>
+    <version>5.0.0-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+
+  <artifactId>impala-package</artifactId>
+  <packaging>pom</packaging>
+  <name>Apache Impala Package</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.impala</groupId>
+      <artifactId>impala-frontend</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.impala</groupId>
+      <artifactId>calcite-planner</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>copy-dependencies</id>
+            <phase>package</phase>
+            <goals>
+              <goal>copy-dependencies</goal>
+            </goals>
+            <configuration>
+              <excludeTypes>pom</excludeTypes>
+              <includeScope>runtime</includeScope>
+              <silent>true</silent>
+            </configuration>
+          </execution>
+          <execution>
+            <id>write-classpath</id>
+            <goals>
+              <goal>build-classpath</goal>
+            </goals>
+            <configuration>
+              <outputFile>${project.build.directory}/package-classpath.txt</outputFile>
+              <includeScope>runtime</includeScope>
+              <excludeTypes>pom</excludeTypes>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>

java/pom.xml

@@ -579,6 +579,7 @@ under the License.
     <module>../fe</module>
     <module>external-frontend</module>
     <module>calcite-planner</module>
+    <module>impala-package</module>
     <module>query-event-hook-api</module>
     <module>shaded-deps/hive-exec</module>
     <module>shaded-deps/s3a-aws-sdk</module>

package/CMakeLists.txt

@@ -25,9 +25,7 @@ message(STATUS "DISTRIB_VERSION_ID: ${OS_DISTRIB_VERSION_ID}")
 install(FILES ${CMAKE_SOURCE_DIR}/LICENSE.txt DESTINATION ${IMPALA_INSTALLDIR} RENAME LICENSE)
 install(FILES ${CMAKE_SOURCE_DIR}/NOTICE.txt DESTINATION ${IMPALA_INSTALLDIR} RENAME NOTICE)
 install(DIRECTORY "${CMAKE_SOURCE_DIR}/www/" DESTINATION ${IMPALA_INSTALLDIR}/www)
-install(FILES ${CMAKE_SOURCE_DIR}/fe/target/impala-frontend-$ENV{IMPALA_VERSION}.jar
+install(DIRECTORY ${CMAKE_SOURCE_DIR}/java/impala-package/target/dependency/
-  DESTINATION ${IMPALA_INSTALLDIR}/lib/jars)
-install(DIRECTORY ${CMAKE_SOURCE_DIR}/fe/target/dependency/
   DESTINATION ${IMPALA_INSTALLDIR}/lib/jars
   FILES_MATCHING PATTERN "*.jar")
 

tests/custom_cluster/test_query_event_hooks.py

@@ -18,17 +18,31 @@
 # Client tests for Query Event Hooks
 
 from __future__ import absolute_import, division, print_function
+import os
 import pytest
 
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 
 
+def add_frontend_test_jar_to_classpath():
+  """Put the impala-frontend-*-tests.jar on the classpath"""
+  os.environ["CUSTOM_CLASSPATH"] = os.path.join(os.getenv("IMPALA_HOME"), "fe",
+        "target", "impala-frontend-{0}-tests.jar".format(os.getenv("IMPALA_VERSION")))
+
+
 class TestHooks(CustomClusterTestSuite):
   """
   Tests for FE QueryEventHook invocations.
   """
   DUMMY_HOOK = "org.apache.impala.testutil.DummyQueryEventHook"
 
+  @classmethod
+  def setup_class(cls):
+    super(TestHooks, cls).setup_class()
+    # This needs the impala-frontend-*-tests.jar on the classpath to make the test query
+    # hooks available.
+    add_frontend_test_jar_to_classpath()
+
   @pytest.mark.execute_serially
   @CustomClusterTestSuite.with_args(
       impala_log_dir="{query_event_hooks_log}",
@@ -65,6 +79,9 @@ class TestHooksStartupFail(CustomClusterTestSuite):
     if cls.exploration_strategy() != 'exhaustive':
       pytest.skip('runs only in exhaustive')
     super(TestHooksStartupFail, cls).setup_class()
+    # This needs the impala-frontend-*-tests.jar on the classpath to make the test query
+    # hooks available.
+    add_frontend_test_jar_to_classpath()
 
   FAILING_HOOK = "org.apache.impala.testutil.AlwaysErrorQueryEventHook"
   NONEXIST_HOOK = "captain.hook"