From 5eea4f6f797e5549623a4dc4891ec82af08a41ac Mon Sep 17 00:00:00 2001
From: Joe McDonnell <joemcdonnell@cloudera.com>
Date: Mon, 6 Oct 2025 16:36:24 -0700
Subject: [PATCH] IMPALA-14559: Ship calcite-planner jar in Impala packages

This adds the java/impala-package Maven project to make it easier
to ship / test the Calcite planner. impala-package has a dependency
on impala-frontend and calcite-planner, so its classpath requires
no extra work when constructing the classpath.

An additional cleanup is that this no longer puts the
impala-frontend-*-tests.jar on the classpath by default. This requires
updating the query event hooks test, as it relies on that jar being
present.

This does not change the default value for the use_calcite_planner
query option, so there is no change in behavior.

Testing:
 - Ran a core job
 - Built docker images and OS packages locally

Change-Id: I81dec2a5b59e279229a735c8bb1a23c77111a793
Reviewed-on: http://gerrit.cloudera.org:8080/23497
Reviewed-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
---
 bin/set-classpath.sh                          | 34 ++------
 docker/setup_build_context.py                 | 24 +++---
 java/impala-package/pom.xml                   | 79 +++++++++++++++++++
 java/pom.xml                                  |  1 +
 package/CMakeLists.txt                        |  4 +-
 .../custom_cluster/test_query_event_hooks.py  | 17 ++++
 6 files changed, 119 insertions(+), 40 deletions(-)
 create mode 100644 java/impala-package/pom.xml

diff --git a/bin/set-classpath.sh b/bin/set-classpath.sh
index abfe54e43..a3e051fb0 100644
--- a/bin/set-classpath.sh
+++ b/bin/set-classpath.sh
@@ -33,40 +33,22 @@ fi
 # frontend tests. This uses jars to avoid issues related to that
 # inconsistency.
 CLASSPATH=\
-"$IMPALA_HOME"/fe/src/test/resources:\
-"$IMPALA_HOME"/fe/target/impala-frontend-${IMPALA_VERSION}.jar:\
-"$IMPALA_HOME"/fe/target/dependency:\
-"$IMPALA_HOME"/fe/target/impala-frontend-${IMPALA_VERSION}-tests.jar:
+"$IMPALA_HOME"/fe/src/test/resources:
 
-FE_CP_FILE="$IMPALA_HOME/fe/target/build-classpath.txt"
+PACKAGE_CP_FILE="$IMPALA_HOME/java/impala-package/target/package-classpath.txt"
 
-if [ ! -s "$FE_CP_FILE" ]; then
-  >&2 echo FE classpath file $FE_CP_FILE missing.
-  >&2 echo Build the front-end first.
+if [ ! -s "$PACKAGE_CP_FILE" ]; then
+  >&2 echo impala-package classpath file "$PACKAGE_CP_FILE" missing.
+  >&2 echo Build the impala-package first.
   return 1
 fi
 
-CLASSPATH=$(cat $FE_CP_FILE):"$CLASSPATH"
-
-# Currently the Calcite planner is experimental and not included by default.
-# If the USE_CALCITE_PLANNER is explicitly set, then the jar dependencies
-# are added to the CLASSPATH
-USE_CALCITE_PLANNER=${USE_CALCITE_PLANNER:-false}
-if [ "true" = "$USE_CALCITE_PLANNER" ]; then
-
- CALCITE_CP_FILE="$IMPALA_HOME/java/calcite-planner/target/calcite-build-classpath.txt"
- if [ ! -s "$CALCITE_CP_FILE" ]; then
-    >&2 echo Calcite front-end classpath file $CALCITE_CP_FILE missing.
-    >&2 echo Build the Calcite front-end first.
-    return 1
-  fi
-  CLASSPATH="$CLASSPATH":$(cat $CALCITE_CP_FILE):\
-"$IMPALA_HOME"/java/calcite-planner/target/calcite-planner-${IMPALA_VERSION}.jar:
-fi
+CLASSPATH=$(cat "$PACKAGE_CP_FILE"):"$CLASSPATH"
 
 if [[ "${1:-notest}" = "test" ]]; then
   FE_TEST_CP_FILE="$IMPALA_HOME/fe/target/test-classpath.txt"
-  CLASSPATH=$(cat $FE_TEST_CP_FILE):"$CLASSPATH"
+  CLASSPATH=$(cat "$FE_TEST_CP_FILE"):\
+"$IMPALA_HOME"/fe/target/impala-frontend-${IMPALA_VERSION}-tests.jar:"$CLASSPATH"
 fi
 
 : ${CUSTOM_CLASSPATH=}
diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py
index 1761452cc..0dd6714cc 100755
--- a/docker/setup_build_context.py
+++ b/docker/setup_build_context.py
@@ -193,26 +193,28 @@ if args.utility_context:
       os.path.join(IMPALA_HOME, "docker/utility_entrypoint.sh"), BIN_DIR)
 else:
   # Impala Coordinator dependencies.
+  impala_package_classpath_file = "java/impala-package/target/package-classpath.txt"
   num_jars_on_classpath = 0
-  dep_classpath = open(os.path.join(IMPALA_HOME, "fe/target/build-classpath.txt")).read()
+  num_frontend_jars = 0
+  num_calcite_jars = 0
+  dep_classpath = open(os.path.join(IMPALA_HOME, impala_package_classpath_file)).read()
   for jar in dep_classpath.split(":"):
     num_jars_on_classpath += 1
     assert os.path.exists(jar), "missing jar from classpath: {0}".format(jar)
+    if jar.find("calcite-planner") != -1:
+      assert jar.find("tests") == -1
+      num_calcite_jars += 1
+    if jar.find("impala-frontend") != -1:
+      assert jar.find("tests") == -1
+      num_frontend_jars += 1
     symlink_file_into_dir(jar, LIB_DIR)
 
   if num_jars_on_classpath == 0:
     raise Exception("No jars listed in {0}".format(os.path.join(IMPALA_HOME,
-        "fe/target/build-classpath.txt")))
+        impala_package_classpath_file)))
 
-  # Impala Coordinator jars.
-  num_frontend_jars = 0
-  for jar in glob.glob(os.path.join(IMPALA_HOME, "fe/target/impala-frontend-*.jar")):
-    # Ignore the tests jar
-    if jar.find("-tests") != -1:
-      continue
-    symlink_file_into_dir(jar, LIB_DIR)
-    num_frontend_jars += 1
-  # There must be exactly one impala-frontend jar.
+  # There must be exactly one jar for each of impala-frontend and calcite-planner
+  assert num_calcite_jars == 1
   assert num_frontend_jars == 1
 
   # Impala Executor dependencies.
diff --git a/java/impala-package/pom.xml b/java/impala-package/pom.xml
new file mode 100644
index 000000000..e256340e4
--- /dev/null
+++ b/java/impala-package/pom.xml
@@ -0,0 +1,79 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <groupId>org.apache.impala</groupId>
+    <artifactId>impala-parent</artifactId>
+    <version>5.0.0-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+
+  <artifactId>impala-package</artifactId>
+  <packaging>pom</packaging>
+  <name>Apache Impala Package</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.impala</groupId>
+      <artifactId>impala-frontend</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.impala</groupId>
+      <artifactId>calcite-planner</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>copy-dependencies</id>
+            <phase>package</phase>
+            <goals>
+              <goal>copy-dependencies</goal>
+            </goals>
+            <configuration>
+              <excludeTypes>pom</excludeTypes>
+              <includeScope>runtime</includeScope>
+              <silent>true</silent>
+            </configuration>
+          </execution>
+          <execution>
+            <id>write-classpath</id>
+            <goals>
+              <goal>build-classpath</goal>
+            </goals>
+            <configuration>
+              <outputFile>${project.build.directory}/package-classpath.txt</outputFile>
+              <includeScope>runtime</includeScope>
+              <excludeTypes>pom</excludeTypes>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+</project>
diff --git a/java/pom.xml b/java/pom.xml
index 1a837e69b..4eed32513 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -579,6 +579,7 @@ under the License.
     <module>../fe</module>
     <module>external-frontend</module>
     <module>calcite-planner</module>
+    <module>impala-package</module>
     <module>query-event-hook-api</module>
     <module>shaded-deps/hive-exec</module>
     <module>shaded-deps/s3a-aws-sdk</module>
diff --git a/package/CMakeLists.txt b/package/CMakeLists.txt
index d264718ee..8e4c4273f 100644
--- a/package/CMakeLists.txt
+++ b/package/CMakeLists.txt
@@ -25,9 +25,7 @@ message(STATUS "DISTRIB_VERSION_ID: ${OS_DISTRIB_VERSION_ID}")
 install(FILES ${CMAKE_SOURCE_DIR}/LICENSE.txt DESTINATION ${IMPALA_INSTALLDIR} RENAME LICENSE)
 install(FILES ${CMAKE_SOURCE_DIR}/NOTICE.txt DESTINATION ${IMPALA_INSTALLDIR} RENAME NOTICE)
 install(DIRECTORY "${CMAKE_SOURCE_DIR}/www/" DESTINATION ${IMPALA_INSTALLDIR}/www)
-install(FILES ${CMAKE_SOURCE_DIR}/fe/target/impala-frontend-$ENV{IMPALA_VERSION}.jar
-  DESTINATION ${IMPALA_INSTALLDIR}/lib/jars)
-install(DIRECTORY ${CMAKE_SOURCE_DIR}/fe/target/dependency/
+install(DIRECTORY ${CMAKE_SOURCE_DIR}/java/impala-package/target/dependency/
   DESTINATION ${IMPALA_INSTALLDIR}/lib/jars
   FILES_MATCHING PATTERN "*.jar")
 
diff --git a/tests/custom_cluster/test_query_event_hooks.py b/tests/custom_cluster/test_query_event_hooks.py
index 84ca8852a..dc45a1cbf 100644
--- a/tests/custom_cluster/test_query_event_hooks.py
+++ b/tests/custom_cluster/test_query_event_hooks.py
@@ -18,17 +18,31 @@
 # Client tests for Query Event Hooks
 
 from __future__ import absolute_import, division, print_function
+import os
 import pytest
 
 from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
 
 
+def add_frontend_test_jar_to_classpath():
+  """Put the impala-frontend-*-tests.jar on the classpath"""
+  os.environ["CUSTOM_CLASSPATH"] = os.path.join(os.getenv("IMPALA_HOME"), "fe",
+        "target", "impala-frontend-{0}-tests.jar".format(os.getenv("IMPALA_VERSION")))
+
+
 class TestHooks(CustomClusterTestSuite):
   """
   Tests for FE QueryEventHook invocations.
   """
   DUMMY_HOOK = "org.apache.impala.testutil.DummyQueryEventHook"
 
+  @classmethod
+  def setup_class(cls):
+    super(TestHooks, cls).setup_class()
+    # This needs the impala-frontend-*-tests.jar on the classpath to make the test query
+    # hooks available.
+    add_frontend_test_jar_to_classpath()
+
   @pytest.mark.execute_serially
   @CustomClusterTestSuite.with_args(
       impala_log_dir="{query_event_hooks_log}",
@@ -65,6 +79,9 @@ class TestHooksStartupFail(CustomClusterTestSuite):
     if cls.exploration_strategy() != 'exhaustive':
       pytest.skip('runs only in exhaustive')
     super(TestHooksStartupFail, cls).setup_class()
+    # This needs the impala-frontend-*-tests.jar on the classpath to make the test query
+    # hooks available.
+    add_frontend_test_jar_to_classpath()
 
   FAILING_HOOK = "org.apache.impala.testutil.AlwaysErrorQueryEventHook"
   NONEXIST_HOOK = "captain.hook"