IMPALA-10198 (part 1): Unify Java in a single java/ directory

This changes all existing Java code to be submodules under
a single root pom. The root pom is impala-parent/pom.xml
with minor changes to add submodules.

This avoids most of the weird CMake/maven interactions,
because there is now a single maven invocation for all
the Java code.

This moves all the Java projects other than fe into
a top level java directory. fe is left where it is
to avoid disruption (but still is compiled via the
java directory's root pom). Various pieces of code
that reference the old locations are updated.

Based on research, there are two options for dealing
with the shaded dependencies. The first is to have an
entirely separate Maven project with a separate Maven
invocation. In this case, the consumers of the shaded
jars will see the reduced set of transitive dependencies.
The second is to have the shaded dependencies as modules
with a single Maven invocation. The consumer would see
all of the original transitive dependencies and need to
exclude them all. See MSHADE-206/MNG-5899. This chooses
the second.

This only moves code around and does not focus on version
numbers or making "mvn versions:set" work.

Testing:
 - Ran a core job
 - Verified existing maven commands from fe/ directory still work
 - Compared the *-classpath.txt files from fe and executor-deps
   and verified they are the same except for paths

Change-Id: I08773f4f9d7cb269b0491080078d6e6f490d8d7a
Reviewed-on: http://gerrit.cloudera.org:8080/16500
Reviewed-by: Joe McDonnell <joemcdonnell@cloudera.com>
Tested-by: Joe McDonnell <joemcdonnell@cloudera.com>

CMakeLists.txt

@@ -425,17 +425,10 @@ endif()
 add_subdirectory(common/function-registry)
 add_subdirectory(common/thrift)
 add_subdirectory(common/fbs)
-add_subdirectory(common/yarn-extras)
 add_subdirectory(common/protobuf)
 add_subdirectory(be)
 add_subdirectory(docker)
-add_subdirectory(mvn-deps/shaded-deps/hive-exec)
+add_subdirectory(java)
-add_subdirectory(mvn-deps/shaded-deps/s3a-aws-sdk)
-add_subdirectory(mvn-deps/executor-deps)
-add_subdirectory(fe)
-add_subdirectory(impala-parent)
-add_subdirectory(ext-data-source)
-add_subdirectory(query-event-hook-api)
 
 # Build target for all generated files which most backend code depends on
 add_custom_target(gen-deps ALL DEPENDS thrift-deps proto-deps fb-deps

bin/clean-cmake.sh

@@ -29,8 +29,8 @@ if [[ -z "${IMPALA_HOME}" || ! -d "${IMPALA_HOME}" ]]; then
 fi
 # Remove trailing /
 ROOT_DIR=${IMPALA_HOME%%/}
-for loc in "${ROOT_DIR}/ -maxdepth 1" "${ROOT_DIR}/be/" "${ROOT_DIR}/fe/" "${ROOT_DIR}/common/"\
+for loc in "${ROOT_DIR}/ -maxdepth 1" "${ROOT_DIR}/be/" "${ROOT_DIR}/java/" \
-           "${ROOT_DIR}/ext-data-source/"; do
+           "${ROOT_DIR}/common/" ; do
   find ${loc} \( -iname CMakeCache.txt -o -iname CMakeFiles \
        -o -iname CTestTestfile.cmake -o -iname cmake_install.cmake \) -exec rm -Rf {} +
 done

bin/clean.sh

@@ -30,7 +30,7 @@ setup_report_build_error
 "${MAKE_CMD:-make}" clean || :
 
 # clean the external data source project
-pushd "${IMPALA_HOME}/ext-data-source"
+pushd "${IMPALA_HOME}/java/ext-data-source"
 rm -rf api/generated-sources/*
 ${IMPALA_HOME}/bin/mvn-quiet.sh clean
 popd

bin/create_testdata.sh

@@ -29,13 +29,13 @@ bin=`cd "$bin"; pwd`
 DATALOC=$IMPALA_HOME/testdata/target
 
 # regenerate the test data generator
-cd $IMPALA_HOME/testdata
+cd $IMPALA_HOME/java/datagenerator
 ${IMPALA_HOME}/bin/mvn-quiet.sh clean
 ${IMPALA_HOME}/bin/mvn-quiet.sh package
 
 # find jars
 CP=""
-JARS=`find target/*.jar 2> /dev/null || true`
+JARS=`find ${IMPALA_HOME}/java/datagenerator/target/*.jar 2> /dev/null || true`
 for i in $JARS; do
     if [ -n "$CP" ]; then
         CP=${CP}:${i}
@@ -47,6 +47,8 @@ done
 # run test data generator
 echo $DATALOC
 mkdir -p $DATALOC
+pushd $IMPALA_HOME/testdata
 "$JAVA" -cp $CP org.apache.impala.datagenerator.TestDataGenerator $DATALOC
 "$JAVA" -cp $CP org.apache.impala.datagenerator.CsvToHBaseConverter
 echo "SUCCESS, data generated into $DATALOC"
+popd

buildall.sh

@@ -424,7 +424,7 @@ bootstrap_dependencies() {
 # Build the Impala frontend and its dependencies.
 build_fe() {
   generate_cmake_files $CMAKE_BUILD_TYPE
-  ${MAKE_CMD} ${IMPALA_MAKE_FLAGS} fe
+  ${MAKE_CMD} ${IMPALA_MAKE_FLAGS} java
 }
 
 # Build all components. The build type is specified as the first argument, and the
@@ -449,7 +449,7 @@ build_all_components() {
     MAKE_TARGETS="impalad statestored catalogd fesupport loggingsupport ImpalaUdf \
         udasample udfsample"
     if (( build_independent_targets )); then
-      MAKE_TARGETS+=" cscope fe tarballs"
+      MAKE_TARGETS+=" cscope java tarballs"
     fi
   fi
   ${MAKE_CMD} -j${IMPALA_BUILD_THREADS:-4} ${IMPALA_MAKE_FLAGS} ${MAKE_TARGETS}

common/thrift/CMakeLists.txt

@@ -177,7 +177,7 @@ set(THRIFT_INCLUDE_DIR_OPTION -I ${THRIFT_CONTRIB_DIR} -I $ENV{HIVE_METASTORE_TH
 set(BE_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/be/generated-sources)
 set(FE_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/fe/generated-sources)
 # TODO: avoid duplicating generated java classes
-set(EXT_DS_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/ext-data-source/api/generated-sources)
+set(EXT_DS_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/java/ext-data-source/api/generated-sources)
 set(PYTHON_OUTPUT_DIR ${CMAKE_SOURCE_DIR}/shell/)
 set(THRIFT11_PYTHON_OUTPUT_DIR ${PYTHON_OUTPUT_DIR}/build/thrift-11-gen)
 MESSAGE("Found output dir: " ${PYTHON_OUTPUT_DIR})

common/yarn-extras/CMakeLists.txt

@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# The dependency on ext-data-source here is fictional, but Maven does not like
-# concurrent invocations. These lead to opaque, non-deterministic errors due to
-# races in how Maven handles its ~/.m2/repository doc.
-add_custom_target(yarn-extras ALL DEPENDS impala-parent ext-data-source
-  COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
-)

docker/CMakeLists.txt

@@ -48,7 +48,7 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
     # with symlinks
     add_custom_target(impala_base_build_context_${build_type}
       COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py ${build_context_args}
-      DEPENDS daemons fe ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
+      DEPENDS daemons java ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py
       COMMENT "Creating impala base build context build_type=${build_type}."
       VERBATIM
     )

docker/setup_build_context.py

@@ -137,7 +137,7 @@ for jar in glob.glob(
 
 # Impala Executor dependencies.
 dep_classpath = file(os.path.join(IMPALA_HOME,
-    "mvn-deps/executor-deps/target/build-executor-deps-classpath.txt")).read()
+    "java/executor-deps/target/build-executor-deps-classpath.txt")).read()
 for jar in dep_classpath.split(":"):
   assert os.path.exists(jar), "missing jar from classpath: {0}".format(jar)
   symlink_file_into_dir(jar, EXEC_LIB_DIR)

ext-data-source/CMakeLists.txt

@@ -1,23 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# The dependency on shaded-deps is only added to avoid parallel downloads
-# of dependencies. For more details see IMPALA-7051, which was a similar issue.
-add_custom_target(ext-data-source ALL DEPENDS gen-deps impala-parent
-    shaded-deps-hive-exec shaded-deps-s3a-aws-sdk
-  COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
-)

fe/CMakeLists.txt

@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-add_custom_target(fe ALL DEPENDS
-  executor-deps shaded-deps-hive-exec shaded-deps-s3a-aws-sdk thrift-deps fb-deps
-      yarn-extras function-registry ext-data-source query-event-hook-api impala-parent
-  COMMAND ${CMAKE_SOURCE_DIR}/bin/mvn-quiet.sh -B install -DskipTests
-)

fe/pom.xml

@@ -24,7 +24,7 @@ under the License.
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
     <version>0.1-SNAPSHOT</version>
-    <relativePath>../impala-parent/pom.xml</relativePath>
+    <relativePath>../java/pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.impala</groupId>

java/CMakeLists.txt

@@ -15,6 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-add_custom_target(impala-parent ALL
+add_custom_target(java ALL DEPENDS gen-deps function-registry
   COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
 )

java/TableFlattener/pom.xml

@@ -23,7 +23,6 @@
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
     <version>0.1-SNAPSHOT</version>
-    <relativePath>../../impala-parent/pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.impala</groupId>

java/datagenerator/pom.xml

@@ -24,7 +24,6 @@ under the License.
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
     <version>0.1-SNAPSHOT</version>
-    <relativePath>../impala-parent/pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

java/executor-deps/pom.xml

@@ -35,7 +35,6 @@ under the License.
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
     <version>0.1-SNAPSHOT</version>
-    <relativePath>../../impala-parent/pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.impala</groupId>
@@ -107,6 +106,12 @@ under the License.
       <groupId>org.apache.impala</groupId>
       <artifactId>impala-minimal-s3a-aws-sdk</artifactId>
       <version>${project.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <dependency>
@@ -159,6 +164,12 @@ under the License.
       <groupId>org.apache.impala</groupId>
       <artifactId>impala-minimal-hive-exec</artifactId>
       <version>${project.version}</version>
+      <exclusions>
+        <exclusion>
+          <groupId>*</groupId>
+          <artifactId>*</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
 
     <!-- IMPALA-8766: Include Knox jars on the classpath -->

java/ext-data-source/pom.xml

@@ -23,7 +23,6 @@
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
     <version>0.1-SNAPSHOT</version>
-    <relativePath>../impala-parent/pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.impala</groupId>

java/pom.xml

@@ -219,4 +219,17 @@ under the License.
     </pluginRepository>
   </pluginRepositories>
 
+  <modules>
+    <module>datagenerator</module>
+    <module>executor-deps</module>
+    <module>ext-data-source</module>
+    <module>../fe</module>
+    <module>query-event-hook-api</module>
+    <module>shaded-deps/hive-exec</module>
+    <module>shaded-deps/s3a-aws-sdk</module>
+    <module>TableFlattener</module>
+    <module>test-hive-udfs</module>
+    <module>yarn-extras</module>
+  </modules>
+
 </project>

java/query-event-hook-api/pom.xml

@@ -23,7 +23,6 @@
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
     <version>0.1-SNAPSHOT</version>
-    <relativePath>../impala-parent/pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.impala</groupId>

java/shaded-deps/hive-exec/pom.xml

@@ -28,7 +28,7 @@ the same dependencies
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
     <version>0.1-SNAPSHOT</version>
-    <relativePath>../../impala-parent/pom.xml</relativePath>
+    <relativePath>../../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.impala</groupId>

java/shaded-deps/s3a-aws-sdk/pom.xml

@@ -26,7 +26,7 @@ though some of them might not be necessary. The exclusions are sorted alphabetic
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
     <version>0.1-SNAPSHOT</version>
-    <relativePath>../../impala-parent/pom.xml</relativePath>
+    <relativePath>../../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.impala</groupId>

java/test-hive-udfs/pom.xml

@@ -23,7 +23,6 @@ under the License.
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
     <version>0.1-SNAPSHOT</version>
-    <relativePath>../../impala-parent/pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

java/yarn-extras/pom.xml

@@ -23,7 +23,6 @@
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
     <version>0.1-SNAPSHOT</version>
-    <relativePath>../../impala-parent/pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.impala</groupId>

mvn-deps/executor-deps/CMakeLists.txt

@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-add_custom_target(executor-deps ALL DEPENDS shaded-deps-hive-exec shaded-deps-s3a-aws-sdk
-  COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
-)

mvn-deps/shaded-deps/hive-exec/CMakeLists.txt

@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-add_custom_target(shaded-deps-hive-exec ALL DEPENDS impala-parent
-  COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
-)

mvn-deps/shaded-deps/s3a-aws-sdk/CMakeLists.txt

@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-add_custom_target(shaded-deps-s3a-aws-sdk ALL DEPENDS impala-parent
-  COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
-)

query-event-hook-api/CMakeLists.txt

@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# The dependency on yarn-extras is only added to avoid parallel downloads
-# of dependencies. For more details see IMPALA-7051, which was a similar issue.
-add_custom_target(query-event-hook-api ALL DEPENDS gen-deps impala-parent yarn-extras
-  COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh -B install -DskipTests
-)

testdata/bin/copy-data-sources.sh

@@ -28,5 +28,5 @@ setup_report_build_error
 hadoop fs -mkdir -p ${FILESYSTEM_PREFIX}/test-warehouse/data-sources/
 
 hadoop fs -put -f \
-  ${IMPALA_HOME}/ext-data-source/test/target/impala-data-source-test-*.jar \
+  ${IMPALA_HOME}/java/ext-data-source/test/target/impala-data-source-test-*.jar \
   ${FILESYSTEM_PREFIX}/test-warehouse/data-sources/test-data-source.jar

testdata/bin/copy-udfs-udas.sh

@@ -51,7 +51,7 @@ then
   pushd "${IMPALA_HOME}"
   "${MAKE_CMD:-make}" ${IMPALA_MAKE_FLAGS} "-j${IMPALA_BUILD_THREADS:-4}" \
       TestUdas TestUdfs test-udfs-ir udfsample udasample udf-sample-ir uda-sample-ir
-  cd "${IMPALA_HOME}/tests/test-hive-udfs"
+  cd "${IMPALA_HOME}/java/test-hive-udfs"
   "${IMPALA_HOME}/bin/mvn-quiet.sh" package
   cp target/test-hive-udfs-1.0.jar "${IMPALA_HOME}/testdata/udfs/impala-hive-udfs.jar"
   # Change one of the Java files to make a new jar for testing purposes, then change it
@@ -64,9 +64,13 @@ then
        bash -c "sed s/'ReplaceStringUdf'/'NewReplaceStringUdf'/g '{}'" \; \
        > src/main/java/org/apache/impala/NewReplaceStringUdf.java
   "${IMPALA_HOME}/bin/mvn-quiet.sh" package
+  cp target/test-hive-udfs-1.0.jar \
+      "${IMPALA_HOME}/testdata/udfs/impala-hive-udfs-modified.jar"
   find . -type f -name 'TestUpdateUdf.java' -execdir \
        bash -c "sed -i s/'Text(\"New UDF\")'/'Text(\"Old UDF\")'/g '{}'" \;
   rm src/main/java/org/apache/impala/NewReplaceStringUdf.java
+  # Rebuild with the original version
+  "${IMPALA_HOME}/bin/mvn-quiet.sh" package
   popd
 fi
 
@@ -91,7 +95,7 @@ ln -s "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so" \
 mkdir "${UDF_TMP_DIR}/udf_test"
 ln -s "${IMPALA_HOME}/be/build/latest/testutil/libTestUdfs.so" "${UDF_TMP_DIR}/udf_test"
 ln -s "${HIVE_HOME}/lib/hive-exec-"*.jar "${UDF_TMP_DIR}/hive-exec.jar"
-ln -s "${IMPALA_HOME}/tests/test-hive-udfs/target/test-hive-udfs-1.0.jar" \
+ln -s "${IMPALA_HOME}/testdata/udfs/impala-hive-udfs.jar" \
   "${UDF_TMP_DIR}/impala-hive-udfs.jar"
 ln -s "${IMPALA_HOME}/be/build/latest/testutil/test-udfs.ll" "${UDF_TMP_DIR}"
 ln -s "${IMPALA_HOME}/be/build/latest/udf_samples/libudfsample.so" "${UDF_TMP_DIR}"

testdata/bin/create-load-data.sh

@@ -479,7 +479,7 @@ function load-custom-data {
 
 function build-and-copy-hive-udfs {
   # Build the test Hive UDFs
-  pushd ${IMPALA_HOME}/tests/test-hive-udfs
+  pushd ${IMPALA_HOME}/java/test-hive-udfs
   ${IMPALA_HOME}/bin/mvn-quiet.sh clean
   ${IMPALA_HOME}/bin/mvn-quiet.sh package
   popd

testdata/bin/generate-load-nested.sh

@@ -55,7 +55,7 @@ while getopts ":n:e:i:f" OPTION; do
 done
 
 if $FLATTEN; then
-  mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/testdata/TableFlattener/pom.xml" package
+  mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/java/TableFlattener/pom.xml" package
 fi
 
 RANDOM_SCHEMA_GENERATOR=${IMPALA_HOME}/testdata/bin/random_avro_schema.py;
@@ -79,12 +79,12 @@ do
   do
     FILE_NAME=$(basename ${AVRO_SCHEMA_PATH})
     TABLE_NAME="${FILE_NAME%.*}"
-    mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/testdata/pom.xml" exec:java \
+    mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/java/datagenerator/pom.xml" exec:java \
       -Dexec.mainClass="org.apache.impala.datagenerator.RandomNestedDataGenerator" \
       -Dexec.args="${AVRO_SCHEMA_PATH} ${NUM_ELEMENTS} ${NUM_FIELDS} ${DB_DIR}/${TABLE_NAME}/${TABLE_NAME}.parquet";
 
     if $FLATTEN; then
-      mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/testdata/TableFlattener/pom.xml" \
+      mvn $IMPALA_MAVEN_OPTIONS -f "${IMPALA_HOME}/java/TableFlattener/pom.xml" \
         exec:java -Dexec.mainClass=org.apache.impala.infra.tableflattener.Main \
         -Dexec.arguments="file://${DB_DIR}/${TABLE_NAME}/${TABLE_NAME}.parquet,file://${DB_DIR}/${TABLE_NAME}_flat,-sfile://${AVRO_SCHEMA_PATH}";
     fi

tests/custom_cluster/test_coordinators.py

@@ -145,7 +145,7 @@ class TestCoordinators(CustomClusterTestSuite):
     old_src_path = os.path.join(
         os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs.jar')
     new_src_path = os.path.join(
-        os.environ['IMPALA_HOME'], 'tests/test-hive-udfs/target/test-hive-udfs-1.0.jar')
+        os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs-modified.jar')
     tgt_dir = get_fs_path('/test-warehouse/{0}.db'.format(db_name))
     tgt_path = tgt_dir + "/tmp.jar"
 

tests/query_test/test_udfs.py

@@ -517,7 +517,7 @@ class TestUdfTargeted(TestUdfBase):
     old_udf = os.path.join(
         os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs.jar')
     new_udf = os.path.join(
-        os.environ['IMPALA_HOME'], 'tests/test-hive-udfs/target/test-hive-udfs-1.0.jar')
+        os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs-modified.jar')
     udf_dst = get_fs_path(
         '/test-warehouse/{0}.db/impala-hive-udfs.jar'.format(unique_database))
 
@@ -550,7 +550,7 @@ class TestUdfTargeted(TestUdfBase):
     old_udf = os.path.join(
         os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs.jar')
     new_udf = os.path.join(
-        os.environ['IMPALA_HOME'], 'tests/test-hive-udfs/target/test-hive-udfs-1.0.jar')
+        os.environ['IMPALA_HOME'], 'testdata/udfs/impala-hive-udfs-modified.jar')
     udf_dst = get_fs_path(
         '/test-warehouse/{0}.db/impala-hive-udfs.jar'.format(unique_database))
     old_function_name = "udf_update_test_create1"

tests/run-tests.py

@@ -44,7 +44,7 @@ VALID_TEST_DIRS = ['failure', 'query_test', 'stress', 'unittests', 'aux_query_te
 # new dir to the list of valid test dirs above. All dirs unders tests/ must be placed
 # into one of these lists, otherwise the script will throw an error. This list can be
 # removed once IMPALA-4417 has been resolved.
-TEST_HELPER_DIRS = ['aux_parquet_data_load', 'test-hive-udfs', 'comparison', 'benchmark',
+TEST_HELPER_DIRS = ['aux_parquet_data_load', 'comparison', 'benchmark',
                      'custom_cluster', 'util', 'experiments', 'verifiers', 'common',
                      'performance', 'beeswax', 'aux_custom_cluster_tests',
                      'authorization']