IMPALA-12212: Bump Maven to 3.9.2, pull dependencies in parallel

Maven 3.9.x offers a new dependency resolver, HttpClient, which allows
downloading project dependencies in parallel.

This patch bumps the Maven version installed by bootstrap_system.sh to
v3.9.2, and adds the flags enabling the new resolver to download
dependencies (including POM files) in parallel. Parallelism is set to
10 threads.

The flags are added to a project-specific Maven setting file in the
newly created java/.mvn directory. The settings file is added to the
RAT exclusion list in bin/rat_exclude_files.txt.

The --show-version flag is added for debugging purposes.

The same flags are added to the JAMM subproject as well.

The new resolver in Maven 3.9 has also changed the warning message
emitted for missing component checksums, so the new warning string
is added to the filter in bin/mvn-quiet.sh
Unfortunately Maven 3.9 has also changed the way it responds to missing
checksum files: the resolver now emits a stack trace when checksums
cannot be determined, and missing checksums are not explicitly ignored.

Detailed documentation for the new Maven resolver in Maven 3.9.0+ is
located at:
https://maven.apache.org/guides/mini/guide-resolver-transport.html
resolver configuration reference:
https://maven.apache.org/resolver/configuration.html

Tests:
- verified in a core-mode test run with Maven 3.9.2 installed
- verified in a local build using an earlier version of Maven
  to verify that the new default setting does not cause regressions
  with the old dependency resolver.

Change-Id: I75d05215effc724f5bd471646fb352f37443e185
Reviewed-on: http://gerrit.cloudera.org:8080/20142
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Reviewed-by: Michael Smith <michael.smith@cloudera.com>

bin/bootstrap_build.sh

@@ -48,12 +48,12 @@ sudo apt-get --yes install openjdk-${JDK_VERSION}-jdk openjdk-${JDK_VERSION}-sou
 export JAVA_HOME=/usr/lib/jvm/java-${JDK_VERSION}-openjdk-amd64
 
 # Download Maven since the packaged version is pretty old.
-if [ ! -d /usr/local/apache-maven-3.5.4 ]; then
+if [ ! -d /usr/local/apache-maven-3.9.2 ]; then
   sudo wget -nv \
-    https://archive.apache.org/dist/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz
+    https://archive.apache.org/dist/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz
-  sha512sum -c - <<< '2a803f578f341e164f6753e410413d16ab60fabe31dc491d1fe35c984a5cce696bc71f57757d4538fe7738be04065a216f3ebad4ef7e0ce1bb4c51bc36d6be86 apache-maven-3.5.4-bin.tar.gz'
+  sha512sum -c - <<< '900bdeeeae550d2d2b3920fe0e00e41b0069f32c019d566465015bdd1b3866395cbe016e22d95d25d51d3a5e614af2c83ec9b282d73309f644859bbad08b63db  apache-maven-3.9.2-bin.tar.gz'
-  sudo tar -C /usr/local -xzf apache-maven-3.5.4-bin.tar.gz
+  sudo tar -C /usr/local -xzf apache-maven-3.9.2-bin.tar.gz
-  sudo ln -s /usr/local/apache-maven-3.5.4/bin/mvn /usr/local/bin
+  sudo ln -s /usr/local/apache-maven-3.9.2/bin/mvn /usr/local/bin
 fi
 
 # Try to prepopulate the m2 directory to save time

bin/bootstrap_system.sh

@@ -340,18 +340,17 @@ redhat sudo yum install -y ccache
 # Clean up yum caches
 redhat sudo yum clean all
 
-# Download maven for all OSes, since the OS-packaged version can be
+# Download Maven since the packaged version is pretty old.
-# pretty old.
+if [ ! -d /usr/local/apache-maven-3.9.2 ]; then
-if [ ! -d /usr/local/apache-maven-3.5.4 ]; then
   sudo wget -nv \
-    https://archive.apache.org/dist/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz
+    https://archive.apache.org/dist/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz
-  sudo sha512sum -c - <<< '2a803f578f341e164f6753e410413d16ab60fabe31dc491d1fe35c984a5cce696bc71f57757d4538fe7738be04065a216f3ebad4ef7e0ce1bb4c51bc36d6be86  apache-maven-3.5.4-bin.tar.gz'
+  sha512sum -c - <<< '900bdeeeae550d2d2b3920fe0e00e41b0069f32c019d566465015bdd1b3866395cbe016e22d95d25d51d3a5e614af2c83ec9b282d73309f644859bbad08b63db  apache-maven-3.9.2-bin.tar.gz'
-  sudo tar -C /usr/local -x --no-same-owner -zf apache-maven-3.5.4-bin.tar.gz
+  sudo tar -C /usr/local -xzf apache-maven-3.9.2-bin.tar.gz
-  sudo ln -sf /usr/local/apache-maven-3.5.4/bin/mvn /usr/local/bin
+  sudo ln -s /usr/local/apache-maven-3.9.2/bin/mvn /usr/local/bin
 
   # reset permissions on redhat8
   # TODO: figure out why this is necessary for redhat8
-  MAVEN_DIRECTORY="/usr/local/apache-maven-3.5.4"
+  MAVEN_DIRECTORY="/usr/local/apache-maven-3.9.2"
   redhat8 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}
   redhat8 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}/{bin,boot}
   redhat9 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}

bin/mvn-quiet.sh

@@ -37,13 +37,13 @@ LOGGING_OPTIONS="-Dorg.slf4j.simpleLogger.showDateTime \
 # Filter out "Checksum validation failed" messages, as they are mostly harmless and
 # make it harder to search for failed tests in the console output. Limit the filtering
 # to WARNING messages.
-CHECKSUM_VALIDATION_FAILED_REGEX="[WARNING].*Checksum validation failed"
+CHECKSUM_VALIDATION_FAILED_REGEX="[WARNING].*(Checksum validation failed|Could not validate integrity of download)"
 
 # Always use maven's batch mode (-B), as it produces output that is easier to parse.
 if ! mvn -B $IMPALA_MAVEN_OPTIONS $LOGGING_OPTIONS "$@" | \
   tee -a "$LOG_FILE" | \
   grep -E -e WARNING -e ERROR -e SUCCESS -e FAILURE -e Test -e "Found Banned" | \
-  grep -v -i "${CHECKSUM_VALIDATION_FAILED_REGEX}"; then
+  grep -E -v -i "${CHECKSUM_VALIDATION_FAILED_REGEX}"; then
   echo "mvn $IMPALA_MAVEN_OPTIONS $@ exited with code $?"
   exit 1
 fi

bin/rat_exclude_files.txt

@@ -135,6 +135,7 @@ fe/src/test/resources/adusers.ldif
 fe/src/test/resources/hbase-jaas-client.conf.template
 fe/src/test/resources/hbase-jaas-server.conf.template
 fe/src/test/resources/users.ldif
+java/.mvn/maven.config
 java/toolchains.xml.tmpl
 testdata/AllTypesError/*.txt
 testdata/AllTypesErrorNoNulls/*.txt

java/.mvn/maven.config

@@ -0,0 +1,5 @@
+--show-version
+--batch-mode
+-Daether.dependencyCollector.impl=bf
+-Daether.connector.basic.threads=10
+-Daether.dependencyCollector.bf.threads=10

java/CMakeLists.txt

@@ -19,6 +19,12 @@ add_custom_target(validate_java_pom_versions ALL
   COMMAND $ENV{IMPALA_HOME}/bin/validate-java-pom-versions.sh
 )
 
+SET(JAMM_MAVEN_FLAGS --show-version --batch-mode)
+# Discover and download dependencies in parallel, using 10 threads
+SET(JAMM_MAVEN_FLAGS ${JAMM_MAVEN_FLAGS} -Daether.dependencyCollector.impl=bf)
+SET(JAMM_MAVEN_FLAGS ${JAMM_MAVEN_FLAGS} -Daether.connector.basic.threads=10)
+SET(JAMM_MAVEN_FLAGS ${JAMM_MAVEN_FLAGS} -Daether.dependencyCollector.bf.threads=10)
+
 include(ExternalProject)
 ExternalProject_Add(jamm
   GIT_REPOSITORY    https://github.com/jbellis/jamm
@@ -27,7 +33,7 @@ ExternalProject_Add(jamm
   PATCH_COMMAND     sed -i.bak s/0.4.0-SNAPSHOT/0.4.0-IMPALA/ pom.xml
   CONFIGURE_COMMAND sed s:JAVA_HOME:$ENV{JAVA_HOME}:g ${CMAKE_CURRENT_SOURCE_DIR}/toolchains.xml.tmpl > toolchains.xml
   BUILD_IN_SOURCE   true
-  BUILD_COMMAND     mvn --batch-mode --toolchains toolchains.xml install -Dmaven.test.skip=true -DskipTests
+  BUILD_COMMAND     mvn install ${JAMM_MAVEN_FLAGS} --toolchains toolchains.xml -Dmaven.test.skip=true -DskipTests
   INSTALL_COMMAND   ""
 )