jprdonnelly/impala
1
0
Fork 0
mirror of https://github.com/apache/impala.git synced 2025-12-19 18:12:08 -05:00
Code Issues Projects Releases Wiki Activity

IMPALA-12212: Bump Maven to 3.9.2, pull dependencies in parallel

Browse Source 
Maven 3.9.x offers a new dependency resolver, HttpClient, which allows
downloading project dependencies in parallel.

This patch bumps the Maven version installed by bootstrap_system.sh to
v3.9.2, and adds the flags enabling the new resolver to download
dependencies (including POM files) in parallel. Parallelism is set to
10 threads.

The flags are added to a project-specific Maven setting file in the
newly created java/.mvn directory. The settings file is added to the
RAT exclusion list in bin/rat_exclude_files.txt.

The --show-version flag is added for debugging purposes.

The same flags are added to the JAMM subproject as well.

The new resolver in Maven 3.9 has also changed the warning message
emitted for missing component checksums, so the new warning string
is added to the filter in bin/mvn-quiet.sh
Unfortunately Maven 3.9 has also changed the way it responds to missing
checksum files: the resolver now emits a stack trace when checksums
cannot be determined, and missing checksums are not explicitly ignored.

Detailed documentation for the new Maven resolver in Maven 3.9.0+ is
located at:
https://maven.apache.org/guides/mini/guide-resolver-transport.html
resolver configuration reference:
https://maven.apache.org/resolver/configuration.html

Tests:
- verified in a core-mode test run with Maven 3.9.2 installed
- verified in a local build using an earlier version of Maven
  to verify that the new default setting does not cause regressions
  with the old dependency resolver.

Change-Id: I75d05215effc724f5bd471646fb352f37443e185
Reviewed-on: http://gerrit.cloudera.org:8080/20142
Tested-by: Impala Public Jenkins <impala-public-jenkins@cloudera.com>
Reviewed-by: Michael Smith <michael.smith@cloudera.com>
This commit is contained in:
Laszlo Gaal
2020-02-22 18:12:12 +01:00
committed by Michael Smith
parent af3f56e6d1
commit ee069687fc
6 changed files with 27 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
bin/bootstrap_build.sh
View File

@@ -48,12 +48,12 @@ sudo apt-get --yes install openjdk-${JDK_VERSION}-jdk openjdk-${JDK_VERSION}-sou
export JAVA_HOME=/usr/lib/jvm/java-${JDK_VERSION}-openjdk-amd64
# Download Maven since the packaged version is pretty old.
if [ ! -d /usr/local/apache-maven-3.5.4 ]; then
if [ ! -d /usr/local/apache-maven-3.9.2 ]; then
sudo wget -nv \
https://archive.apache.org/dist/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz
sha512sum -c - <<< '2a803f578f341e164f6753e410413d16ab60fabe31dc491d1fe35c984a5cce696bc71f57757d4538fe7738be04065a216f3ebad4ef7e0ce1bb4c51bc36d6be86 apache-maven-3.5.4-bin.tar.gz'
sudo tar -C /usr/local -xzf apache-maven-3.5.4-bin.tar.gz
sudo ln -s /usr/local/apache-maven-3.5.4/bin/mvn /usr/local/bin
https://archive.apache.org/dist/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz
sha512sum -c - <<< '900bdeeeae550d2d2b3920fe0e00e41b0069f32c019d566465015bdd1b3866395cbe016e22d95d25d51d3a5e614af2c83ec9b282d73309f644859bbad08b63db apache-maven-3.9.2-bin.tar.gz'
sudo tar -C /usr/local -xzf apache-maven-3.9.2-bin.tar.gz
sudo ln -s /usr/local/apache-maven-3.9.2/bin/mvn /usr/local/bin
fi
# Try to prepopulate the m2 directory to save time

15
bin/bootstrap_system.sh
View File

@@ -340,18 +340,17 @@ redhat sudo yum install -y ccache
# Clean up yum caches
redhat sudo yum clean all
# Download maven for all OSes, since the OS-packaged version can be
# pretty old.
if [ ! -d /usr/local/apache-maven-3.5.4 ]; then
# Download Maven since the packaged version is pretty old.
if [ ! -d /usr/local/apache-maven-3.9.2 ]; then
sudo wget -nv \
https://archive.apache.org/dist/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz
sudo sha512sum -c - <<< '2a803f578f341e164f6753e410413d16ab60fabe31dc491d1fe35c984a5cce696bc71f57757d4538fe7738be04065a216f3ebad4ef7e0ce1bb4c51bc36d6be86 apache-maven-3.5.4-bin.tar.gz'
sudo tar -C /usr/local -x --no-same-owner -zf apache-maven-3.5.4-bin.tar.gz
sudo ln -sf /usr/local/apache-maven-3.5.4/bin/mvn /usr/local/bin
https://archive.apache.org/dist/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz
sha512sum -c - <<< '900bdeeeae550d2d2b3920fe0e00e41b0069f32c019d566465015bdd1b3866395cbe016e22d95d25d51d3a5e614af2c83ec9b282d73309f644859bbad08b63db apache-maven-3.9.2-bin.tar.gz'
sudo tar -C /usr/local -xzf apache-maven-3.9.2-bin.tar.gz
sudo ln -s /usr/local/apache-maven-3.9.2/bin/mvn /usr/local/bin
# reset permissions on redhat8
# TODO: figure out why this is necessary for redhat8
MAVEN_DIRECTORY="/usr/local/apache-maven-3.5.4"
MAVEN_DIRECTORY="/usr/local/apache-maven-3.9.2"
redhat8 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}
redhat8 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}/{bin,boot}
redhat9 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}

4
bin/mvn-quiet.sh
View File

@@ -37,13 +37,13 @@ LOGGING_OPTIONS="-Dorg.slf4j.simpleLogger.showDateTime \
# Filter out "Checksum validation failed" messages, as they are mostly harmless and
# make it harder to search for failed tests in the console output. Limit the filtering
# to WARNING messages.
CHECKSUM_VALIDATION_FAILED_REGEX="[WARNING].*Checksum validation failed"
CHECKSUM_VALIDATION_FAILED_REGEX="[WARNING].*(Checksum validation failed|Could not validate integrity of download)"
# Always use maven's batch mode (-B), as it produces output that is easier to parse.
if ! mvn -B $IMPALA_MAVEN_OPTIONS $LOGGING_OPTIONS "$@" | \
tee -a "$LOG_FILE" | \
grep -E -e WARNING -e ERROR -e SUCCESS -e FAILURE -e Test -e "Found Banned" | \
grep -v -i "${CHECKSUM_VALIDATION_FAILED_REGEX}"; then
grep -E -v -i "${CHECKSUM_VALIDATION_FAILED_REGEX}"; then
echo "mvn $IMPALA_MAVEN_OPTIONS $@ exited with code $?"
exit 1
fi

1
bin/rat_exclude_files.txt
View File

@@ -135,6 +135,7 @@ fe/src/test/resources/adusers.ldif
fe/src/test/resources/hbase-jaas-client.conf.template
fe/src/test/resources/hbase-jaas-server.conf.template
fe/src/test/resources/users.ldif
java/.mvn/maven.config
java/toolchains.xml.tmpl
testdata/AllTypesError/*.txt
testdata/AllTypesErrorNoNulls/*.txt

5
java/.mvn/maven.config Normal file
View File

@@ -0,0 +1,5 @@
--show-version
--batch-mode
-Daether.dependencyCollector.impl=bf
-Daether.connector.basic.threads=10
-Daether.dependencyCollector.bf.threads=10

8
java/CMakeLists.txt
View File

@@ -19,6 +19,12 @@ add_custom_target(validate_java_pom_versions ALL
COMMAND $ENV{IMPALA_HOME}/bin/validate-java-pom-versions.sh
)
SET(JAMM_MAVEN_FLAGS --show-version --batch-mode)
# Discover and download dependencies in parallel, using 10 threads
SET(JAMM_MAVEN_FLAGS ${JAMM_MAVEN_FLAGS} -Daether.dependencyCollector.impl=bf)
SET(JAMM_MAVEN_FLAGS ${JAMM_MAVEN_FLAGS} -Daether.connector.basic.threads=10)
SET(JAMM_MAVEN_FLAGS ${JAMM_MAVEN_FLAGS} -Daether.dependencyCollector.bf.threads=10)
include(ExternalProject)
ExternalProject_Add(jamm
GIT_REPOSITORY https://github.com/jbellis/jamm
@@ -27,7 +33,7 @@ ExternalProject_Add(jamm
PATCH_COMMAND sed -i.bak s/0.4.0-SNAPSHOT/0.4.0-IMPALA/ pom.xml
CONFIGURE_COMMAND sed s:JAVA_HOME:$ENV{JAVA_HOME}:g ${CMAKE_CURRENT_SOURCE_DIR}/toolchains.xml.tmpl > toolchains.xml
BUILD_IN_SOURCE true
BUILD_COMMAND mvn --batch-mode --toolchains toolchains.xml install -Dmaven.test.skip=true -DskipTests
BUILD_COMMAND mvn install ${JAMM_MAVEN_FLAGS} --toolchains toolchains.xml -Dmaven.test.skip=true -DskipTests
INSTALL_COMMAND ""
)