diff --git a/bin/bootstrap_build.sh b/bin/bootstrap_build.sh index d69394e3d..3e9f56a1f 100755 --- a/bin/bootstrap_build.sh +++ b/bin/bootstrap_build.sh @@ -48,12 +48,12 @@ sudo apt-get --yes install openjdk-${JDK_VERSION}-jdk openjdk-${JDK_VERSION}-sou export JAVA_HOME=/usr/lib/jvm/java-${JDK_VERSION}-openjdk-amd64 # Download Maven since the packaged version is pretty old. -if [ ! -d /usr/local/apache-maven-3.5.4 ]; then +if [ ! -d /usr/local/apache-maven-3.9.2 ]; then sudo wget -nv \ - https://archive.apache.org/dist/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz - sha512sum -c - <<< '2a803f578f341e164f6753e410413d16ab60fabe31dc491d1fe35c984a5cce696bc71f57757d4538fe7738be04065a216f3ebad4ef7e0ce1bb4c51bc36d6be86 apache-maven-3.5.4-bin.tar.gz' - sudo tar -C /usr/local -xzf apache-maven-3.5.4-bin.tar.gz - sudo ln -s /usr/local/apache-maven-3.5.4/bin/mvn /usr/local/bin + https://archive.apache.org/dist/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz + sha512sum -c - <<< '900bdeeeae550d2d2b3920fe0e00e41b0069f32c019d566465015bdd1b3866395cbe016e22d95d25d51d3a5e614af2c83ec9b282d73309f644859bbad08b63db apache-maven-3.9.2-bin.tar.gz' + sudo tar -C /usr/local -xzf apache-maven-3.9.2-bin.tar.gz + sudo ln -s /usr/local/apache-maven-3.9.2/bin/mvn /usr/local/bin fi # Try to prepopulate the m2 directory to save time diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh index 15f14f32b..c619c4756 100755 --- a/bin/bootstrap_system.sh +++ b/bin/bootstrap_system.sh @@ -340,18 +340,17 @@ redhat sudo yum install -y ccache # Clean up yum caches redhat sudo yum clean all -# Download maven for all OSes, since the OS-packaged version can be -# pretty old. -if [ ! -d /usr/local/apache-maven-3.5.4 ]; then +# Download Maven since the packaged version is pretty old. +if [ ! -d /usr/local/apache-maven-3.9.2 ]; then sudo wget -nv \ - https://archive.apache.org/dist/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz - sudo sha512sum -c - <<< '2a803f578f341e164f6753e410413d16ab60fabe31dc491d1fe35c984a5cce696bc71f57757d4538fe7738be04065a216f3ebad4ef7e0ce1bb4c51bc36d6be86 apache-maven-3.5.4-bin.tar.gz' - sudo tar -C /usr/local -x --no-same-owner -zf apache-maven-3.5.4-bin.tar.gz - sudo ln -sf /usr/local/apache-maven-3.5.4/bin/mvn /usr/local/bin + https://archive.apache.org/dist/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz + sha512sum -c - <<< '900bdeeeae550d2d2b3920fe0e00e41b0069f32c019d566465015bdd1b3866395cbe016e22d95d25d51d3a5e614af2c83ec9b282d73309f644859bbad08b63db apache-maven-3.9.2-bin.tar.gz' + sudo tar -C /usr/local -xzf apache-maven-3.9.2-bin.tar.gz + sudo ln -s /usr/local/apache-maven-3.9.2/bin/mvn /usr/local/bin # reset permissions on redhat8 # TODO: figure out why this is necessary for redhat8 - MAVEN_DIRECTORY="/usr/local/apache-maven-3.5.4" + MAVEN_DIRECTORY="/usr/local/apache-maven-3.9.2" redhat8 indocker sudo chmod 0755 ${MAVEN_DIRECTORY} redhat8 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}/{bin,boot} redhat9 indocker sudo chmod 0755 ${MAVEN_DIRECTORY} diff --git a/bin/mvn-quiet.sh b/bin/mvn-quiet.sh index c7c557e78..15b3b8bfd 100755 --- a/bin/mvn-quiet.sh +++ b/bin/mvn-quiet.sh @@ -37,13 +37,13 @@ LOGGING_OPTIONS="-Dorg.slf4j.simpleLogger.showDateTime \ # Filter out "Checksum validation failed" messages, as they are mostly harmless and # make it harder to search for failed tests in the console output. Limit the filtering # to WARNING messages. -CHECKSUM_VALIDATION_FAILED_REGEX="[WARNING].*Checksum validation failed" +CHECKSUM_VALIDATION_FAILED_REGEX="[WARNING].*(Checksum validation failed|Could not validate integrity of download)" # Always use maven's batch mode (-B), as it produces output that is easier to parse. if ! mvn -B $IMPALA_MAVEN_OPTIONS $LOGGING_OPTIONS "$@" | \ tee -a "$LOG_FILE" | \ grep -E -e WARNING -e ERROR -e SUCCESS -e FAILURE -e Test -e "Found Banned" | \ - grep -v -i "${CHECKSUM_VALIDATION_FAILED_REGEX}"; then + grep -E -v -i "${CHECKSUM_VALIDATION_FAILED_REGEX}"; then echo "mvn $IMPALA_MAVEN_OPTIONS $@ exited with code $?" exit 1 fi diff --git a/bin/rat_exclude_files.txt b/bin/rat_exclude_files.txt index 8b4ca15f5..69bc099cb 100644 --- a/bin/rat_exclude_files.txt +++ b/bin/rat_exclude_files.txt @@ -135,6 +135,7 @@ fe/src/test/resources/adusers.ldif fe/src/test/resources/hbase-jaas-client.conf.template fe/src/test/resources/hbase-jaas-server.conf.template fe/src/test/resources/users.ldif +java/.mvn/maven.config java/toolchains.xml.tmpl testdata/AllTypesError/*.txt testdata/AllTypesErrorNoNulls/*.txt diff --git a/java/.mvn/maven.config b/java/.mvn/maven.config new file mode 100644 index 000000000..1c5c609f6 --- /dev/null +++ b/java/.mvn/maven.config @@ -0,0 +1,5 @@ +--show-version +--batch-mode +-Daether.dependencyCollector.impl=bf +-Daether.connector.basic.threads=10 +-Daether.dependencyCollector.bf.threads=10 diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt index aa3c6de12..c39cee3b6 100644 --- a/java/CMakeLists.txt +++ b/java/CMakeLists.txt @@ -19,6 +19,12 @@ add_custom_target(validate_java_pom_versions ALL COMMAND $ENV{IMPALA_HOME}/bin/validate-java-pom-versions.sh ) +SET(JAMM_MAVEN_FLAGS --show-version --batch-mode) +# Discover and download dependencies in parallel, using 10 threads +SET(JAMM_MAVEN_FLAGS ${JAMM_MAVEN_FLAGS} -Daether.dependencyCollector.impl=bf) +SET(JAMM_MAVEN_FLAGS ${JAMM_MAVEN_FLAGS} -Daether.connector.basic.threads=10) +SET(JAMM_MAVEN_FLAGS ${JAMM_MAVEN_FLAGS} -Daether.dependencyCollector.bf.threads=10) + include(ExternalProject) ExternalProject_Add(jamm GIT_REPOSITORY https://github.com/jbellis/jamm @@ -27,7 +33,7 @@ ExternalProject_Add(jamm PATCH_COMMAND sed -i.bak s/0.4.0-SNAPSHOT/0.4.0-IMPALA/ pom.xml CONFIGURE_COMMAND sed s:JAVA_HOME:$ENV{JAVA_HOME}:g ${CMAKE_CURRENT_SOURCE_DIR}/toolchains.xml.tmpl > toolchains.xml BUILD_IN_SOURCE true - BUILD_COMMAND mvn --batch-mode --toolchains toolchains.xml install -Dmaven.test.skip=true -DskipTests + BUILD_COMMAND mvn install ${JAMM_MAVEN_FLAGS} --toolchains toolchains.xml -Dmaven.test.skip=true -DskipTests INSTALL_COMMAND "" )