You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2023/07/24 19:54:05 UTC
[impala] branch master updated: IMPALA-12212: Bump Maven to 3.9.2, pull dependencies in parallel
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new ee069687f IMPALA-12212: Bump Maven to 3.9.2, pull dependencies in parallel
ee069687f is described below
commit ee069687fcaa06c29404e2220ff577767d905a98
Author: Laszlo Gaal <la...@cloudera.com>
AuthorDate: Sat Feb 22 18:12:12 2020 +0100
IMPALA-12212: Bump Maven to 3.9.2, pull dependencies in parallel
Maven 3.9.x offers a new dependency resolver, HttpClient, which allows
downloading project dependencies in parallel.
This patch bumps the Maven version installed by bootstrap_system.sh to
v3.9.2, and adds the flags enabling the new resolver to download
dependencies (including POM files) in parallel. Parallelism is set to
10 threads.
The flags are added to a project-specific Maven setting file in the
newly created java/.mvn directory. The settings file is added to the
RAT exclusion list in bin/rat_exclude_files.txt.
The --show-version flag is added for debugging purposes.
The same flags are added to the JAMM subproject as well.
The new resolver in Maven 3.9 has also changed the warning message
emitted for missing component checksums, so the new warning string
is added to the filter in bin/mvn-quiet.sh
Unfortunately Maven 3.9 has also changed the way it responds to missing
checksum files: the resolver now emits a stack trace when checksums
cannot be determined, and missing checksums are not explicitly ignored.
Detailed documentation for the new Maven resolver in Maven 3.9.0+ is
located at:
https://maven.apache.org/guides/mini/guide-resolver-transport.html
resolver configuration reference:
https://maven.apache.org/resolver/configuration.html
Tests:
- verified in a core-mode test run with Maven 3.9.2 installed
- verified in a local build using an earlier version of Maven
to verify that the new default setting does not cause regressions
with the old dependency resolver.
Change-Id: I75d05215effc724f5bd471646fb352f37443e185
Reviewed-on: http://gerrit.cloudera.org:8080/20142
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Michael Smith <mi...@cloudera.com>
---
bin/bootstrap_build.sh | 10 +++++-----
bin/bootstrap_system.sh | 15 +++++++--------
bin/mvn-quiet.sh | 4 ++--
bin/rat_exclude_files.txt | 1 +
java/.mvn/maven.config | 5 +++++
java/CMakeLists.txt | 8 +++++++-
6 files changed, 27 insertions(+), 16 deletions(-)
diff --git a/bin/bootstrap_build.sh b/bin/bootstrap_build.sh
index d69394e3d..3e9f56a1f 100755
--- a/bin/bootstrap_build.sh
+++ b/bin/bootstrap_build.sh
@@ -48,12 +48,12 @@ sudo apt-get --yes install openjdk-${JDK_VERSION}-jdk openjdk-${JDK_VERSION}-sou
export JAVA_HOME=/usr/lib/jvm/java-${JDK_VERSION}-openjdk-amd64
# Download Maven since the packaged version is pretty old.
-if [ ! -d /usr/local/apache-maven-3.5.4 ]; then
+if [ ! -d /usr/local/apache-maven-3.9.2 ]; then
sudo wget -nv \
- https://archive.apache.org/dist/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz
- sha512sum -c - <<< '2a803f578f341e164f6753e410413d16ab60fabe31dc491d1fe35c984a5cce696bc71f57757d4538fe7738be04065a216f3ebad4ef7e0ce1bb4c51bc36d6be86 apache-maven-3.5.4-bin.tar.gz'
- sudo tar -C /usr/local -xzf apache-maven-3.5.4-bin.tar.gz
- sudo ln -s /usr/local/apache-maven-3.5.4/bin/mvn /usr/local/bin
+ https://archive.apache.org/dist/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz
+ sha512sum -c - <<< '900bdeeeae550d2d2b3920fe0e00e41b0069f32c019d566465015bdd1b3866395cbe016e22d95d25d51d3a5e614af2c83ec9b282d73309f644859bbad08b63db apache-maven-3.9.2-bin.tar.gz'
+ sudo tar -C /usr/local -xzf apache-maven-3.9.2-bin.tar.gz
+ sudo ln -s /usr/local/apache-maven-3.9.2/bin/mvn /usr/local/bin
fi
# Try to prepopulate the m2 directory to save time
diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh
index 15f14f32b..c619c4756 100755
--- a/bin/bootstrap_system.sh
+++ b/bin/bootstrap_system.sh
@@ -340,18 +340,17 @@ redhat sudo yum install -y ccache
# Clean up yum caches
redhat sudo yum clean all
-# Download maven for all OSes, since the OS-packaged version can be
-# pretty old.
-if [ ! -d /usr/local/apache-maven-3.5.4 ]; then
+# Download Maven since the packaged version is pretty old.
+if [ ! -d /usr/local/apache-maven-3.9.2 ]; then
sudo wget -nv \
- https://archive.apache.org/dist/maven/maven-3/3.5.4/binaries/apache-maven-3.5.4-bin.tar.gz
- sudo sha512sum -c - <<< '2a803f578f341e164f6753e410413d16ab60fabe31dc491d1fe35c984a5cce696bc71f57757d4538fe7738be04065a216f3ebad4ef7e0ce1bb4c51bc36d6be86 apache-maven-3.5.4-bin.tar.gz'
- sudo tar -C /usr/local -x --no-same-owner -zf apache-maven-3.5.4-bin.tar.gz
- sudo ln -sf /usr/local/apache-maven-3.5.4/bin/mvn /usr/local/bin
+ https://archive.apache.org/dist/maven/maven-3/3.9.2/binaries/apache-maven-3.9.2-bin.tar.gz
+ sha512sum -c - <<< '900bdeeeae550d2d2b3920fe0e00e41b0069f32c019d566465015bdd1b3866395cbe016e22d95d25d51d3a5e614af2c83ec9b282d73309f644859bbad08b63db apache-maven-3.9.2-bin.tar.gz'
+ sudo tar -C /usr/local -xzf apache-maven-3.9.2-bin.tar.gz
+ sudo ln -s /usr/local/apache-maven-3.9.2/bin/mvn /usr/local/bin
# reset permissions on redhat8
# TODO: figure out why this is necessary for redhat8
- MAVEN_DIRECTORY="/usr/local/apache-maven-3.5.4"
+ MAVEN_DIRECTORY="/usr/local/apache-maven-3.9.2"
redhat8 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}
redhat8 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}/{bin,boot}
redhat9 indocker sudo chmod 0755 ${MAVEN_DIRECTORY}
diff --git a/bin/mvn-quiet.sh b/bin/mvn-quiet.sh
index c7c557e78..15b3b8bfd 100755
--- a/bin/mvn-quiet.sh
+++ b/bin/mvn-quiet.sh
@@ -37,13 +37,13 @@ LOGGING_OPTIONS="-Dorg.slf4j.simpleLogger.showDateTime \
# Filter out "Checksum validation failed" messages, as they are mostly harmless and
# make it harder to search for failed tests in the console output. Limit the filtering
# to WARNING messages.
-CHECKSUM_VALIDATION_FAILED_REGEX="[WARNING].*Checksum validation failed"
+CHECKSUM_VALIDATION_FAILED_REGEX="[WARNING].*(Checksum validation failed|Could not validate integrity of download)"
# Always use maven's batch mode (-B), as it produces output that is easier to parse.
if ! mvn -B $IMPALA_MAVEN_OPTIONS $LOGGING_OPTIONS "$@" | \
tee -a "$LOG_FILE" | \
grep -E -e WARNING -e ERROR -e SUCCESS -e FAILURE -e Test -e "Found Banned" | \
- grep -v -i "${CHECKSUM_VALIDATION_FAILED_REGEX}"; then
+ grep -E -v -i "${CHECKSUM_VALIDATION_FAILED_REGEX}"; then
echo "mvn $IMPALA_MAVEN_OPTIONS $@ exited with code $?"
exit 1
fi
diff --git a/bin/rat_exclude_files.txt b/bin/rat_exclude_files.txt
index 8b4ca15f5..69bc099cb 100644
--- a/bin/rat_exclude_files.txt
+++ b/bin/rat_exclude_files.txt
@@ -135,6 +135,7 @@ fe/src/test/resources/adusers.ldif
fe/src/test/resources/hbase-jaas-client.conf.template
fe/src/test/resources/hbase-jaas-server.conf.template
fe/src/test/resources/users.ldif
+java/.mvn/maven.config
java/toolchains.xml.tmpl
testdata/AllTypesError/*.txt
testdata/AllTypesErrorNoNulls/*.txt
diff --git a/java/.mvn/maven.config b/java/.mvn/maven.config
new file mode 100644
index 000000000..1c5c609f6
--- /dev/null
+++ b/java/.mvn/maven.config
@@ -0,0 +1,5 @@
+--show-version
+--batch-mode
+-Daether.dependencyCollector.impl=bf
+-Daether.connector.basic.threads=10
+-Daether.dependencyCollector.bf.threads=10
diff --git a/java/CMakeLists.txt b/java/CMakeLists.txt
index aa3c6de12..c39cee3b6 100644
--- a/java/CMakeLists.txt
+++ b/java/CMakeLists.txt
@@ -19,6 +19,12 @@ add_custom_target(validate_java_pom_versions ALL
COMMAND $ENV{IMPALA_HOME}/bin/validate-java-pom-versions.sh
)
+SET(JAMM_MAVEN_FLAGS --show-version --batch-mode)
+# Discover and download dependencies in parallel, using 10 threads
+SET(JAMM_MAVEN_FLAGS ${JAMM_MAVEN_FLAGS} -Daether.dependencyCollector.impl=bf)
+SET(JAMM_MAVEN_FLAGS ${JAMM_MAVEN_FLAGS} -Daether.connector.basic.threads=10)
+SET(JAMM_MAVEN_FLAGS ${JAMM_MAVEN_FLAGS} -Daether.dependencyCollector.bf.threads=10)
+
include(ExternalProject)
ExternalProject_Add(jamm
GIT_REPOSITORY https://github.com/jbellis/jamm
@@ -27,7 +33,7 @@ ExternalProject_Add(jamm
PATCH_COMMAND sed -i.bak s/0.4.0-SNAPSHOT/0.4.0-IMPALA/ pom.xml
CONFIGURE_COMMAND sed s:JAVA_HOME:$ENV{JAVA_HOME}:g ${CMAKE_CURRENT_SOURCE_DIR}/toolchains.xml.tmpl > toolchains.xml
BUILD_IN_SOURCE true
- BUILD_COMMAND mvn --batch-mode --toolchains toolchains.xml install -Dmaven.test.skip=true -DskipTests
+ BUILD_COMMAND mvn install ${JAMM_MAVEN_FLAGS} --toolchains toolchains.xml -Dmaven.test.skip=true -DskipTests
INSTALL_COMMAND ""
)