You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2020/10/15 19:30:43 UTC
[impala] 02/02: IMPALA-10198 (part 2): Add support for mvn
versions:set
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 97792c4bad526c84fbad01c98ef6b09b6c7cbcdf
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Tue Oct 6 17:06:40 2020 -0700
IMPALA-10198 (part 2): Add support for mvn versions:set
This adds support for setting the version of Java
artifacts through "mvn versions:set". It changes
the modules to inherit the version from the parent
pom.
Previously, we used a mix of 0.1-SNAPSHOT and
1.0-SNAPSHOT. This now uses 4.0.0-SNAPSHOT across the
board. With each release, we can use "mvn versions:set"
to update the versions. The only exception is the
Hive UDF code that we build for testing. This remains
at version 1.0 to avoid test changes.
Testing:
- Ran core job
- Added build-all-flag-combinations.sh case that
does "mvn versions:set" and runs a build
Change-Id: I661b32e1e445169bac2ffe4f9474f14090031743
Reviewed-on: http://gerrit.cloudera.org:8080/16559
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/jenkins/build-all-flag-combinations.sh | 42 +++++++++++++++++++++++++++++-
docker/setup_build_context.py | 10 +++++--
fe/pom.xml | 4 +--
java/TableFlattener/pom.xml | 4 +--
java/datagenerator/pom.xml | 8 +++---
java/executor-deps/pom.xml | 2 +-
java/ext-data-source/api/pom.xml | 4 +--
java/ext-data-source/pom.xml | 4 +--
java/ext-data-source/sample/pom.xml | 4 +--
java/ext-data-source/test/pom.xml | 8 +-----
java/pom.xml | 6 ++---
java/query-event-hook-api/pom.xml | 4 +--
java/shaded-deps/hive-exec/pom.xml | 3 +--
java/shaded-deps/s3a-aws-sdk/pom.xml | 3 +--
java/test-hive-udfs/pom.xml | 3 +--
java/yarn-extras/pom.xml | 31 +++++++++++++++++++---
16 files changed, 93 insertions(+), 47 deletions(-)
diff --git a/bin/jenkins/build-all-flag-combinations.sh b/bin/jenkins/build-all-flag-combinations.sh
index 9209e48..87ad92a 100755
--- a/bin/jenkins/build-all-flag-combinations.sh
+++ b/bin/jenkins/build-all-flag-combinations.sh
@@ -88,8 +88,48 @@ for CONFIG in "${CONFIGS[@]}"; do
cat logs/mvn/mvn.log >> "${TMP_DIR}/mvn_accumulated.log"
done
+# It is useful to be able to use "mvn versions:set" to modify the jar
+# versions produced by Impala. This tests that Impala continues to build
+# after running that command.
+NEW_MAVEN_VERSION=4.0.0-$(date +%Y%m%d)
+DESCRIPTION="Options: mvn versions:set -DnewVersion=${NEW_MAVEN_VERSION}"
+
+if [[ $# == 1 && $1 == "--dryrun" ]]; then
+ echo $DESCRIPTION
+else
+ # Note: this command modifies files in the git checkout
+ pushd java
+ mvn versions:set -DnewVersion=${NEW_MAVEN_VERSION}
+ popd
+
+ if ! ./bin/clean.sh; then
+ echo "Clean failed"
+ exit 1
+ fi
+
+ echo "Building with OPTIONS: $DESCRIPTION"
+ if ! time -p ./buildall.sh -skiptests -noclean ; then
+ echo "Build failed: $DESCRIPTION"
+ FAILED="${FAILED}:${DESCRIPTION}"
+ fi
+
+ # Reset the files changed by mvn versions:set
+ git reset --hard HEAD
+
+ ccache -s
+ bin/jenkins/get_maven_statistics.sh logs/mvn/mvn.log
+
+ # Keep each maven log from each round of the build
+ cp logs/mvn/mvn.log "${TMP_DIR}/mvn.$(date +%s.%N).log"
+ # Append the maven log to the accumulated maven log
+ cat logs/mvn/mvn.log >> "${TMP_DIR}/mvn_accumulated.log"
+fi
+
# Restore the maven logs (these don't interfere with existing mvn.log)
-cp ${TMP_DIR}/mvn* logs/mvn
+# These files may not exist if this is doing a dryrun.
+if ls ${TMP_DIR}/mvn* > /dev/null 2>&1 ; then
+ cp ${TMP_DIR}/mvn* logs/mvn
+fi
if [[ "$FAILED" != "" ]]
then
diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py
index f41a757..c1164e3 100755
--- a/docker/setup_build_context.py
+++ b/docker/setup_build_context.py
@@ -131,9 +131,15 @@ for jar in dep_classpath.split(":"):
symlink_file_into_dir(jar, LIB_DIR)
# Impala Coordinator jars.
-for jar in glob.glob(
- os.path.join(IMPALA_HOME, "fe/target/impala-frontend-*-SNAPSHOT.jar")):
+num_frontend_jars = 0
+for jar in glob.glob(os.path.join(IMPALA_HOME, "fe/target/impala-frontend-*.jar")):
+ # Ignore the tests jar
+ if jar.find("-tests") != -1:
+ continue
symlink_file_into_dir(jar, LIB_DIR)
+ num_frontend_jars += 1
+# There must be exactly one impala-frontend jar.
+assert num_frontend_jars == 1
# Impala Executor dependencies.
dep_classpath = file(os.path.join(IMPALA_HOME,
diff --git a/fe/pom.xml b/fe/pom.xml
index d84e16d..9bb7c7b 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -23,13 +23,11 @@ under the License.
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../java/pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>impala-frontend</artifactId>
- <version>0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Impala Query Engine Frontend</name>
diff --git a/java/TableFlattener/pom.xml b/java/TableFlattener/pom.xml
index 40a404c..53374b2 100644
--- a/java/TableFlattener/pom.xml
+++ b/java/TableFlattener/pom.xml
@@ -22,13 +22,11 @@
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>nested-table-flattener</artifactId>
<name>Impala Nested Table Flattener</name>
- <version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<build>
diff --git a/java/datagenerator/pom.xml b/java/datagenerator/pom.xml
index 03195da..b14a6e4 100644
--- a/java/datagenerator/pom.xml
+++ b/java/datagenerator/pom.xml
@@ -23,16 +23,14 @@ under the License.
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
- <artifactId>impala-testdata</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <artifactId>impala-data-generator</artifactId>
<packaging>jar</packaging>
- <name>Build some test data</name>
+ <name>Test Data Generator</name>
<dependencies>
<!-- Force json-smart dependency.
diff --git a/java/executor-deps/pom.xml b/java/executor-deps/pom.xml
index f9e8897..4817f0a 100644
--- a/java/executor-deps/pom.xml
+++ b/java/executor-deps/pom.xml
@@ -34,7 +34,7 @@ under the License.
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.impala</groupId>
diff --git a/java/ext-data-source/api/pom.xml b/java/ext-data-source/api/pom.xml
index 82e6a88..6a6bd20 100644
--- a/java/ext-data-source/api/pom.xml
+++ b/java/ext-data-source/api/pom.xml
@@ -23,11 +23,9 @@
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-data-source</artifactId>
- <version>1.0-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
- <groupId>org.apache.impala</groupId>
<artifactId>impala-data-source-api</artifactId>
- <version>1.0-SNAPSHOT</version>
<name>Apache Impala External Data Source API</name>
<description>External Data Source API interface definition</description>
<packaging>jar</packaging>
diff --git a/java/ext-data-source/pom.xml b/java/ext-data-source/pom.xml
index 8b4081d..6d02159 100644
--- a/java/ext-data-source/pom.xml
+++ b/java/ext-data-source/pom.xml
@@ -22,13 +22,11 @@
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>impala-data-source</artifactId>
<name>Apache Impala External Data Source</name>
- <version>1.0-SNAPSHOT</version>
<packaging>pom</packaging>
<modules>
diff --git a/java/ext-data-source/sample/pom.xml b/java/ext-data-source/sample/pom.xml
index c0df755..29a88f3 100644
--- a/java/ext-data-source/sample/pom.xml
+++ b/java/ext-data-source/sample/pom.xml
@@ -23,11 +23,9 @@
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-data-source</artifactId>
- <version>1.0-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
- <groupId>org.apache.impala</groupId>
<artifactId>impala-data-source-sample</artifactId>
- <version>1.0-SNAPSHOT</version>
<name>Apache Impala External Data Source Sample</name>
<description>External Data Source Sample</description>
<packaging>jar</packaging>
diff --git a/java/ext-data-source/test/pom.xml b/java/ext-data-source/test/pom.xml
index c96c86a..b2a175b 100644
--- a/java/ext-data-source/test/pom.xml
+++ b/java/ext-data-source/test/pom.xml
@@ -23,20 +23,14 @@
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-data-source</artifactId>
- <version>1.0-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
- <groupId>org.apache.impala</groupId>
<artifactId>impala-data-source-test</artifactId>
- <version>0.1-SNAPSHOT</version>
<name>Apache Impala External Data Source Test Library</name>
<description>Test External Data Source</description>
<packaging>jar</packaging>
<url>.</url>
- <properties>
- <impala.extdatasrc.api.version>1.0-SNAPSHOT</impala.extdatasrc.api.version>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.impala</groupId>
diff --git a/java/pom.xml b/java/pom.xml
index 5c8649e..05c9a7b 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -21,7 +21,7 @@ under the License.
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Impala Parent POM</name>
@@ -45,8 +45,8 @@ under the License.
<kite.version>${env.IMPALA_KITE_VERSION}</kite.version>
<knox.version>${env.IMPALA_KNOX_VERSION}</knox.version>
<thrift.version>0.9.3-1</thrift.version>
- <impala.extdatasrc.api.version>1.0-SNAPSHOT</impala.extdatasrc.api.version>
- <impala.query.event.hook.api.version>1.0-SNAPSHOT</impala.query.event.hook.api.version>
+ <impala.extdatasrc.api.version>${project.version}</impala.extdatasrc.api.version>
+ <impala.query.event.hook.api.version>${project.version}</impala.query.event.hook.api.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<kudu.version>${env.IMPALA_KUDU_JAVA_VERSION}</kudu.version>
<commons-io.version>2.6</commons-io.version>
diff --git a/java/query-event-hook-api/pom.xml b/java/query-event-hook-api/pom.xml
index 4e9cb99..ddd64ad 100644
--- a/java/query-event-hook-api/pom.xml
+++ b/java/query-event-hook-api/pom.xml
@@ -22,13 +22,11 @@
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>query-event-hook-api</artifactId>
<name>Apache Impala Query Event Hook API</name>
- <version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<url>.</url>
diff --git a/java/shaded-deps/hive-exec/pom.xml b/java/shaded-deps/hive-exec/pom.xml
index ea28a6b..ce1f876 100644
--- a/java/shaded-deps/hive-exec/pom.xml
+++ b/java/shaded-deps/hive-exec/pom.xml
@@ -27,11 +27,10 @@ the same dependencies
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>impala-minimal-hive-exec</artifactId>
<packaging>jar</packaging>
diff --git a/java/shaded-deps/s3a-aws-sdk/pom.xml b/java/shaded-deps/s3a-aws-sdk/pom.xml
index aebf3d8..665ebaf 100644
--- a/java/shaded-deps/s3a-aws-sdk/pom.xml
+++ b/java/shaded-deps/s3a-aws-sdk/pom.xml
@@ -25,11 +25,10 @@ though some of them might not be necessary. The exclusions are sorted alphabetic
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>impala-minimal-s3a-aws-sdk</artifactId>
<packaging>jar</packaging>
diff --git a/java/test-hive-udfs/pom.xml b/java/test-hive-udfs/pom.xml
index dc3b431..eb63bb6 100644
--- a/java/test-hive-udfs/pom.xml
+++ b/java/test-hive-udfs/pom.xml
@@ -22,11 +22,10 @@ under the License.
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>test-hive-udfs</artifactId>
<version>1.0</version>
<packaging>jar</packaging>
diff --git a/java/yarn-extras/pom.xml b/java/yarn-extras/pom.xml
index ce2d7f8..68ba84d 100644
--- a/java/yarn-extras/pom.xml
+++ b/java/yarn-extras/pom.xml
@@ -22,12 +22,10 @@
<parent>
<groupId>org.apache.impala</groupId>
<artifactId>impala-parent</artifactId>
- <version>0.1-SNAPSHOT</version>
+ <version>4.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.impala</groupId>
<artifactId>yarn-extras</artifactId>
- <version>0.1-SNAPSHOT</version>
<name>YARN Extras</name>
<description>Extra Hadoop classes from YARN needed by Impala</description>
<packaging>jar</packaging>
@@ -45,7 +43,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>28.1-jre</version>
+ <version>${guava.version}</version>
</dependency>
<dependency>
@@ -58,6 +56,23 @@
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- IMPALA-9468: Avoid pulling in netty for security reasons -->
+ <groupId>io.netty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ </exclusion>
</exclusions>
</dependency>
@@ -77,6 +92,14 @@
<groupId>net.minidev</groupId>
<artifactId>json-smart</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-servlet</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>