You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2017/07/12 19:57:38 UTC
flink git commit: [FLINK-7047] [travis] Reorganize build profiles
Repository: flink
Updated Branches:
refs/heads/master 8987df556 -> 1da93a4c7
[FLINK-7047] [travis] Reorganize build profiles
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/1da93a4c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/1da93a4c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/1da93a4c
Branch: refs/heads/master
Commit: 1da93a4c73d48b6635e44214383d27cd3d31fc14
Parents: 8987df5
Author: zentol <ch...@apache.org>
Authored: Tue Jul 11 11:20:34 2017 +0200
Committer: zentol <ch...@apache.org>
Committed: Wed Jul 12 20:19:35 2017 +0200
----------------------------------------------------------------------
.travis.yml | 48 +++++++++-----
pom.xml | 35 ----------
tools/travis_mvn_watchdog.sh | 136 ++++++++++++++++++++++++++++++++++++--
3 files changed, 161 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/1da93a4c/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 5bfb549..0e4624f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,34 +15,46 @@ language: java
#See https://issues.apache.org/jira/browse/FLINK-1072
matrix:
include:
- # Always run test groups A and B together
- jdk: "oraclejdk8"
- env: PROFILE="-Dhadoop.version=2.8.0 -Pflink-fast-tests-a,include-kinesis,jdk8"
+ env:
+ - TEST="core"
+ - PROFILE="-Dhadoop.version=2.8.0 -Pjdk8"
- jdk: "oraclejdk8"
- env: PROFILE="-Dhadoop.version=2.8.0 -Pflink-fast-tests-b,include-kinesis,jdk8"
+ env:
+ - TEST="libraries"
+ - PROFILE="-Dhadoop.version=2.8.0 -Pjdk8"
- jdk: "oraclejdk8"
- env: PROFILE="-Dhadoop.version=2.8.0 -Pflink-fast-tests-c,include-kinesis,jdk8"
-
- - jdk: "oraclejdk8"
- env: PROFILE="-Dhadoop.version=2.7.3 -Dscala-2.10 -Pflink-fast-tests-a,include-kinesis"
+ env:
+ - TEST="connectors"
+ - PROFILE="-Dhadoop.version=2.8.0 -Pjdk8,include-kinesis"
- jdk: "oraclejdk8"
- env: PROFILE="-Dhadoop.version=2.7.3 -Dscala-2.10 -Pflink-fast-tests-b,include-kinesis"
+ env:
+ - TEST="tests"
+ - PROFILE="-Dhadoop.version=2.8.0 -Pjdk8"
- jdk: "oraclejdk8"
- env: PROFILE="-Dhadoop.version=2.7.3 -Dscala-2.10 -Pflink-fast-tests-c,include-kinesis"
-
+ env:
+ - TEST="misc"
+ - PROFILE="-Dhadoop.version=2.8.0 -Pjdk8"
- jdk: "openjdk7"
- env: PROFILE="-Dhadoop.version=2.6.5 -Pflink-fast-tests-a,include-kinesis"
+ env:
+ - TEST="core"
+ - PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10"
- jdk: "openjdk7"
- env: PROFILE="-Dhadoop.version=2.6.5 -Pflink-fast-tests-b,include-kinesis"
- - jdk: "openjdk7"
- env: PROFILE="-Dhadoop.version=2.6.5 -Pflink-fast-tests-c,include-kinesis"
-
+ env:
+ - TEST="libraries"
+ - PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10"
- jdk: "openjdk7"
- env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10 -Pflink-fast-tests-a,include-kinesis"
+ env:
+ - TEST="connectors"
+ - PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10 -Pinclude-kinesis"
- jdk: "openjdk7"
- env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10 -Pflink-fast-tests-b,include-kinesis"
+ env:
+ - TEST="tests"
+ - PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10"
- jdk: "openjdk7"
- env: PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10 -Pflink-fast-tests-c,include-kinesis"
+ env:
+ - TEST="misc"
+ - PROFILE="-Dhadoop.version=2.4.1 -Dscala-2.10"
git:
depth: 100
http://git-wip-us.apache.org/repos/asf/flink/blob/1da93a4c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3de92c7..f136594 100644
--- a/pom.xml
+++ b/pom.xml
@@ -83,8 +83,6 @@ under the License.
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
- <!-- Internal property to reduce build times on TravisCi -->
- <flink-fast-tests-pattern>never-match-me</flink-fast-tests-pattern>
<hadoop.version>2.4.1</hadoop.version>
<!-- Need to use a user property here because the surefire
forkCount is not exposed as a property. With this we can set
@@ -864,32 +862,6 @@ under the License.
</plugins>
</build>
</profile>
- <!--
- Test profile A excludes all tests starting with (A-M).
- profile B excludes tests (N-Z).
- This allows us to split up the test execution into separate profiles.
- -->
- <profile>
- <id>flink-fast-tests-a</id>
- <properties>
- <!-- Allow A-I by forbidding J-Z -->
- <flink-fast-tests-pattern>%regex[.*/[J-Z].*]</flink-fast-tests-pattern>
- </properties>
- </profile>
- <profile>
- <id>flink-fast-tests-b</id>
- <properties>
- <!-- Allow J-R, forbid A-I and S - Z -->
- <flink-fast-tests-pattern>%regex[.*/[A-IS-Z].*]</flink-fast-tests-pattern>
- </properties>
- </profile>
- <profile>
- <id>flink-fast-tests-c</id>
- <properties>
- <!-- Allow S-Z, forbid A-R -->
- <flink-fast-tests-pattern>%regex[.*/[A-R].*]</flink-fast-tests-pattern>
- </properties>
- </profile>
</profiles>
<build>
@@ -1099,10 +1071,6 @@ under the License.
<includes>
<include>**/*Test.*</include>
</includes>
- <excludes>
- <exclude>**/*ITCase.*</exclude>
- <exclude>${flink-fast-tests-pattern}</exclude>
- </excludes>
</configuration>
</execution>
<!--execute all the integration tests-->
@@ -1116,9 +1084,6 @@ under the License.
<includes>
<include>**/*ITCase.*</include>
</includes>
- <excludes>
- <exclude>${flink-fast-tests-pattern}</exclude>
- </excludes>
<reuseForks>false</reuseForks>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/flink/blob/1da93a4c/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index b22bfe0..606492f 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -43,11 +43,107 @@ SLEEP_TIME=20
LOG4J_PROPERTIES=${HERE}/log4j-travis.properties
+MODULES_CORE="\
+flink-test-utils-parent/flink-test-utils,\
+flink-contrib/flink-statebackend-rocksdb,\
+flink-clients,\
+flink-core,\
+flink-java,\
+flink-optimizer,\
+flink-runtime,\
+flink-runtime-web,\
+flink-scala,\
+flink-scala-shell,\
+flink-streaming-java,\
+flink-streaming-scala"
+
+MODULES_LIBRARIES="\
+flink-contrib/flink-storm,\
+flink-contrib/flink-storm-examples,\
+flink-libraries/flink-cep,\
+flink-libraries/flink-cep-scala,\
+flink-libraries/flink-gelly,\
+flink-libraries/flink-gelly-scala,\
+flink-libraries/flink-gelly-examples,\
+flink-libraries/flink-ml,\
+flink-libraries/flink-python,\
+flink-libraries/flink-table"
+
+MODULES_CONNECTORS="\
+flink-contrib/flink-connector-wikiedits,\
+flink-connectors/flink-avro,\
+flink-connectors/flink-hbase,\
+flink-connectors/flink-hcatalog,\
+flink-connectors/flink-hadoop-compatibility,\
+flink-connectors/flink-jdbc,\
+flink-connectors/flink-connector-cassandra,\
+flink-connectors/flink-connector-elasticsearch,\
+flink-connectors/flink-connector-elasticsearch2,\
+flink-connectors/flink-connector-elasticsearch-base,\
+flink-connectors/flink-connector-filesystem,\
+flink-connectors/flink-connector-kafka-0.8,\
+flink-connectors/flink-connector-kafka-0.9,\
+flink-connectors/flink-connector-kafka-0.10,\
+flink-connectors/flink-connector-kafka-base,\
+flink-connectors/flink-connector-nifi,\
+flink-connectors/flink-connector-rabbitmq,\
+flink-connectors/flink-connector-twitter"
+
+MODULES_TESTS="\
+flink-tests"
+
+if [[ $PROFILE == *"jdk8"* ]]; then
+ case $TEST in
+ (connectors)
+ MODULES_CONNECTORS="$MODULES_CONNECTORS,flink-connectors/flink-connector-elasticsearch5"
+ ;;
+ esac
+fi
+
+if [[ $PROFILE == *"include-kinesis"* ]]; then
+ case $TEST in
+ (connectors)
+ MODULES_CONNECTORS="$MODULES_CONNECTORS,flink-connectors/flink-connector-kinesis"
+ ;;
+ esac
+fi
+
+MVN_COMPILE_MODULES=""
+MVN_TEST_MODULES=""
+case $TEST in
+ (core)
+ MVN_COMPILE_MODULES="-pl $MODULES_CORE -am"
+ MVN_TEST_MODULES="-pl $MODULES_CORE"
+ ;;
+ (libraries)
+ MVN_COMPILE_MODULES="-pl $MODULES_LIBRARIES -am"
+ MVN_TEST_MODULES="-pl $MODULES_LIBRARIES"
+ ;;
+ (connectors)
+ MVN_COMPILE_MODULES="-pl $MODULES_CONNECTORS -am"
+ MVN_TEST_MODULES="-pl $MODULES_CONNECTORS"
+ ;;
+ (tests)
+ MVN_COMPILE_MODULES="-pl $MODULES_TESTS -am"
+ MVN_TEST_MODULES="-pl $MODULES_TESTS"
+ ;;
+ (misc)
+ NEGATED_CORE=\!${MODULES_CORE//,/,\!}
+ NEGATED_LIBRARIES=\!${MODULES_LIBRARIES//,/,\!}
+ NEGATED_CONNECTORS=\!${MODULES_CONNECTORS//,/,\!}
+ NEGATED_TESTS=\!${MODULES_TESTS//,/,\!}
+ # compile everything since dist needs it anyway
+ MVN_COMPILE_MODULES=""
+ MVN_TEST_MODULES="-pl $NEGATED_CORE,$NEGATED_LIBRARIES,$NEGATED_CONNECTORS,$NEGATED_TESTS"
+ ;;
+esac
+
# Maven command to run. We set the forkCount manually, because otherwise Maven sees too many cores
# on the Travis VMs. Set forkCountTestPackage to 1 for container-based environment (4 GiB memory)
# and 2 for sudo-enabled environment (7.5 GiB memory).
MVN_LOGGING_OPTIONS="-Dlog.dir=${ARTIFACTS_DIR} -Dlog4j.configuration=file://$LOG4J_PROPERTIES -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
-MVN="mvn -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.javadoc.skip=true -B $PROFILE $MVN_LOGGING_OPTIONS clean install"
+MVN_COMPILE="mvn -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -DskipTests -Dmaven.javadoc.skip=true -B $PROFILE $MVN_LOGGING_OPTIONS $MVN_COMPILE_MODULES clean install"
+MVN_TEST="mvn -Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dmaven.javadoc.skip=true -B $PROFILE $MVN_LOGGING_OPTIONS $MVN_TEST_MODULES verify"
MVN_PID="${ARTIFACTS_DIR}/watchdog.mvn.pid"
MVN_EXIT="${ARTIFACTS_DIR}/watchdog.mvn.exit"
@@ -226,12 +322,35 @@ echo -en "travis_fold:end:disk_info${FOLD_ESCAPE}"
# Make sure to be in project root
cd $HERE/../
-echo "RUNNING '${MVN}'."
+# Compile modules
+
+echo "RUNNING '${MVN_COMPILE}'."
+
+# Run $MVN_COMPILE and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID to
+# allow the watchdog to kill $MVN if it is not producing any output anymore. $MVN_EXIT contains
+# the exit code. This is important for Travis' build life-cycle (success/failure).
+( $MVN_COMPILE & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$MVN_PID 4>$MVN_EXIT | tee $MVN_OUT
+
+echo "Trying to KILL watchdog (${WD_PID})."
+
+# Make sure to kill the watchdog in any case after $MVN has completed
+( kill $WD_PID 2>&1 ) > /dev/null
+
+EXIT_CODE=$(<$MVN_EXIT)
+
+echo "MVN exited with EXIT CODE: ${EXIT_CODE}."
-# Run $MVN and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID to
+rm $MVN_PID
+rm $MVN_EXIT
+
+# Run tests
+
+echo "RUNNING '${MVN_TEST}'."
+
+# Run $MVN_TEST and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID to
# allow the watchdog to kill $MVN if it is not producing any output anymore. $MVN_EXIT contains
# the exit code. This is important for Travis' build life-cycle (success/failure).
-( $MVN & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$MVN_PID 4>$MVN_EXIT | tee $MVN_OUT
+( $MVN_TEST & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$MVN_PID 4>$MVN_EXIT | tee $MVN_OUT
echo "Trying to KILL watchdog (${WD_PID})."
@@ -245,7 +364,14 @@ echo "MVN exited with EXIT CODE: ${EXIT_CODE}."
rm $MVN_PID
rm $MVN_EXIT
-check_shaded_artifacts
+# Post
+
+# only misc builds flink-dist
+case $TEST in
+ (misc)
+ check_shaded_artifacts
+ ;;
+esac
put_yarn_logs_to_artifacts