You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ji...@apache.org on 2019/05/15 01:33:46 UTC
[flink] branch master updated: [FLINK-12370][python][travis]
Integrated Travis for Python API.
This is an automated email from the ASF dual-hosted git repository.
jincheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new d95d395 [FLINK-12370][python][travis] Integrated Travis for Python API.
d95d395 is described below
commit d95d395d92acca210e107663ed0e96c3285f0cf5
Author: Wei Zhong <we...@gmail.com>
AuthorDate: Fri May 10 09:36:27 2019 +0800
[FLINK-12370][python][travis] Integrated Travis for Python API.
Brief change log:
- Added python stage for Python API CI testing.
- Integrated flink-python/dev/lint-python.sh for CI testing of the Python API.
This closes #8392
---
.travis.yml | 17 +++++
tools/travis/stage.sh | 1 +
tools/travis_controller.sh | 4 +-
.../{travis_mvn_watchdog.sh => travis_watchdog.sh} | 89 +++++++++++++---------
4 files changed, 75 insertions(+), 36 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index f8243b4..3f15835 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -84,6 +84,10 @@ jobs:
env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11"
name: core
- if: type in (pull_request, push)
+ script: ./tools/travis_controller.sh python
+ env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11"
+ name: python
+ - if: type in (pull_request, push)
script: ./tools/travis_controller.sh libraries
env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11"
name: libraries
@@ -116,6 +120,10 @@ jobs:
env: PROFILE="-Dhadoop.version=2.4.1 -Pinclude-kinesis"
name: core - hadoop 2.4.1
- if: type = cron
+ script: ./tools/travis_controller.sh python
+ env: PROFILE="-Dhadoop.version=2.4.1 -Pinclude-kinesis"
+ name: python - hadoop 2.4.1
+ - if: type = cron
script: ./tools/travis_controller.sh libraries
env: PROFILE="-Dhadoop.version=2.4.1 -Pinclude-kinesis"
name: libraries - hadoop 2.4.1
@@ -148,6 +156,10 @@ jobs:
env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.12"
name: core - scala 2.12
- if: type = cron
+ script: ./tools/travis_controller.sh python
+ env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.12"
+ name: python - scala 2.12
+ - if: type = cron
script: ./tools/travis_controller.sh libraries
env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.12"
name: libraries - scala 2.12
@@ -183,6 +195,11 @@ jobs:
name: core - jdk 9
- if: type = cron
jdk: "openjdk9"
+ script: ./tools/travis_controller.sh python
+ env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11 -Djdk9"
+ name: python
+ - if: type = cron
+ jdk: "openjdk9"
script: ./tools/travis_controller.sh libraries
env: PROFILE="-Dhadoop.version=2.8.3 -Pinclude-kinesis -Dinclude_hadoop_aws -Dscala-2.11 -Djdk9"
name: libraries - jdk 9
diff --git a/tools/travis/stage.sh b/tools/travis/stage.sh
index 8fa4367..3e84b2a 100644
--- a/tools/travis/stage.sh
+++ b/tools/travis/stage.sh
@@ -19,6 +19,7 @@
STAGE_COMPILE="compile"
STAGE_CORE="core"
+STAGE_PYTHON="python"
STAGE_LIBRARIES="libraries"
STAGE_CONNECTORS="connectors"
STAGE_TESTS="tests"
diff --git a/tools/travis_controller.sh b/tools/travis_controller.sh
index d659143..eef6503 100755
--- a/tools/travis_controller.sh
+++ b/tools/travis_controller.sh
@@ -158,7 +158,7 @@ if [ $STAGE == "$STAGE_COMPILE" ]; then
# by removing files not required for subsequent stages
# jars are re-built in subsequent stages, so no need to cache them (cannot be avoided)
- find "$CACHE_FLINK_DIR" -maxdepth 8 -type f -name '*.jar' | xargs rm -rf
+ find "$CACHE_FLINK_DIR" -maxdepth 8 -type f -name '*.jar' ! -path "$CACHE_FLINK_DIR/flink-dist/*" ! -path "*tests.jar" | xargs rm -rf
# .git directory
# not deleting this can cause build stability issues
@@ -201,7 +201,7 @@ elif [ $STAGE != "$STAGE_CLEANUP" ]; then
travis_time_finish
end_fold "adjust_timestamps"
- TEST="$STAGE" "./tools/travis_mvn_watchdog.sh" 300
+ TEST="$STAGE" "./tools/travis_watchdog.sh" 300
EXIT_CODE=$?
elif [ $STAGE == "$STAGE_CLEANUP" ]; then
echo "Cleaning up $CACHE_BUILD_DIR"
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_watchdog.sh
similarity index 77%
rename from tools/travis_mvn_watchdog.sh
rename to tools/travis_watchdog.sh
index 4dff95f..d53526f 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_watchdog.sh
@@ -45,6 +45,11 @@ SLEEP_TIME=20
LOG4J_PROPERTIES=${HERE}/log4j-travis.properties
+PYTHON_TEST="./flink-python/dev/lint-python.sh"
+PYTHON_PID="${ARTIFACTS_DIR}/watchdog.python.pid"
+PYTHON_EXIT="${ARTIFACTS_DIR}/watchdog.python.exit"
+PYTHON_OUT="${ARTIFACTS_DIR}/python.out"
+
MVN_COMPILE_MODULES=$(get_compile_modules_for_stage ${TEST})
MVN_TEST_MODULES=$(get_test_modules_for_stage ${TEST})
@@ -78,6 +83,20 @@ UPLOAD_SECRET_KEY=$ARTIFACTS_AWS_SECRET_KEY
ARTIFACTS_FILE=${TRAVIS_JOB_NUMBER}.tar.gz
+if [ $TEST == $STAGE_PYTHON ]; then
+ CMD=$PYTHON_TEST
+ CMD_PID=$PYTHON_PID
+ CMD_OUT=$PYTHON_OUT
+ CMD_EXIT=$PYTHON_EXIT
+ CMD_TYPE="PYTHON"
+else
+ CMD=$MVN_COMPILE
+ CMD_PID=$MVN_PID
+ CMD_OUT=$MVN_OUT
+ CMD_EXIT=$MVN_EXIT
+ CMD_TYPE="MVN"
+fi
+
# =============================================================================
# FUNCTIONS
# =============================================================================
@@ -145,10 +164,10 @@ put_yarn_logs_to_artifacts() {
mod_time () {
if [[ `uname` == 'Darwin' ]]; then
- eval $(stat -s $MVN_OUT)
+ eval $(stat -s $CMD_OUT)
echo $st_mtime
else
- echo `stat -c "%Y" $MVN_OUT`
+ echo `stat -c "%Y" $CMD_OUT`
fi
}
@@ -157,7 +176,7 @@ the_time() {
}
watchdog () {
- touch $MVN_OUT
+ touch $CMD_OUT
while true; do
sleep $SLEEP_TIME
@@ -171,7 +190,7 @@ watchdog () {
print_stacktraces | tee $TRACE_OUT
- kill $(<$MVN_PID)
+ kill $(<$CMD_PID)
exit 1
fi
@@ -194,54 +213,56 @@ cd $HERE/../
# Compile modules
-echo "RUNNING '${MVN_COMPILE}'."
+echo "RUNNING '${CMD}'."
-# Run $MVN_COMPILE and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID to
-# allow the watchdog to kill $MVN if it is not producing any output anymore. $MVN_EXIT contains
+# Run $CMD and pipe output to $CMD_OUT for the watchdog. The PID is written to $CMD_PID to
+# allow the watchdog to kill $CMD if it is not producing any output anymore. $CMD_EXIT contains
# the exit code. This is important for Travis' build life-cycle (success/failure).
-( $MVN_COMPILE & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$MVN_PID 4>$MVN_EXIT | tee $MVN_OUT
+( $CMD & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$CMD_PID 4>$CMD_EXIT | tee $CMD_OUT
-EXIT_CODE=$(<$MVN_EXIT)
+EXIT_CODE=$(<$CMD_EXIT)
-echo "MVN exited with EXIT CODE: ${EXIT_CODE}."
+echo "${CMD_TYPE} exited with EXIT CODE: ${EXIT_CODE}."
-# Make sure to kill the watchdog in any case after $MVN_COMPILE has completed
+# Make sure to kill the watchdog in any case after $CMD has completed
echo "Trying to KILL watchdog (${WD_PID})."
( kill $WD_PID 2>&1 ) > /dev/null
-rm $MVN_PID
-rm $MVN_EXIT
+rm $CMD_PID
+rm $CMD_EXIT
# Run tests if compilation was successful
-if [ $EXIT_CODE == 0 ]; then
+if [ $CMD_TYPE == "MVN" ]; then
+ if [ $EXIT_CODE == 0 ]; then
- # Start watching $MVN_OUT
- watchdog &
- echo "STARTED watchdog (${WD_PID})."
+ # Start watching $MVN_OUT
+ watchdog &
+ echo "STARTED watchdog (${WD_PID})."
- WD_PID=$!
+ WD_PID=$!
- echo "RUNNING '${MVN_TEST}'."
+ echo "RUNNING '${MVN_TEST}'."
- # Run $MVN_TEST and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID to
- # allow the watchdog to kill $MVN if it is not producing any output anymore. $MVN_EXIT contains
- # the exit code. This is important for Travis' build life-cycle (success/failure).
- ( $MVN_TEST & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$MVN_PID 4>$MVN_EXIT | tee $MVN_OUT
+ # Run $MVN_TEST and pipe output to $MVN_OUT for the watchdog. The PID is written to $MVN_PID to
+ # allow the watchdog to kill $MVN if it is not producing any output anymore. $MVN_EXIT contains
+ # the exit code. This is important for Travis' build life-cycle (success/failure).
+ ( $MVN_TEST & PID=$! ; echo $PID >&3 ; wait $PID ; echo $? >&4 ) 3>$MVN_PID 4>$MVN_EXIT | tee $MVN_OUT
- EXIT_CODE=$(<$MVN_EXIT)
+ EXIT_CODE=$(<$MVN_EXIT)
- echo "MVN exited with EXIT CODE: ${EXIT_CODE}."
+ echo "MVN exited with EXIT CODE: ${EXIT_CODE}."
- # Make sure to kill the watchdog in any case after $MVN_TEST has completed
- echo "Trying to KILL watchdog (${WD_PID})."
- ( kill $WD_PID 2>&1 ) > /dev/null
+ # Make sure to kill the watchdog in any case after $MVN_TEST has completed
+ echo "Trying to KILL watchdog (${WD_PID})."
+ ( kill $WD_PID 2>&1 ) > /dev/null
- rm $MVN_PID
- rm $MVN_EXIT
-else
- echo "=============================================================================="
- echo "Compilation failure detected, skipping test execution."
- echo "=============================================================================="
+ rm $MVN_PID
+ rm $MVN_EXIT
+ else
+ echo "=============================================================================="
+ echo "Compilation failure detected, skipping test execution."
+ echo "=============================================================================="
+ fi
fi
# Post