You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2018/05/17 06:08:57 UTC
[11/13] flink git commit: [FLINK-9354][travis] Print execution times
for nightly E2E tests
[FLINK-9354][travis] Print execution times for nightly E2E tests
This closes #6008.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/c7c7b558
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/c7c7b558
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/c7c7b558
Branch: refs/heads/release-1.5
Commit: c7c7b558de44748545b64111649119540340a038
Parents: c8d6f06
Author: zentol <ch...@apache.org>
Authored: Mon May 14 12:27:34 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Wed May 16 20:22:33 2018 +0200
----------------------------------------------------------------------
flink-end-to-end-tests/run-nightly-tests.sh | 96 +++++---------------
flink-end-to-end-tests/run-pre-commit-tests.sh | 44 ++++-----
flink-end-to-end-tests/test-scripts/common.sh | 31 +++++++
.../test_resume_externalized_checkpoints.sh | 4 +-
.../test-scripts/test_resume_savepoint.sh | 7 +-
5 files changed, 74 insertions(+), 108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/c7c7b558/flink-end-to-end-tests/run-nightly-tests.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index d00885d..d4309c1 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -30,6 +30,8 @@ if [ -z "$FLINK_DIR" ] ; then
exit 1
fi
+source "$(dirname "$0")"/test-scripts/common.sh
+
FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd )`" # absolutized and normalized
echo "flink-end-to-end-test directory: $END_TO_END_DIR"
@@ -40,147 +42,93 @@ EXIT_CODE=0
# Template for adding a test:
# if [ $EXIT_CODE == 0 ]; then
-# printf "\n==============================================================================\n"
-# printf "Running my fancy nightly end-to-end test\n"
-# printf "==============================================================================\n"
-# $END_TO_END_DIR/test-scripts/test_something_very_fancy.sh
-# EXIT_CODE=$?
+# run_test "<description>" "$END_TO_END_DIR/test-scripts/<script_name>"
+# EXIT_CODE=$?
# fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running HA end-to-end test\n"
- printf "==============================================================================\n"
- $END_TO_END_DIR/test-scripts/test_ha.sh
- EXIT_CODE=$?
+ run_test "HA end-to-end test" "$END_TO_END_DIR/test-scripts/test_ha.sh"
+ EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Resuming Savepoint (file, async, no parallelism change) end-to-end test\n"
- printf "==============================================================================\n"
- STATE_BACKEND_TYPE=file STATE_BACKEND_FILE_ASYNC=true $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
+ run_test "Resuming Savepoint (file, async, no parallelism change) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 file true"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Resuming Savepoint (file, sync, no parallelism change) end-to-end test\n"
- printf "==============================================================================\n"
- STATE_BACKEND_TYPE=file STATE_BACKEND_FILE_ASYNC=false $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
+ run_test "Resuming Savepoint (file, sync, no parallelism change) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 file false"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Resuming Savepoint (file, async, scale up) end-to-end test\n"
- printf "==============================================================================\n"
- STATE_BACKEND_TYPE=file STATE_BACKEND_FILE_ASYNC=true $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4
+ run_test "Resuming Savepoint (file, async, scale up) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 file true"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Resuming Savepoint (file, sync, scale up) end-to-end test\n"
- printf "==============================================================================\n"
- STATE_BACKEND_TYPE=file STATE_BACKEND_FILE_ASYNC=false $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4
+ run_test "Resuming Savepoint (file, sync, scale up) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 file false"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Resuming Savepoint (file, async, scale down) end-to-end test\n"
- printf "==============================================================================\n"
- STATE_BACKEND_TYPE=file STATE_BACKEND_FILE_ASYNC=true $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2
+ run_test "Resuming Savepoint (file, async, scale down) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 file true"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Resuming Savepoint (file, sync, scale down) end-to-end test\n"
- printf "==============================================================================\n"
- STATE_BACKEND_TYPE=file STATE_BACKEND_FILE_ASYNC=false $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2
+ run_test "Resuming Savepoint (file, sync, scale down) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 file false"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Resuming Savepoint (rocks, no parallelism change) end-to-end test\n"
- printf "==============================================================================\n"
- STATE_BACKEND_TYPE=rocks $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
+ run_test "Resuming Savepoint (rocks, no parallelism change) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2 rocks"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Resuming Savepoint (rocks, scale up) end-to-end test\n"
- printf "==============================================================================\n"
- STATE_BACKEND_TYPE=rocks $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4
+ run_test "Resuming Savepoint (rocks, scale up) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4 rocks"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Resuming Savepoint (rocks, scale down) end-to-end test\n"
- printf "==============================================================================\n"
- STATE_BACKEND_TYPE=rocks $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2
+ run_test "Resuming Savepoint (rocks, scale down) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2 rocks"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Resuming Externalized Checkpoint (file, async) end-to-end test\n"
- printf "==============================================================================\n"
- STATE_BACKEND_TYPE=file STATE_BACKEND_FILE_ASYNC=true $END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh
+ run_test "Resuming Externalized Checkpoint (file, async) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file true"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Resuming Externalized Checkpoint (file, sync) end-to-end test\n"
- printf "==============================================================================\n"
- STATE_BACKEND_TYPE=file STATE_BACKEND_FILE_ASYNC=false $END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh
+ run_test "Resuming Externalized Checkpoint (file, sync) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh file false"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Resuming Externalized Checkpoint (rocks) end-to-end test\n"
- printf "==============================================================================\n"
- STATE_BACKEND_TYPE=rocks $END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh
+ run_test "Resuming Externalized Checkpoint (rocks) end-to-end test" "$END_TO_END_DIR/test-scripts/test_resume_externalized_checkpoints.sh rocks"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running DataSet allround nightly end-to-end test\n"
- printf "==============================================================================\n"
- $END_TO_END_DIR/test-scripts/test_batch_allround.sh
+ run_test "DataSet allround end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_allround.sh"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Streaming SQL nightly end-to-end test\n"
- printf "==============================================================================\n"
- $END_TO_END_DIR/test-scripts/test_streaming_sql.sh
+ run_test "Streaming SQL end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_sql.sh"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Streaming bucketing nightly end-to-end test\n"
- printf "==============================================================================\n"
- $END_TO_END_DIR/test-scripts/test_streaming_bucketing.sh
+ run_test "Streaming bucketing end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_bucketing.sh"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running stateful stream job upgrade nightly end-to-end test\n"
- printf "==============================================================================\n"
- $END_TO_END_DIR/test-scripts/test_stateful_stream_job_upgrade.sh 2 4
+ run_test "stateful stream job upgrade end-to-end test" "$END_TO_END_DIR/test-scripts/test_stateful_stream_job_upgrade.sh 2 4"
EXIT_CODE=$?
fi
http://git-wip-us.apache.org/repos/asf/flink/blob/c7c7b558/flink-end-to-end-tests/run-pre-commit-tests.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/run-pre-commit-tests.sh b/flink-end-to-end-tests/run-pre-commit-tests.sh
index 2c1810b..030e3eb 100755
--- a/flink-end-to-end-tests/run-pre-commit-tests.sh
+++ b/flink-end-to-end-tests/run-pre-commit-tests.sh
@@ -30,6 +30,8 @@ if [ -z "$FLINK_DIR" ] ; then
exit 1
fi
+source "$(dirname "$0")"/test-scripts/common.sh
+
FLINK_DIR="`( cd \"$FLINK_DIR\" && pwd )`" # absolutized and normalized
echo "flink-end-to-end-test directory: $END_TO_END_DIR"
@@ -37,59 +39,45 @@ echo "Flink distribution directory: $FLINK_DIR"
EXIT_CODE=0
+# Template for adding a test:
+
+# if [ $EXIT_CODE == 0 ]; then
+# run_test "<description>" "$END_TO_END_DIR/test-scripts/<script_name>"
+# EXIT_CODE=$?
+# fi
+
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Wordcount end-to-end test\n"
- printf "==============================================================================\n"
- $END_TO_END_DIR/test-scripts/test_batch_wordcount.sh
+ run_test "Streaming Python Wordcount end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_python_wordcount.sh"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Kafka end-to-end test\n"
- printf "==============================================================================\n"
- $END_TO_END_DIR/test-scripts/test_streaming_kafka010.sh
+ run_test "Wordcount end-to-end test" "$END_TO_END_DIR/test-scripts/test_batch_wordcount.sh"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running class loading end-to-end test\n"
- printf "==============================================================================\n"
- $END_TO_END_DIR/test-scripts/test_streaming_classloader.sh
+ run_test "Kafka end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_kafka010.sh"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Shaded Hadoop S3A end-to-end test\n"
- printf "==============================================================================\n"
- $END_TO_END_DIR/test-scripts/test_shaded_hadoop_s3a.sh
+ run_test "class loading end-to-end test" "$END_TO_END_DIR/test-scripts/test_streaming_classloader.sh"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Shaded Presto S3 end-to-end test\n"
- printf "==============================================================================\n"
- $END_TO_END_DIR/test-scripts/test_shaded_presto_s3.sh
+ run_test "Shaded Hadoop S3A end-to-end test" "$END_TO_END_DIR/test-scripts/test_shaded_hadoop_s3a.sh"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Hadoop-free Wordcount end-to-end test\n"
- printf "==============================================================================\n"
- CLUSTER_MODE=cluster $END_TO_END_DIR/test-scripts/test_hadoop_free.sh
+ run_test "Shaded Presto S3 end-to-end test" "$END_TO_END_DIR/test-scripts/test_shaded_presto_s3.sh"
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
- printf "\n==============================================================================\n"
- printf "Running Streaming Python Wordcount end-to-end test\n"
- printf "==============================================================================\n"
- $END_TO_END_DIR/test-scripts/test_streaming_python_wordcount.sh
+ run_test "Hadoop-free Wordcount end-to-end test" "$END_TO_END_DIR/test-scripts/test_hadoop_free.sh"
EXIT_CODE=$?
fi
http://git-wip-us.apache.org/repos/asf/flink/blob/c7c7b558/flink-end-to-end-tests/test-scripts/common.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh
index b2a09d0..0fbac42 100644
--- a/flink-end-to-end-tests/test-scripts/common.sh
+++ b/flink-end-to-end-tests/test-scripts/common.sh
@@ -399,6 +399,37 @@ function wait_num_checkpoints {
done
}
+# Starts the timer. Note that nested timers are not supported.
+function start_timer {
+ SECONDS=0
+}
+
+# prints the number of minutes and seconds that have elapsed since the last call to start_timer
+function end_timer {
+ duration=$SECONDS
+ echo "$(($duration / 60)) minutes and $(($duration % 60)) seconds elapsed."
+}
+
+#######################################
+# Prints the given description, runs the given test and prints how long the execution took.
+# Arguments:
+# $1: description of the test
+# $2: command to execute
+#######################################
+function run_test {
+ description="$1"
+ command="$2"
+
+ printf "\n==============================================================================\n"
+ printf "Running ${description}\n"
+ printf "==============================================================================\n"
+ start_timer
+ ${command}
+ exit_code="$?"
+ end_timer
+ return "${exit_code}"
+}
+
# make sure to clean up even in case of failures
function cleanup {
stop_cluster
http://git-wip-us.apache.org/repos/asf/flink/blob/c7c7b558/flink-end-to-end-tests/test-scripts/test_resume_externalized_checkpoints.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/test-scripts/test_resume_externalized_checkpoints.sh b/flink-end-to-end-tests/test-scripts/test_resume_externalized_checkpoints.sh
index 69a5851..3dc9909 100755
--- a/flink-end-to-end-tests/test-scripts/test_resume_externalized_checkpoints.sh
+++ b/flink-end-to-end-tests/test-scripts/test_resume_externalized_checkpoints.sh
@@ -19,8 +19,8 @@
source "$(dirname "$0")"/common.sh
-STATE_BACKEND_TYPE=${STATE_BACKEND_TYPE:-file}
-STATE_BACKEND_FILE_ASYNC=${STATE_BACKEND_FILE_ASYNC:-true}
+STATE_BACKEND_TYPE=${1:-file}
+STATE_BACKEND_FILE_ASYNC=${2:-true}
setup_flink_slf4j_metric_reporter
start_cluster
http://git-wip-us.apache.org/repos/asf/flink/blob/c7c7b558/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh b/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh
index 259296a..b19aa61 100755
--- a/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh
+++ b/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh
@@ -18,7 +18,7 @@
################################################################################
if [ -z $1 ] || [ -z $2 ]; then
- echo "Usage: ./test_resume_savepoint.sh <original_dop> <new_dop>"
+ echo "Usage: ./test_resume_savepoint.sh <original_dop> <new_dop> <state_backend_setting> <state_backend_file_async_setting>"
exit 1
fi
@@ -26,6 +26,8 @@ source "$(dirname "$0")"/common.sh
ORIGINAL_DOP=$1
NEW_DOP=$2
+STATE_BACKEND_TYPE=${3:-file}
+STATE_BACKEND_FILE_ASYNC=${4:-true}
if (( $ORIGINAL_DOP >= $NEW_DOP )); then
NUM_SLOTS=$ORIGINAL_DOP
@@ -33,9 +35,6 @@ else
NUM_SLOTS=$NEW_DOP
fi
-STATE_BACKEND_TYPE=${STATE_BACKEND_TYPE:-file}
-STATE_BACKEND_FILE_ASYNC=${STATE_BACKEND_FILE_ASYNC:-true}
-
backup_config
change_conf "taskmanager.numberOfTaskSlots" "1" "${NUM_SLOTS}"
setup_flink_slf4j_metric_reporter