You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tz...@apache.org on 2018/05/09 04:33:32 UTC
flink git commit: [FLINK-9093] [e2e-tests] Extend savepoint e2e tests
for different state backends
Repository: flink
Updated Branches:
refs/heads/master ba3e2711a -> d6d2c36c6
[FLINK-9093] [e2e-tests] Extend savepoint e2e tests for different state backends
This closes #5926.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d6d2c36c
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d6d2c36c
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d6d2c36c
Branch: refs/heads/master
Commit: d6d2c36c6f89fa40277db9e6bd84f531c901c353
Parents: ba3e271
Author: Tzu-Li (Gordon) Tai <tz...@apache.org>
Authored: Fri Apr 27 15:52:55 2018 +0800
Committer: Tzu-Li (Gordon) Tai <tz...@apache.org>
Committed: Wed May 9 12:29:43 2018 +0800
----------------------------------------------------------------------
flink-end-to-end-tests/run-nightly-tests.sh | 60 ++++++++++++++++++--
.../test-scripts/test_resume_savepoint.sh | 7 +++
2 files changed, 61 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/d6d2c36c/flink-end-to-end-tests/run-nightly-tests.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index 117a131..98d75f03 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -58,25 +58,73 @@ fi
if [ $EXIT_CODE == 0 ]; then
printf "\n==============================================================================\n"
- printf "Running Resuming Savepoint (no parallelism change) end-to-end test\n"
+ printf "Running Resuming Savepoint (file, async, no parallelism change) end-to-end test\n"
printf "==============================================================================\n"
- $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
+ STATE_BACKEND_TYPE=file STATE_BACKEND_FILE_ASYNC=true $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
printf "\n==============================================================================\n"
- printf "Running Resuming Savepoint (scale up) end-to-end test\n"
+ printf "Running Resuming Savepoint (file, sync, no parallelism change) end-to-end test\n"
printf "==============================================================================\n"
- $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4
+ STATE_BACKEND_TYPE=file STATE_BACKEND_FILE_ASYNC=false $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
EXIT_CODE=$?
fi
if [ $EXIT_CODE == 0 ]; then
printf "\n==============================================================================\n"
- printf "Running Resuming Savepoint (scale down) end-to-end test\n"
+ printf "Running Resuming Savepoint (file, async, scale up) end-to-end test\n"
printf "==============================================================================\n"
- $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2
+ STATE_BACKEND_TYPE=file STATE_BACKEND_FILE_ASYNC=true $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4
+ EXIT_CODE=$?
+fi
+
+if [ $EXIT_CODE == 0 ]; then
+ printf "\n==============================================================================\n"
+ printf "Running Resuming Savepoint (file, sync, scale up) end-to-end test\n"
+ printf "==============================================================================\n"
+ STATE_BACKEND_TYPE=file STATE_BACKEND_FILE_ASYNC=false $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4
+ EXIT_CODE=$?
+fi
+
+if [ $EXIT_CODE == 0 ]; then
+ printf "\n==============================================================================\n"
+ printf "Running Resuming Savepoint (file, async, scale down) end-to-end test\n"
+ printf "==============================================================================\n"
+ STATE_BACKEND_TYPE=file STATE_BACKEND_FILE_ASYNC=true $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2
+ EXIT_CODE=$?
+fi
+
+if [ $EXIT_CODE == 0 ]; then
+ printf "\n==============================================================================\n"
+ printf "Running Resuming Savepoint (file, sync, scale down) end-to-end test\n"
+ printf "==============================================================================\n"
+ STATE_BACKEND_TYPE=file STATE_BACKEND_FILE_ASYNC=false $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2
+ EXIT_CODE=$?
+fi
+
+if [ $EXIT_CODE == 0 ]; then
+ printf "\n==============================================================================\n"
+ printf "Running Resuming Savepoint (rocks, no parallelism change) end-to-end test\n"
+ printf "==============================================================================\n"
+ STATE_BACKEND_TYPE=rocks $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 2
+ EXIT_CODE=$?
+fi
+
+if [ $EXIT_CODE == 0 ]; then
+ printf "\n==============================================================================\n"
+ printf "Running Resuming Savepoint (rocks, scale up) end-to-end test\n"
+ printf "==============================================================================\n"
+ STATE_BACKEND_TYPE=rocks $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 2 4
+ EXIT_CODE=$?
+fi
+
+if [ $EXIT_CODE == 0 ]; then
+ printf "\n==============================================================================\n"
+ printf "Running Resuming Savepoint (rocks, scale down) end-to-end test\n"
+ printf "==============================================================================\n"
+ STATE_BACKEND_TYPE=rocks $END_TO_END_DIR/test-scripts/test_resume_savepoint.sh 4 2
EXIT_CODE=$?
fi
http://git-wip-us.apache.org/repos/asf/flink/blob/d6d2c36c/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh b/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh
index ae37f4a..259296a 100755
--- a/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh
+++ b/flink-end-to-end-tests/test-scripts/test_resume_savepoint.sh
@@ -33,6 +33,9 @@ else
NUM_SLOTS=$NEW_DOP
fi
+STATE_BACKEND_TYPE=${STATE_BACKEND_TYPE:-file}
+STATE_BACKEND_FILE_ASYNC=${STATE_BACKEND_FILE_ASYNC:-true}
+
backup_config
change_conf "taskmanager.numberOfTaskSlots" "1" "${NUM_SLOTS}"
setup_flink_slf4j_metric_reporter
@@ -62,7 +65,9 @@ TEST_PROGRAM_JAR=$TEST_INFRA_DIR/../../flink-end-to-end-tests/flink-datastream-a
DATASTREAM_JOB=$($FLINK_DIR/bin/flink run -d -p $ORIGINAL_DOP $TEST_PROGRAM_JAR \
--test.semantics exactly-once \
--environment.parallelism $ORIGINAL_DOP \
+ --state_backend $STATE_BACKEND_TYPE \
--state_backend.checkpoint_directory $CHECKPOINT_DIR \
+ --state_backend.file.async $STATE_BACKEND_FILE_ASYNC \
--sequence_generator_source.sleep_time 15 \
--sequence_generator_source.sleep_after_elements 1 \
| grep "Job has been submitted with JobID" | sed 's/.* //g')
@@ -85,7 +90,9 @@ OLD_NUM_METRICS=$(get_num_metric_samples)
DATASTREAM_JOB=$($FLINK_DIR/bin/flink run -s $SAVEPOINT_PATH -p $NEW_DOP -d $TEST_PROGRAM_JAR \
--test.semantics exactly-once \
--environment.parallelism $NEW_DOP \
+ --state_backend $STATE_BACKEND_TYPE \
--state_backend.checkpoint_directory $CHECKPOINT_DIR \
+ --state_backend.file.async $STATE_BACKEND_FILE_ASYNC \
--sequence_generator_source.sleep_time 15 \
--sequence_generator_source.sleep_after_elements 1 \
| grep "Job has been submitted with JobID" | sed 's/.* //g')