You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2018/07/03 10:00:09 UTC
[1/3] flink git commit: [hotfix][tests] Reuse existing functions for
cleaning logs
Repository: flink
Updated Branches:
refs/heads/master 75d12f967 -> 41277f6b7
[hotfix][tests] Reuse existing functions for cleaning logs
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a9d08a22
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a9d08a22
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a9d08a22
Branch: refs/heads/master
Commit: a9d08a22b569b74281b5f471beda9834cfc30237
Parents: 75d12f9
Author: zentol <ch...@apache.org>
Authored: Wed Jun 27 12:58:49 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Tue Jul 3 11:44:08 2018 +0200
----------------------------------------------------------------------
flink-end-to-end-tests/test-scripts/test-runner-common.sh | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a9d08a22/flink-end-to-end-tests/test-scripts/test-runner-common.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/test-scripts/test-runner-common.sh b/flink-end-to-end-tests/test-scripts/test-runner-common.sh
index eeae0f7..8758da0 100644
--- a/flink-end-to-end-tests/test-scripts/test-runner-common.sh
+++ b/flink-end-to-end-tests/test-scripts/test-runner-common.sh
@@ -70,7 +70,8 @@ function cleanup {
jm_kill_all
rm -rf $TEST_DATA_DIR 2> /dev/null
revert_default_config
- rm -rf $FLINK_DIR/log/* 2> /dev/null
+ clean_log_files
+ clean_stdout_files
}
trap cleanup EXIT
[2/3] flink git commit: [hotfix][tests] Simplify jar paths to QS tests
Posted by ch...@apache.org.
[hotfix][tests] Simplify jar paths to QS tests
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/02f016e4
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/02f016e4
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/02f016e4
Branch: refs/heads/master
Commit: 02f016e47dd2e70f7e856d0bef386635ef01a496
Parents: a9d08a2
Author: zentol <ch...@apache.org>
Authored: Wed Jun 27 12:59:16 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Tue Jul 3 11:44:13 2018 +0200
----------------------------------------------------------------------
.../test-scripts/test_queryable_state_restart_tm.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/02f016e4/flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh b/flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh
index 06199ea..f3bdcd3 100755
--- a/flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh
+++ b/flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh
@@ -20,8 +20,8 @@
source "$(dirname "$0")"/common.sh
source "$(dirname "$0")"/queryable_state_base.sh
-QUERYABLE_STATE_SERVER_JAR=${TEST_INFRA_DIR}/../../flink-end-to-end-tests/flink-queryable-state-test/target/QsStateProducer.jar
-QUERYABLE_STATE_CLIENT_JAR=${TEST_INFRA_DIR}/../../flink-end-to-end-tests/flink-queryable-state-test/target/QsStateClient.jar
+QUERYABLE_STATE_SERVER_JAR=${END_TO_END_DIR}/flink-queryable-state-test/target/QsStateProducer.jar
+QUERYABLE_STATE_CLIENT_JAR=${END_TO_END_DIR}/flink-queryable-state-test/target/QsStateClient.jar
#####################
# Test that queryable state works as expected with HA mode when restarting a taskmanager
[3/3] flink git commit: [FLINK-9674][tests] Replace hard-coded sleeps
in QS E2E test
Posted by ch...@apache.org.
[FLINK-9674][tests] Replace hard-coded sleeps in QS E2E test
This closes #6216.
This closes #6025.
This closes #5297.
This closes #6211.
This closes #5899.
This closes #5888.
This closes #5901.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/41277f6b
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/41277f6b
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/41277f6b
Branch: refs/heads/master
Commit: 41277f6b7447f3542e439b7ae82b99be862df7c2
Parents: 02f016e
Author: zentol <ch...@apache.org>
Authored: Wed Jun 27 13:01:06 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Tue Jul 3 11:44:16 2018 +0200
----------------------------------------------------------------------
flink-end-to-end-tests/test-scripts/common.sh | 20 +++++++++++++++++++-
.../test_queryable_state_restart_tm.sh | 20 ++++++++++++++------
2 files changed, 33 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/41277f6b/flink-end-to-end-tests/test-scripts/common.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh
index 610be2d..4ed83b0 100644
--- a/flink-end-to-end-tests/test-scripts/common.sh
+++ b/flink-end-to-end-tests/test-scripts/common.sh
@@ -260,9 +260,27 @@ function stop_cluster {
fi
}
+function wait_for_job_state_transition {
+ local job=$1
+ local initial_state=$2
+ local next_state=$3
+
+ echo "Waiting for job ($job) to switch from state ${initial_state} to state ${next_state} ..."
+
+ while : ; do
+ N=$(grep -o "($job) switched from state ${initial_state} to ${next_state}" $FLINK_DIR/log/*standalonesession*.log | tail -1)
+
+ if [[ -z $N ]]; then
+ sleep 1
+ else
+ break
+ fi
+ done
+}
+
function wait_job_running {
for i in {1..10}; do
- JOB_LIST_RESULT=$("$FLINK_DIR"/bin/flink list | grep "$1")
+ JOB_LIST_RESULT=$("$FLINK_DIR"/bin/flink list -r | grep "$1")
if [[ "$JOB_LIST_RESULT" == "" ]]; then
echo "Job ($1) is not yet running."
http://git-wip-us.apache.org/repos/asf/flink/blob/41277f6b/flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh b/flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh
index f3bdcd3..d8d08df 100755
--- a/flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh
+++ b/flink-end-to-end-tests/test-scripts/test_queryable_state_restart_tm.sh
@@ -55,6 +55,11 @@ function run_test() {
clean_log_files
clean_stdout_files
+ backup_config
+ # speeds up TM loss detection
+ set_conf "heartbeat.interval" "2000"
+ set_conf "heartbeat.timeout" "10000"
+
link_queryable_state_lib
start_cluster
@@ -85,20 +90,23 @@ function run_test() {
exit 1
fi
- local current_num_checkpoints=current_num_checkpoints$(get_completed_number_of_checkpoints ${JOB_ID})
-
kill_random_taskmanager
latest_snapshot_count=$(cat $FLINK_DIR/log/*out* | grep "on snapshot" | tail -n 1 | awk '{print $4}')
echo "Latest snapshot count was ${latest_snapshot_count}"
- sleep 65 # this is a little longer than the heartbeat timeout so that the TM is gone
+ # wait until the TM loss was detected
+ wait_for_job_state_transition ${JOB_ID} "RESTARTING" "CREATED"
start_and_wait_for_tm
+ wait_job_running ${JOB_ID}
+
+ local current_num_checkpoints="$(get_completed_number_of_checkpoints ${JOB_ID})"
# wait for some more checkpoint to have happened
- ((current_num_checkpoints+=2))
- wait_for_number_of_checkpoints ${JOB_ID} ${current_num_checkpoints} 60
+ local expected_num_checkpoints=$((current_num_checkpoints + 5))
+
+ wait_for_number_of_checkpoints ${JOB_ID} ${expected_num_checkpoints} 60
local num_entries_in_map_state_after=$(java -jar ${QUERYABLE_STATE_CLIENT_JAR} \
--host ${SERVER} \
@@ -135,7 +143,7 @@ function wait_for_number_of_checkpoints {
local timeout=$3
local count=0
- echo "Starting to wait for checkpoints"
+ echo "Starting to wait for completion of ${expected_num_checkpoints} checkpoints"
while (($(get_completed_number_of_checkpoints ${job_id}) < ${expected_num_checkpoints})); do
if [[ ${count} -gt ${timeout} ]]; then