You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ch...@apache.org on 2018/07/03 19:09:07 UTC
[2/5] flink git commit: [FLINK-9301][tests] Enable
high-parallellism-iterations test
[FLINK-9301][tests] Enable high-parallellism-iterations test
This closes #6024.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/885640f7
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/885640f7
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/885640f7
Branch: refs/heads/master
Commit: 885640f781aa66359d929eb387f27a6024d75025
Parents: 81d1355
Author: Andrey Zagrebin <az...@gmail.com>
Authored: Wed Jun 27 11:16:51 2018 +0200
Committer: zentol <ch...@apache.org>
Committed: Tue Jul 3 21:08:45 2018 +0200
----------------------------------------------------------------------
flink-end-to-end-tests/run-nightly-tests.sh | 2 ++
flink-end-to-end-tests/test-scripts/common.sh | 29 ++++++++++++++++++++
.../test_high_parallelism_iterations.sh | 24 ++++------------
3 files changed, 37 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/885640f7/flink-end-to-end-tests/run-nightly-tests.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index a493f49..15c73d5 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -43,6 +43,8 @@ echo "Flink distribution directory: $FLINK_DIR"
# run_test "<description>" "$END_TO_END_DIR/test-scripts/<script_name>"
+run_test "ConnectedComponents iterations with high parallelism end-to-end test" "$END_TO_END_DIR/test-scripts/test_high_parallelism_iterations.sh 25"
+
run_test "Queryable state (rocksdb) end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state.sh rocksdb"
run_test "Queryable state (rocksdb) with TM restart end-to-end test" "$END_TO_END_DIR/test-scripts/test_queryable_state_restart_tm.sh"
http://git-wip-us.apache.org/repos/asf/flink/blob/885640f7/flink-end-to-end-tests/test-scripts/common.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/test-scripts/common.sh b/flink-end-to-end-tests/test-scripts/common.sh
index 4ed83b0..e5ad458 100644
--- a/flink-end-to-end-tests/test-scripts/common.sh
+++ b/flink-end-to-end-tests/test-scripts/common.sh
@@ -24,6 +24,14 @@ if [[ -z $FLINK_DIR ]]; then
exit 1
fi
+case "$(uname -s)" in
+ Linux*) OS_TYPE=linux;;
+ Darwin*) OS_TYPE=mac;;
+ CYGWIN*) OS_TYPE=cygwin;;
+ MINGW*) OS_TYPE=mingw;;
+ *) OS_TYPE="UNKNOWN:${unameOut}"
+esac
+
export EXIT_CODE=0
echo "Flink dist directory: $FLINK_DIR"
@@ -39,6 +47,27 @@ cd $TEST_ROOT
export TEST_DATA_DIR=$TEST_INFRA_DIR/temp-test-directory-$(date +%S%N)
echo "TEST_DATA_DIR: $TEST_DATA_DIR"
+function print_mem_use_osx {
+ declare -a mem_types=("active" "inactive" "wired down")
+ used=""
+ for mem_type in "${mem_types[@]}"
+ do
+ used_type=$(vm_stat | grep "Pages ${mem_type}:" | awk '{print $NF}' | rev | cut -c 2- | rev)
+ let used_type="(${used_type}*4096)/1024/1024"
+ used="$used $mem_type=${used_type}MB"
+ done
+ let mem=$(sysctl -n hw.memsize)/1024/1024
+ echo "Memory Usage: ${used} total=${mem}MB"
+}
+
+function print_mem_use {
+ if [[ "$OS_TYPE" == "mac" ]]; then
+ print_mem_use_osx
+ else
+ free -m | awk 'NR==2{printf "Memory Usage: used=%sMB total=%sMB %.2f%%\n", $3,$2,$3*100/$2 }'
+ fi
+}
+
function backup_config() {
# back up the masters and flink-conf.yaml
cp $FLINK_DIR/conf/masters $FLINK_DIR/conf/masters.bak
http://git-wip-us.apache.org/repos/asf/flink/blob/885640f7/flink-end-to-end-tests/test-scripts/test_high_parallelism_iterations.sh
----------------------------------------------------------------------
diff --git a/flink-end-to-end-tests/test-scripts/test_high_parallelism_iterations.sh b/flink-end-to-end-tests/test-scripts/test_high_parallelism_iterations.sh
index dbdacef..aa937ae 100755
--- a/flink-end-to-end-tests/test-scripts/test_high_parallelism_iterations.sh
+++ b/flink-end-to-end-tests/test-scripts/test_high_parallelism_iterations.sh
@@ -19,15 +19,14 @@
source "$(dirname "$0")"/common.sh
-PARALLELISM="${PARALLELISM:-100}"
+PARALLELISM="${1:-25}"
TEST=flink-high-parallelism-iterations-test
TEST_PROGRAM_NAME=HighParallelismIterationsTestProgram
TEST_PROGRAM_JAR=${END_TO_END_DIR}/$TEST/target/$TEST_PROGRAM_NAME.jar
-echo "Run Not So MiniCluster Iterations Graph Connected Components Program"
+backup_config
-cp $FLINK_DIR/conf/flink-conf.yaml $FLINK_DIR/conf/flink-conf.yaml.bak
echo "taskmanager.heap.mb: 52" >> $FLINK_DIR/conf/flink-conf.yaml # 52Mb x 100 TMs = 5Gb total heap
@@ -41,27 +40,16 @@ echo "taskmanager.network.netty.client.numThreads: 1" >> $FLINK_DIR/conf/flink-c
echo "taskmanager.numberOfTaskSlots: 1" >> $FLINK_DIR/conf/flink-conf.yaml
+print_mem_use
start_cluster
+print_mem_use
let TMNUM=$PARALLELISM-1
echo "Start $TMNUM more task managers"
for i in `seq 1 $TMNUM`; do
$FLINK_DIR/bin/taskmanager.sh start
+ print_mem_use
done
-function test_cleanup {
- # don't call ourselves again for another signal interruption
- trap "exit -1" INT
- # don't call ourselves again for normal exit
- trap "" EXIT
-
- stop_cluster
- $FLINK_DIR/bin/taskmanager.sh stop-all
-
- # revert our modifications to the Flink distribution
- mv -f $FLINK_DIR/conf/flink-conf.yaml.bak $FLINK_DIR/conf/flink-conf.yaml
-}
-trap test_cleanup INT
-trap test_cleanup EXIT
-
$FLINK_DIR/bin/flink run -p $PARALLELISM $TEST_PROGRAM_JAR
+print_mem_use