You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2018/02/21 21:09:12 UTC
[2/7] flink git commit: [FLINK-8696] [shell scripts] Remove
JobManager local mode from shell scripts
[FLINK-8696] [shell scripts] Remove JobManager local mode from shell scripts
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d61664ca
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d61664ca
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d61664ca
Branch: refs/heads/master
Commit: d61664ca64bcb82c4e8ddf03a2ed38fe8edafa98
Parents: c68f95c
Author: Stephan Ewen <se...@apache.org>
Authored: Fri Feb 16 21:15:49 2018 +0100
Committer: Stephan Ewen <se...@apache.org>
Committed: Wed Feb 21 19:33:19 2018 +0100
----------------------------------------------------------------------
flink-dist/src/main/flink-bin/bin/jobmanager.sh | 23 +++-------------
.../src/main/flink-bin/bin/start-cluster.sh | 6 ++--
.../src/main/flink-bin/bin/start-local.sh | 29 --------------------
flink-dist/src/main/flink-bin/bin/stop-local.sh | 27 ------------------
test-infra/end-to-end-test/common.sh | 19 ++-----------
tools/travis_mvn_watchdog.sh | 12 ++++----
6 files changed, 15 insertions(+), 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/d61664ca/flink-dist/src/main/flink-bin/bin/jobmanager.sh
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/flink-bin/bin/jobmanager.sh b/flink-dist/src/main/flink-bin/bin/jobmanager.sh
index f226a03..10ff775 100755
--- a/flink-dist/src/main/flink-bin/bin/jobmanager.sh
+++ b/flink-dist/src/main/flink-bin/bin/jobmanager.sh
@@ -18,12 +18,11 @@
################################################################################
# Start/stop a Flink JobManager.
-USAGE="Usage: jobmanager.sh ((start|start-foreground) (local|cluster) [host] [webui-port])|stop|stop-all"
+USAGE="Usage: jobmanager.sh ((start|start-foreground) [host] [webui-port])|stop|stop-all"
STARTSTOP=$1
-EXECUTIONMODE=$2
-HOST=$3 # optional when starting multiple instances
-WEBUIPORT=$4 # optional when starting multiple instances
+HOST=$2 # optional when starting multiple instances
+WEBUIPORT=$3 # optional when starting multiple instances
if [[ $STARTSTOP != "start" ]] && [[ $STARTSTOP != "start-foreground" ]] && [[ $STARTSTOP != "stop" ]] && [[ $STARTSTOP != "stop-all" ]]; then
echo $USAGE
@@ -42,25 +41,11 @@ if [[ "${FLINK_MODE}" == "flip6" ]]; then
fi
if [[ $STARTSTOP == "start" ]] || [[ $STARTSTOP == "start-foreground" ]]; then
- if [ -z $EXECUTIONMODE ]; then
- echo "Missing execution mode (local|cluster) argument. $USAGE."
- exit 1
- fi
-
if [[ ! ${FLINK_JM_HEAP} =~ $IS_NUMBER ]] || [[ "${FLINK_JM_HEAP}" -lt "0" ]]; then
echo "[ERROR] Configured JobManager memory size is not a valid value. Please set '${KEY_JOBM_MEM_SIZE}' in ${FLINK_CONF_FILE}."
exit 1
fi
- if [ "$EXECUTIONMODE" = "local" ]; then
- if [[ ! ${FLINK_TM_HEAP} =~ $IS_NUMBER ]] || [[ "${FLINK_TM_HEAP}" -lt "0" ]]; then
- echo "[ERROR] Configured TaskManager memory size is not a valid value. Please set ${KEY_TASKM_MEM_SIZE} in ${FLINK_CONF_FILE}."
- exit 1
- fi
-
- FLINK_JM_HEAP=`expr $FLINK_JM_HEAP + $FLINK_TM_HEAP`
- fi
-
if [ "${FLINK_JM_HEAP}" -gt "0" ]; then
export JVM_ARGS="$JVM_ARGS -Xms"$FLINK_JM_HEAP"m -Xmx"$FLINK_JM_HEAP"m"
fi
@@ -69,7 +54,7 @@ if [[ $STARTSTOP == "start" ]] || [[ $STARTSTOP == "start-foreground" ]]; then
export FLINK_ENV_JAVA_OPTS="${FLINK_ENV_JAVA_OPTS} ${FLINK_ENV_JAVA_OPTS_JM}"
# Startup parameters
- args=("--configDir" "${FLINK_CONF_DIR}" "--executionMode" "${EXECUTIONMODE}")
+ args=("--configDir" "${FLINK_CONF_DIR}" "--executionMode" "cluster")
if [ ! -z $HOST ]; then
args+=("--host")
args+=("${HOST}")
http://git-wip-us.apache.org/repos/asf/flink/blob/d61664ca/flink-dist/src/main/flink-bin/bin/start-cluster.sh
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/flink-bin/bin/start-cluster.sh b/flink-dist/src/main/flink-bin/bin/start-cluster.sh
index d0f4ff9..4b6eba1 100755
--- a/flink-dist/src/main/flink-bin/bin/start-cluster.sh
+++ b/flink-dist/src/main/flink-bin/bin/start-cluster.sh
@@ -35,9 +35,9 @@ if [[ $HIGH_AVAILABILITY == "zookeeper" ]]; then
webuiport=${WEBUIPORTS[i]}
if [ ${MASTERS_ALL_LOCALHOST} = true ] ; then
- "${FLINK_BIN_DIR}"/jobmanager.sh start cluster "${master}" "${webuiport}"
+ "${FLINK_BIN_DIR}"/jobmanager.sh start "${master}" "${webuiport}"
else
- ssh -n $FLINK_SSH_OPTS $master -- "nohup /bin/bash -l \"${FLINK_BIN_DIR}/jobmanager.sh\" start cluster ${master} ${webuiport} &"
+ ssh -n $FLINK_SSH_OPTS $master -- "nohup /bin/bash -l \"${FLINK_BIN_DIR}/jobmanager.sh\" start ${master} ${webuiport} &"
fi
done
@@ -45,7 +45,7 @@ else
echo "Starting cluster."
# Start single JobManager on this machine
- "$FLINK_BIN_DIR"/jobmanager.sh start cluster
+ "$FLINK_BIN_DIR"/jobmanager.sh start
fi
shopt -u nocasematch
http://git-wip-us.apache.org/repos/asf/flink/blob/d61664ca/flink-dist/src/main/flink-bin/bin/start-local.sh
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/flink-bin/bin/start-local.sh b/flink-dist/src/main/flink-bin/bin/start-local.sh
deleted file mode 100755
index 2df3c22..0000000
--- a/flink-dist/src/main/flink-bin/bin/start-local.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-STREAMING_MODE=$1
-
-echo "Warning: this file is deprecated and will be removed in 1.5."
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-. "$bin"/config.sh
-
-"$FLINK_BIN_DIR"/start-cluster.sh
http://git-wip-us.apache.org/repos/asf/flink/blob/d61664ca/flink-dist/src/main/flink-bin/bin/stop-local.sh
----------------------------------------------------------------------
diff --git a/flink-dist/src/main/flink-bin/bin/stop-local.sh b/flink-dist/src/main/flink-bin/bin/stop-local.sh
deleted file mode 100755
index 8d079e0..0000000
--- a/flink-dist/src/main/flink-bin/bin/stop-local.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-echo "Warning: this file is deprecated and will be removed in 1.5."
-
-bin=`dirname "$0"`
-bin=`cd "$bin"; pwd`
-
-. "$bin"/config.sh
-
-"$FLINK_BIN_DIR"/stop-cluster.sh
http://git-wip-us.apache.org/repos/asf/flink/blob/d61664ca/test-infra/end-to-end-test/common.sh
----------------------------------------------------------------------
diff --git a/test-infra/end-to-end-test/common.sh b/test-infra/end-to-end-test/common.sh
index da8d22c..3c41e43 100644
--- a/test-infra/end-to-end-test/common.sh
+++ b/test-infra/end-to-end-test/common.sh
@@ -23,10 +23,6 @@ if [[ -z $FLINK_DIR ]]; then
echo "FLINK_DIR needs to point to a Flink distribution directory"
exit 1
fi
-if [[ -z $CLUSTER_MODE ]]; then
- echo "CLUSTER_MODE needs to be one of local or cluster."
- exit 1
-fi
export PASS=1
@@ -44,14 +40,7 @@ export TEST_DATA_DIR=$TEST_INFRA_DIR/temp-test-directory-$(date +%S%N)
echo "TEST_DATA_DIR: $TEST_DATA_DIR"
function start_cluster {
- if [[ "$CLUSTER_MODE" == "local" ]]; then
- $FLINK_DIR/bin/start-local.sh
- elif [[ "$CLUSTER_MODE" == "cluster" ]]; then
- $FLINK_DIR/bin/start-cluster.sh
- else
- echo "Unrecognized cluster mode: $CLUSTER_MODE"
- exit
- fi
+ "$FLINK_DIR"/bin/start-cluster.sh
# wait for TaskManager to come up
# wait roughly 10 seconds
@@ -71,11 +60,7 @@ function start_cluster {
}
function stop_cluster {
- if [[ "$CLUSTER_MODE" == "local" ]]; then
- $FLINK_DIR/bin/stop-local.sh
- elif [[ "$CLUSTER_MODE" == "cluster" ]]; then
- $FLINK_DIR/bin/stop-cluster.sh
- fi
+ "$FLINK_DIR"/bin/stop-cluster.sh
if grep -rv "GroupCoordinatorNotAvailableException" $FLINK_DIR/log \
| grep -v "RetriableCommitFailedException" \
http://git-wip-us.apache.org/repos/asf/flink/blob/d61664ca/tools/travis_mvn_watchdog.sh
----------------------------------------------------------------------
diff --git a/tools/travis_mvn_watchdog.sh b/tools/travis_mvn_watchdog.sh
index 82493ce..0ef74db 100755
--- a/tools/travis_mvn_watchdog.sh
+++ b/tools/travis_mvn_watchdog.sh
@@ -580,7 +580,7 @@ case $TEST in
printf "\n==============================================================================\n"
printf "Running Wordcount end-to-end test\n"
printf "==============================================================================\n"
- FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_batch_wordcount.sh
+ FLINK_DIR=build-target test-infra/end-to-end-test/test_batch_wordcount.sh
EXIT_CODE=$?
fi
@@ -588,7 +588,7 @@ case $TEST in
printf "\n==============================================================================\n"
printf "Running Kafka end-to-end test\n"
printf "==============================================================================\n"
- FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_streaming_kafka010.sh
+ FLINK_DIR=build-target test-infra/end-to-end-test/test_streaming_kafka010.sh
EXIT_CODE=$?
fi
@@ -596,7 +596,7 @@ case $TEST in
printf "\n==============================================================================\n"
printf "Running class loading end-to-end test\n"
printf "==============================================================================\n"
- FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_streaming_classloader.sh
+ FLINK_DIR=build-target test-infra/end-to-end-test/test_streaming_classloader.sh
EXIT_CODE=$?
fi
@@ -604,7 +604,7 @@ case $TEST in
printf "\n==============================================================================\n"
printf "Running Shaded Hadoop S3A end-to-end test\n"
printf "==============================================================================\n"
- FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_shaded_hadoop_s3a.sh
+ FLINK_DIR=build-target test-infra/end-to-end-test/test_shaded_hadoop_s3a.sh
EXIT_CODE=$?
fi
@@ -612,7 +612,7 @@ case $TEST in
printf "\n==============================================================================\n"
printf "Running Shaded Presto S3 end-to-end test\n"
printf "==============================================================================\n"
- FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_shaded_presto_s3.sh
+ FLINK_DIR=build-target test-infra/end-to-end-test/test_shaded_presto_s3.sh
EXIT_CODE=$?
fi
@@ -620,7 +620,7 @@ case $TEST in
printf "\n==============================================================================\n"
printf "Running Streaming Python Wordcount end-to-end test\n"
printf "==============================================================================\n"
- FLINK_DIR=build-target CLUSTER_MODE=cluster test-infra/end-to-end-test/test_streaming_python_wordcount.sh
+ FLINK_DIR=build-target test-infra/end-to-end-test/test_streaming_python_wordcount.sh
EXIT_CODE=$?
fi
else