You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gearpump.apache.org by ma...@apache.org on 2016/05/27 06:39:53 UTC
incubator-gearpump git commit: fix GEARPUMP-117 add scripts to start
and stop the cluster
Repository: incubator-gearpump
Updated Branches:
refs/heads/master a1b235252 -> e95a31707
fix GEARPUMP-117 add scripts to start and stop the cluster
Author: huafengw <fv...@gmail.com>
Closes #23 from huafengw/deploy.
Project: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/commit/e95a3170
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/tree/e95a3170
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gearpump/diff/e95a3170
Branch: refs/heads/master
Commit: e95a317078da9c6462b1dfcb8ac61482adf42b4f
Parents: a1b2352
Author: huafengw <fv...@gmail.com>
Authored: Fri May 27 14:39:37 2016 +0800
Committer: manuzhang <ow...@gmail.com>
Committed: Fri May 27 14:39:37 2016 +0800
----------------------------------------------------------------------
.gitignore | 1 -
bin/config.sh | 104 +++++++++++++++++++++++++++++
bin/gear-daemon.sh | 132 +++++++++++++++++++++++++++++++++++++
bin/start-cluster.sh | 50 ++++++++++++++
bin/stop-cluster.sh | 28 ++++++++
conf/dashboard | 1 +
conf/masters | 3 +
conf/slaves | 0
conf/workers | 3 +
docs/deployment-standalone.md | 11 +++-
project/Pack.scala | 1 +
11 files changed, 332 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/e95a3170/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 1efaafc..37071de 100644
--- a/.gitignore
+++ b/.gitignore
@@ -40,7 +40,6 @@ intellij
.project
.cache
.classpath
-bin/
tmp/
*.tmp
*.bak
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/e95a3170/bin/config.sh
----------------------------------------------------------------------
diff --git a/bin/config.sh b/bin/config.sh
new file mode 100644
index 0000000..885dbf0
--- /dev/null
+++ b/bin/config.sh
@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+
+GEARPUMP_PID_DIR="" # Directory to store *.pid files to
+DEFAULT_ENV_PID_DIR="/tmp" # Default directory to store *.pid files to
+
+GEARPUMP_LOG_DIR=""
+DEFAULT_ENV_LOG_DIR="/tmp/gear-logs"
+
+if [ "$GEARPUMP_PID_DIR" = "" ]; then
+ GEARPUMP_PID_DIR=${DEFAULT_ENV_PID_DIR}
+fi
+
+if [ "$GEARPUMP_LOG_DIR" = "" ]; then
+ GEARPUMP_LOG_DIR=${DEFAULT_ENV_LOG_DIR}
+fi
+
+bin=`dirname "$0"`
+SYMLINK_RESOLVED=`cd "$bin"; pwd -P`
+
+# Define the main directory of the gearpump installation
+export GEAR_ROOT_DIR=`dirname "$SYMLINK_RESOLVED"`
+export GEAR_BIN_DIR="$GEAR_ROOT_DIR/bin"
+export GEAR_CONF_DIR="$GEAR_ROOT_DIR/conf"
+
+readMasters() {
+ MASTERS_FILE="${GEAR_CONF_DIR}/masters"
+
+ if [[ ! -f "${MASTERS_FILE}" ]]; then
+ echo "No masters file. Please specify masters in 'conf/masters'."
+ exit 1
+ fi
+
+ MASTERS=()
+ MASTERPORTS=()
+
+ while IFS='' read -r line || [[ -n "$line" ]]; do
+ HOSTPORT=$( extractHostName $line)
+
+ if [ -n "$HOSTPORT" ]; then
+ HOST=$(echo $HOSTPORT | cut -f1 -d:)
+ PORT=$(echo $HOSTPORT | cut -s -f2 -d:)
+ MASTERS+=(${HOST})
+
+ if [ -z "$PORT" ]; then
+ MASTERPORTS+=(3000)
+ else
+ MASTERPORTS+=(${PORT})
+ fi
+ fi
+ done < "$MASTERS_FILE"
+}
+
+readWorkers() {
+ WORKERS_FILE="${GEAR_CONF_DIR}/workers"
+
+ if [[ ! -f "$WORKERS_FILE" ]]; then
+ echo "No workers file. Please specify workers in 'conf/workers'."
+ exit 1
+ fi
+
+ WORKERS=()
+
+ while IFS='' read -r line || [[ -n "$line" ]];
+ do
+ HOST=$( extractHostName $line)
+ if [ -n "$HOST" ]; then
+ WORKERS+=(${HOST})
+ fi
+ done < "$WORKERS_FILE"
+}
+
+readDashboard() {
+ DASHBOARD_FILE="${GEAR_CONF_DIR}/dashboard"
+
+ if [[ ! -f "$DASHBOARD_FILE" ]]; then
+ echo "No dashboard file. Please specify dashboard in 'conf/dashboard'."
+ return
+ fi
+
+ DASHBOARD=""
+
+ while IFS='' read -r line || [[ -n "$line" ]];
+ do
+ HOST=$( extractHostName $line)
+ if [ -n "$HOST" ]; then
+ DASHBOARD=${HOST}
+ break
+ fi
+ done < "$DASHBOARD_FILE"
+}
+
+# Auxilliary function which extracts the name of host from a line which
+# also potentially includes topology information and the taskManager type
+extractHostName() {
+ # handle comments: extract first part of string (before first # character)
+ HOST=`echo $1 | cut -d'#' -f 1`
+
+ # Extract the hostname from the network hierarchy
+ if [[ "$HOST" =~ ^.*/([0-9a-zA-Z.-]+)$ ]]; then
+ HOST=${BASH_REMATCH[1]}
+ fi
+
+ echo $HOST
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/e95a3170/bin/gear-daemon.sh
----------------------------------------------------------------------
diff --git a/bin/gear-daemon.sh b/bin/gear-daemon.sh
new file mode 100644
index 0000000..f7488cc
--- /dev/null
+++ b/bin/gear-daemon.sh
@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+
+USAGE="Usage: gear-daemon.sh (start|stop|stop-all) (local|master|worker|services) [args]"
+
+OPERATION=$1
+DAEMON=$2
+ARGS=("${@:3}") # get remaining arguments as array
+
+case $DAEMON in
+ (master)
+ BASH_TO_RUN=master
+ ;;
+
+ (worker)
+ BASH_TO_RUN=worker
+ ;;
+
+ (local)
+ BASH_TO_RUN=local
+ ;;
+
+ (services)
+ BASH_TO_RUN=services
+ ;;
+
+ (*)
+ echo "Unknown daemon '${DAEMON}'. $USAGE."
+ exit 1
+ ;;
+esac
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/config.sh
+
+if [ "$GEARPUMP_IDENT_STRING" = "" ]; then
+ GEARPUMP_IDENT_STRING="$USER"
+fi
+
+pid=$GEARPUMP_PID_DIR/gear-$GEARPUMP_IDENT_STRING-$DAEMON.pid
+
+mkdir -p "$GEARPUMP_PID_DIR"
+mkdir -p "$GEARPUMP_LOG_DIR"
+
+# Ascending ID depending on number of lines in pid file.
+# This allows us to start multiple daemon of each type.
+id=$([ -f "$pid" ] && echo $(wc -l < $pid) || echo "0")
+
+out="${GEARPUMP_LOG_DIR}/gearpump-${GEARPUMP_IDENT_STRING}-${DAEMON}-${id}-${HOSTNAME}.out"
+
+case $OPERATION in
+
+ (start)
+ # Print a warning if daemons are already running on host
+ if [ -f $pid ]; then
+ active=()
+ while IFS='' read -r p || [[ -n "$p" ]]; do
+ kill -0 $p >/dev/null 2>&1
+ if [ $? -eq 0 ]; then
+ active+=($p)
+ fi
+ done < "${pid}"
+
+ count="${#active[@]}"
+
+ if [ ${count} -gt 0 ]; then
+ echo "[INFO] $count instance(s) of $DAEMON are already running on $HOSTNAME."
+ fi
+ fi
+
+ echo "Starting $DAEMON daemon on host $HOSTNAME."
+ "$bin"/${BASH_TO_RUN} "${ARGS[@]}" > "$out" 2>&1 < /dev/null &
+
+ mypid=$!
+
+ # Add to pid file if successful start
+ if [[ ${mypid} =~ ${IS_NUMBER} ]] && kill -0 $mypid > /dev/null 2>&1 ; then
+ echo $mypid >> $pid
+ else
+ echo "Error starting $DAEMON daemon."
+ exit 1
+ fi
+ ;;
+
+ (stop)
+ if [ -f $pid ]; then
+ # Remove last in pid file
+ to_stop=$(tail -n 1 $pid)
+
+ if [ -z $to_stop ]; then
+ rm $pid # If all stopped, clean up pid file
+ echo "No $DAEMON daemon to stop on host $HOSTNAME."
+ else
+ sed \$d $pid > $pid.tmp # all but last line
+
+ # If all stopped, clean up pid file
+ [ $(wc -l < $pid.tmp) -eq 0 ] && rm $pid $pid.tmp || mv $pid.tmp $pid
+
+ if kill -0 $to_stop > /dev/null 2>&1; then
+ echo "Stopping $DAEMON daemon (pid: $to_stop) on host $HOSTNAME."
+ kill $to_stop
+ else
+ echo "No $DAEMON daemon (pid: $to_stop) is running anymore on $HOSTNAME."
+ fi
+ fi
+ else
+ echo "No $DAEMON daemon to stop on host $HOSTNAME."
+ fi
+ ;;
+
+ (stop-all)
+ if [ -f $pid ]; then
+ mv $pid ${pid}.tmp
+
+ while read to_stop; do
+ if kill -0 $to_stop > /dev/null 2>&1; then
+ echo "Stopping $DAEMON daemon (pid: $to_stop) on host $HOSTNAME."
+ kill $to_stop
+ else
+ echo "Skipping $DAEMON daemon (pid: $to_stop), because it is not running anymore on $HOSTNAME."
+ fi
+ done < ${pid}.tmp
+ rm ${pid}.tmp
+ fi
+ ;;
+
+ (*)
+ echo "Unexpected argument '$STARTSTOP'. $USAGE."
+ exit 1
+ ;;
+esac
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/e95a3170/bin/start-cluster.sh
----------------------------------------------------------------------
diff --git a/bin/start-cluster.sh b/bin/start-cluster.sh
new file mode 100644
index 0000000..a40a916
--- /dev/null
+++ b/bin/start-cluster.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/config.sh
+
+# Start Masters
+readMasters
+
+MASTER_CONFIG=""
+
+# Init master configuration
+for((i=0;i<${#MASTERS[@]};++i)); do
+ MASTER_CONFIG+="-Dgearpump.cluster.masters.${i}=${MASTERS[i]}:${MASTERPORTS[i]} "
+done
+
+# Start masters
+for((i=0;i<${#MASTERS[@]};++i)); do
+ master=${MASTERS[i]}
+ port=${MASTERPORTS[i]}
+ ssh $master "export JAVA_OPTS='$JAVA_OPTS ${MASTER_CONFIG} -Dgearpump.hostname=${master}';\\
+ cd $GEAR_ROOT_DIR; $GEAR_BIN_DIR/gear-daemon.sh start master -ip $master -port $port"
+done
+
+# Wait for all Master start
+sleep 2
+
+# Start dashboard
+
+readDashboard
+
+if [ -n "$DASHBOARD" ]; then
+ HOST=$(echo $DASHBOARD | cut -f1 -d:)
+ PORT=$(echo $DASHBOARD | cut -s -f2 -d:)
+ ssh $HOST "export JAVA_OPTS='$JAVA_OPTS ${MASTER_CONFIG} -Dgearpump.hostname=${HOST} -Dgearpump.services.host=${HOST} -Dgearpump.services.port=${PORT}';\\
+ cd $GEAR_ROOT_DIR; $GEAR_BIN_DIR/gear-daemon.sh start services"
+else
+ echo "No dashboard specified, please check file conf/dashboard"
+fi
+
+
+# Start workers
+
+readWorkers
+
+for worker in ${WORKERS[@]}; do
+ ssh $worker "export JAVA_OPTS='$JAVA_OPTS ${MASTER_CONFIG} -Dgearpump.hostname=${worker}';\\
+ cd $GEAR_ROOT_DIR; $GEAR_BIN_DIR/gear-daemon.sh start worker"
+done
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/e95a3170/bin/stop-cluster.sh
----------------------------------------------------------------------
diff --git a/bin/stop-cluster.sh b/bin/stop-cluster.sh
new file mode 100644
index 0000000..e0cb45e
--- /dev/null
+++ b/bin/stop-cluster.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+
+. "$bin"/config.sh
+
+# Stop workers
+readWorkers
+
+for worker in ${WORKERS[@]}; do
+ ssh $worker "$GEAR_BIN_DIR/gear-daemon.sh stop-all worker"
+done
+
+# Stop dashboard
+readDashboard
+
+if [ -n "$DASHBOARD" ]; then
+ HOST=$(echo $DASHBOARD | cut -f1 -d:)
+ ssh $HOST "$GEAR_BIN_DIR/gear-daemon.sh stop-all services"
+fi
+
+# Stop Masters
+readMasters
+
+for master in ${MASTERS[@]}; do
+ ssh $master "$GEAR_BIN_DIR/gear-daemon.sh stop-all master"
+done
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/e95a3170/conf/dashboard
----------------------------------------------------------------------
diff --git a/conf/dashboard b/conf/dashboard
new file mode 100644
index 0000000..03a8679
--- /dev/null
+++ b/conf/dashboard
@@ -0,0 +1 @@
+#127.0.0.1:8090
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/e95a3170/conf/masters
----------------------------------------------------------------------
diff --git a/conf/masters b/conf/masters
index e69de29..d93bdd6 100644
--- a/conf/masters
+++ b/conf/masters
@@ -0,0 +1,3 @@
+127.0.0.1:3000
+#node2:3000
+#node3:3000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/e95a3170/conf/slaves
----------------------------------------------------------------------
diff --git a/conf/slaves b/conf/slaves
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/e95a3170/conf/workers
----------------------------------------------------------------------
diff --git a/conf/workers b/conf/workers
new file mode 100644
index 0000000..05bd9ef
--- /dev/null
+++ b/conf/workers
@@ -0,0 +1,3 @@
+127.0.0.1
+#node2
+#node3
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/e95a3170/docs/deployment-standalone.md
----------------------------------------------------------------------
diff --git a/docs/deployment-standalone.md b/docs/deployment-standalone.md
index d77ffc8..f171354 100644
--- a/docs/deployment-standalone.md
+++ b/docs/deployment-standalone.md
@@ -18,7 +18,7 @@ To install Gearpump, you at least need to change the configuration in `conf/gear
Config | Default value | Description
------------ | ---------------|------------
gearpump.hostname | "127.0.0.1" | Host or IP address of current machine. The ip/host need to be reachable from other machines in the cluster.
-Gearpump.cluster.masters | ["127.0.0.1:3000"] | List of all master nodes, with each item represents host and port of one master.
+gearpump.cluster.masters | ["127.0.0.1:3000"] | List of all master nodes, with each item represents host and port of one master.
gearpump.worker.slots | 1000 | how many slots this worker has
Besides this, there are other optional configurations related with logs, metrics, transports, ui. You can refer to [Configuration Guide](deployment-configuration.html) for more details.
@@ -55,3 +55,12 @@ The default username and password is "admin:admin", you can check
**NOTE:** The UI port can be configured in `gear.conf`. Check [Configuration Guide](deployment-configuration.html) for information.
+
+### Bash tool to start cluster
+
+There is a bash tool `bin/start-cluster.sh` can launch the cluster conveniently. You need to change the file `conf/masters`, `conf/workers` and `conf/dashboard` to specify the corresponding machines.
+Before running the bash tool, please make sure the Gearpump package is already unzipped to the same directory path on every machine.
+`bin/stop-cluster.sh` is used to stop the whole cluster of course.
+
+The bash tool is able to launch the cluster without changing the `conf/gear.conf` on every machine. The bash sets the `gearpump.cluster.masters` and other configurations using JAVA_OPTS.
+However, please note when you log into any these unconfigured machine and try to launch the dashboard or submit the application, you still need to modify `conf/gear.conf` manually because the JAVA_OPTS is missing.
http://git-wip-us.apache.org/repos/asf/incubator-gearpump/blob/e95a3170/project/Pack.scala
----------------------------------------------------------------------
diff --git a/project/Pack.scala b/project/Pack.scala
index a675bdf..570610b 100644
--- a/project/Pack.scala
+++ b/project/Pack.scala
@@ -119,6 +119,7 @@ object Pack extends sbt.Build {
),
packExclude := Seq(thisProjectRef.value.project),
+ packResourceDir += (baseDirectory.value / ".." / "bin" -> "bin"),
packResourceDir += (baseDirectory.value / ".." / "conf" -> "conf"),
packResourceDir += (baseDirectory.value / ".." / "yarnconf" -> "conf/yarnconf"),
packResourceDir += (baseDirectory.value / ".." / "unmanagedlibs" /