You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by jo...@apache.org on 2020/05/12 08:49:50 UTC
[incubator-dolphinscheduler] branch dev-1.3.0 updated: add worker
group config (#2681)
This is an automated email from the ASF dual-hosted git repository.
journey pushed a commit to branch dev-1.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev-1.3.0 by this push:
new 06c4ddf add worker group config (#2681)
06c4ddf is described below
commit 06c4ddf8e36ca79b45484281e09a0bf9e0e1b969
Author: dailidong <da...@gmail.com>
AuthorDate: Tue May 12 16:49:40 2020 +0800
add worker group config (#2681)
* update README about DolphinScheduler
* Update issue templates
* update
* regularize api pom xml
update rpc maven compile to 1.8
* regularize api pom xml
* change commons.lang3.StringUtils to common.utils.StringUtils
* update pom.xml
* update
* correct equals method
* jasper-runtime is needed when api server start
* jasper-runtime jar is needed when api server start
* combine logback config of master/worker/alert/api server to one logback.xml
* remove tomcat runtime jar
* add UT
* add license
* remove jasper-runtime jar, not need anymore
* sovle jar conflict, remove servlet-api 2.5 jar
* remove servlet 2.5
* add embedded database h2 and update embedded zookeeper
* delete logs
* add license
* change log level to INFO
* simplify and optimize config
* add development state
* simplify common config
* add worker group config
* add worker group config
Co-authored-by: lidongdai <es...@outlook.com>
Co-authored-by: dailidong <do...@gmail.com>
---
.../src/main/resources/config/install_config.conf | 12 ++++++------
dolphinscheduler-server/src/main/resources/worker.properties | 4 ++--
script/scp-hosts.sh | 11 +++++++++++
script/start-all.sh | 3 +--
script/stop-all.sh | 3 +--
5 files changed, 21 insertions(+), 12 deletions(-)
diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf
index fab6b32..f6b09b9 100644
--- a/dolphinscheduler-server/src/main/resources/config/install_config.conf
+++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf
@@ -118,7 +118,7 @@ apiServerPort="12345"
# install hosts
# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
-ips="ark0,ark1,ark2,ark3,ark4"
+ips="ds1,ds2,ds3,ds4,ds5"
# ssh port, default 22
# Note: if ssh port is not default, modify here
@@ -126,19 +126,19 @@ sshPort="22"
# run master machine
# Note: list of hosts hostname for deploying master
-masters="ark0,ark1"
+masters="ds1,ds2"
# run worker machine
-# note: list of machine hostnames for deploying workers
-workers="ark2,ark3,ark4"
+# note: need to write the worker group name of each worker, the default value is "default"
+workersGroup=(["ds1"]="default" ["ds2"]="default" ["ds3"]="default" ["ds4"]="default" ["ds5"]="default")
# run alert machine
# note: list of machine hostnames for deploying alert server
-alertServer="ark3"
+alertServer="ds3"
# run api machine
# note: list of machine hostnames for deploying api server
-apiServers="ark1"
+apiServers="ds1"
# whether to start monitoring self-starting scripts
monitorServerState="false"
diff --git a/dolphinscheduler-server/src/main/resources/worker.properties b/dolphinscheduler-server/src/main/resources/worker.properties
index d078f26..36bc132 100644
--- a/dolphinscheduler-server/src/main/resources/worker.properties
+++ b/dolphinscheduler-server/src/main/resources/worker.properties
@@ -21,7 +21,7 @@
# worker heartbeat interval
#worker.heartbeat.interval=10
-# submit the number of tasks at a time
+# submit the number of tasks at a time TODO
#worker.fetch.task.num = 3
# only less than cpu avg load, worker server can work. default value : the number of cpu cores * 2
@@ -34,4 +34,4 @@
#worker.listen.port: 1234
# default worker group
-#worker.group=default
\ No newline at end of file
+worker.group=default
\ No newline at end of file
diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh
index f4949f3..adee7d7 100644
--- a/script/scp-hosts.sh
+++ b/script/scp-hosts.sh
@@ -20,6 +20,12 @@ workDir=`dirname $0`
workDir=`cd ${workDir};pwd`
source $workDir/../conf/config/install_config.conf
+txt=""
+if [[ "$OSTYPE" == "darwin"* ]]; then
+ # Mac OSX
+ txt="''"
+fi
+
hostsArr=(${ips//,/ })
for host in ${hostsArr[@]}
do
@@ -33,6 +39,11 @@ do
for dsDir in bin conf lib script sql ui install.sh
do
+ # if worker in workersGroup
+ if [[ "${map[${host}]}" ]] && [[ "${dsDir}" -eq "conf" ]]; then
+ sed -i ${txt} "s#worker.group.*#worker.group=${map[${host}]}#g" $workDir/../conf/worker.properties
+ fi
+
echo "start to scp $dsDir to $host/$installPath"
scp -P $sshPort -r $workDir/../$dsDir $host:$installPath
done
diff --git a/script/start-all.sh b/script/start-all.sh
index 11e4572..5579a7d 100644
--- a/script/start-all.sh
+++ b/script/start-all.sh
@@ -28,8 +28,7 @@ do
done
-workersHost=(${workers//,/ })
-for worker in ${workersHost[@]}
+for worker in ${!workersGroup[*]}
do
echo "$worker worker server is starting"
diff --git a/script/stop-all.sh b/script/stop-all.sh
index f761579..e4ccf75 100644
--- a/script/stop-all.sh
+++ b/script/stop-all.sh
@@ -29,8 +29,7 @@ do
done
-workersHost=(${workers//,/ })
-for worker in ${workersHost[@]}
+for worker in ${!workersGroup[*]}
do
echo "$worker worker server is stopping"
ssh -p $sshPort $worker "cd $installPath/; sh bin/dolphinscheduler-daemon.sh stop worker-server;"