You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by le...@apache.org on 2021/10/14 06:21:42 UTC
[dolphinscheduler] branch 2.0-prepare updated: Fix install.sh for
DS 2.0 and add comment to install_config.conf (#6506) (#6525)
This is an automated email from the ASF dual-hosted git repository.
leonbao pushed a commit to branch 2.0-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/2.0-prepare by this push:
new 5f19d5b Fix install.sh for DS 2.0 and add comment to install_config.conf (#6506) (#6525)
5f19d5b is described below
commit 5f19d5be41e66919b7006cc4667cdd7ba604fd05
Author: Jiajie Zhong <zh...@hotmail.com>
AuthorDate: Thu Oct 14 14:21:35 2021 +0800
Fix install.sh for DS 2.0 and add comment to install_config.conf (#6506) (#6525)
* Fix install.sh for DS 2.0
* Use #* make it work in macos
* Add comment for properties change
* Add JAVA HOME for only one file change to deploy
* Reduce config change by using default value of: plugin registry, alert, task as
* Change javaHome default value
* Delete config param zkQuorum
* Reorder param
* Add comment for install_config.conf
---
.../src/main/resources/config/install_config.conf | 148 ++++++++++++---------
install.sh | 29 ++--
script/dolphinscheduler-daemon.sh | 1 +
script/remove-zk-node.sh | 3 +-
script/scp-hosts.sh | 3 +-
5 files changed, 111 insertions(+), 73 deletions(-)
diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf
index 9706897..73282ac 100755
--- a/dolphinscheduler-server/src/main/resources/config/install_config.conf
+++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf
@@ -15,54 +15,109 @@
# limitations under the License.
#
+# ---------------------------------------------------------
+# INSTALL MACHINE
+# ---------------------------------------------------------
+# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
+# including master, worker, api, alert. If you want to deploy in pseudo-distributed
+# mode, just write a pseudo-distributed hostname
+# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IP: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
+ips="ds1,ds2,ds3,ds4,ds5"
+
+# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
+# modify it if you use different ssh port
+sshPort="22"
+
+# A comma separated list of machine hostname or IP would be installed Master server, it
+# must be a subset of configuration `ips`.
+# Example for hostnames: ips="ds1,ds2", Example for IP: ips="192.168.8.1,192.168.8.2"
+masters="ds1,ds2"
+
+# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
+# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
+# Example for hostnames: ips="ds1:default,ds2:default,ds3:default", Example for IP: ips="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
+workers="ds1:default,ds2:default,ds3:default,ds4:default,ds5:default"
+
+# A comma separated list of machine hostname or IP would be installed Alert server, it
+# must be a subset of configuration `ips`.
+# Example for hostnames: ips="ds3", Example for IP: ips="192.168.8.3"
+alertServer="ds3"
+
+# A comma separated list of machine hostname or IP would be installed API server, it
+# must be a subset of configuration `ips`.
+# Example for hostnames: ips="ds1", Example for IP: ips="192.168.8.1"
+apiServers="ds1"
+
+# The directory to install DolphinScheduler for all machine we config above. It will automatically created by `install.sh` script if not exists.
+# **DO NOT** set this configuration same as the current path (pwd)
+installPath="/data1_1T/dolphinscheduler"
+
+# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before run `install.sh`
+# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
+# to be created by this user
+deployUser="dolphinscheduler"
+
+# The directory to store local data for all machine we config above. Make sure user `deployUser` have permissions to read and write this directory.
+dataBasedirPath="/tmp/dolphinscheduler"
+
+# ---------------------------------------------------------
+# DolphinScheduler ENV
+# ---------------------------------------------------------
+# JAVA_HOME, we recommend use same JAVA_HOME in all machine you going to install DolphinScheduler
+# and this configuration only support one parameter so far.
+javaHome="/your/java/home/here"
-# NOTICE: If the following config has special characters in the variable `.*[]^${}\+?|()@#&`, Please escape, for example, `[` escape to `\[`
-# postgresql or mysql
+# DolphinScheduler API service port, also this your DolphinScheduler UI component's URL port, default values is 12345
+apiServerPort="12345"
+
+# ---------------------------------------------------------
+# Database
+# NOTICE: If database value has special characters, such as `.*[]^${}\+?|()@#&`, Please add prefix `\` for escaping.
+# ---------------------------------------------------------
+# The type for the metadata database
+# Supported values: ``postgresql``, ``mysql``.
dbtype="mysql"
-# db config
-# db address and port
+# The <HOST>:<PORT> connection pair DolphinScheduler connect to the metadata database
dbhost="192.168.xx.xx:3306"
-# db username
+# The username DolphinScheduler connect to the metadata database
username="xx"
-# db password
-# NOTICE: if there are special characters, please use the \ to escape, for example, `[` escape to `\[`
+# The password DolphinScheduler connect to the metadata database
password="xx"
-# database name
+# The database DolphinScheduler connect to the metadata database
dbname="dolphinscheduler"
+# ---------------------------------------------------------
+# Registry Server
+# ---------------------------------------------------------
+# Registry Server plugin dir. DolphinScheduler will find and load the registry plugin jar package from this dir.
+# For now default registry server is zookeeper, so the default value is `lib/plugin/registry/zookeeper`.
+# If you want to implement your own registry server, please see https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/registry_spi.html
+registryPluginDir="lib/plugin/registry/zookeeper"
-# zk cluster
-zkQuorum="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
-
-# zk root directory
-zkRoot="/dolphinscheduler"
-
-# registry config
-# registry plugin dir
-# Note: find and load the Registry Plugin Jar from this dir.
-registryPluginDir="/data1_1T/dolphinscheduler/lib/plugin/registry"
+# Registry Server plugin name, should be a substring of `registryPluginDir`, DolphinScheduler use this for verifying configuration consistency
registryPluginName="zookeeper"
-registryServers="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
+# Registry Server address.
+registryServers="192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181"
-# Note: the target installation path for dolphinscheduler, please not config as the same as the current path (pwd)
-installPath="/data1_1T/dolphinscheduler"
+# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
+zkRoot="/dolphinscheduler"
-# deployment user
-# Note: the deployment user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled, the root directory needs to be created by itself
-deployUser="dolphinscheduler"
+# ---------------------------------------------------------
+# Alert Server
+# ---------------------------------------------------------
+# Alert Server plugin dir. DolphinScheduler will find and load the alert plugin jar package from this dir.
+alertPluginDir="lib/plugin/alert"
-# alert config
-# alert plugin dir
-# Note: find and load the Alert Plugin Jar from this dir.
-alertPluginDir="/data1_1T/dolphinscheduler/lib/plugin/alert"
-
-# user data local directory path, please make sure the directory exists and have read write permissions
-dataBasedirPath="/tmp/dolphinscheduler"
+# ---------------------------------------------------------
+# Worker Task Server
+# ---------------------------------------------------------
+# Worker Task Server plugin dir. DolphinScheduler will find and load the worker task plugin jar package from this dir.
+taskPluginDir="lib/plugin/task"
# resource storage type: HDFS, S3, NONE
resourceStorageType="NONE"
@@ -110,34 +165,3 @@ sudoEnable="true"
# worker tenant auto create
workerTenantAutoCreate="false"
-
-# worker task plugin dir
-taskPluginDir="/data1_1T/dolphinscheduler/lib/plugin/task"
-
-# api server port
-apiServerPort="12345"
-
-
-# install hosts
-# Note: install the scheduled hostname list. If it is pseudo-distributed, just write a pseudo-distributed hostname
-ips="ds1,ds2,ds3,ds4,ds5"
-
-# ssh port, default 22
-# Note: if ssh port is not default, modify here
-sshPort="22"
-
-# run master machine
-# Note: list of hosts hostname for deploying master
-masters="ds1,ds2"
-
-# run worker machine
-# note: need to write the worker group name of each worker, the default value is "default"
-workers="ds1:default,ds2:default,ds3:default,ds4:default,ds5:default"
-
-# run alert machine
-# note: list of machine hostnames for deploying alert server
-alertServer="ds3"
-
-# run api machine
-# note: list of machine hostnames for deploying api server
-apiServers="ds1"
\ No newline at end of file
diff --git a/install.sh b/install.sh
index 25e5d91..c25d852 100755
--- a/install.sh
+++ b/install.sh
@@ -34,11 +34,17 @@ datasourceDriverClassname="com.mysql.jdbc.Driver"
if [ $dbtype == "postgresql" ];then
datasourceDriverClassname="org.postgresql.Driver"
fi
+
+# Change configuration in conf/config/dolphinscheduler_env.sh
+sed -i ${txt} "s@^export JAVA_HOME=.*@export JAVA_HOME=${javaHome}@g" conf/env/dolphinscheduler_env.sh
+
+# Change configuration in conf/datasource.properties
sed -i ${txt} "s@^spring.datasource.driver-class-name=.*@spring.datasource.driver-class-name=${datasourceDriverClassname}@g" conf/datasource.properties
sed -i ${txt} "s@^spring.datasource.url=.*@spring.datasource.url=jdbc:${dbtype}://${dbhost}/${dbname}?characterEncoding=UTF-8\&allowMultiQueries=true@g" conf/datasource.properties
sed -i ${txt} "s@^spring.datasource.username=.*@spring.datasource.username=${username}@g" conf/datasource.properties
sed -i ${txt} "s@^spring.datasource.password=.*@spring.datasource.password=${password}@g" conf/datasource.properties
+# Change configuration in conf/common.properties
sed -i ${txt} "s@^data.basedir.path=.*@data.basedir.path=${dataBasedirPath}@g" conf/common.properties
sed -i ${txt} "s@^resource.storage.type=.*@resource.storage.type=${resourceStorageType}@g" conf/common.properties
sed -i ${txt} "s@^resource.upload.path=.*@resource.upload.path=${resourceUploadPath}@g" conf/common.properties
@@ -58,17 +64,22 @@ sed -i ${txt} "s@^yarn.application.status.address=.*@yarn.application.status.add
sed -i ${txt} "s@^yarn.job.history.status.address=.*@yarn.job.history.status.address=http://${singleYarnIp}:19888/ws/v1/history/mapreduce/jobs/%s@g" conf/common.properties
sed -i ${txt} "s@^sudo.enable=.*@sudo.enable=${sudoEnable}@g" conf/common.properties
-# the following configurations may be commented, so ddd #\? to ensure successful sed
-sed -i ${txt} "s@^#\?worker.tenant.auto.create=.*@worker.tenant.auto.create=${workerTenantAutoCreate}@g" conf/worker.properties
-sed -i ${txt} "s@^#\?alert.listen.host=.*@alert.listen.host=${alertServer}@g" conf/worker.properties
-sed -i ${txt} "s@^#\?task.plugin.dir=.*@task.plugin.dir=${taskPluginDir}@g" conf/worker.properties
+# The following configurations may be commented, so ddd #* to ensure sed work correct
+# Change configuration in conf/worker.properties
+sed -i ${txt} "s@^#*worker.tenant.auto.create=.*@worker.tenant.auto.create=${workerTenantAutoCreate}@g" conf/worker.properties
+sed -i ${txt} "s@^#*alert.listen.host=.*@alert.listen.host=${alertServer}@g" conf/worker.properties
+sed -i ${txt} "s@^#*task.plugin.dir=.*@task.plugin.dir=${installPath}/${taskPluginDir}@g" conf/worker.properties
+
+# Change configuration in conf/alert.properties
+sed -i ${txt} "s@^#*alert.plugin.dir=.*@alert.plugin.dir=${installPath}/${alertPluginDir}@g" conf/alert.properties
-sed -i ${txt} "s@^#\?alert.plugin.dir=.*@alert.plugin.dir=${alertPluginDir}@g" conf/alert.properties
-sed -i ${txt} "s@^#\?server.port=.*@server.port=${apiServerPort}@g" conf/application-api.properties
+# Change configuration in conf/application-api.properties
+sed -i ${txt} "s@^#*server.port=.*@server.port=${apiServerPort}@g" conf/application-api.properties
-sed -i ${txt} "s@^#\?registry.plugin.dir=.*@registry.plugin.dir=${registryPluginDir}@g" conf/registry.properties
-sed -i ${txt} "s@^#\?registry.plugin.name=.*@registry.plugin.name=${registryPluginName}@g" conf/registry.properties
-sed -i ${txt} "s@^#\?registry.servers=.*@registry.servers=${registryServers}@g" conf/registry.properties
+# Change configuration in conf/registry.properties
+sed -i ${txt} "s@^#*registry.plugin.dir=.*@registry.plugin.dir=${installPath}/${registryPluginDir}@g" conf/registry.properties
+sed -i ${txt} "s@^#*registry.plugin.name=.*@registry.plugin.name=${registryPluginName}@g" conf/registry.properties
+sed -i ${txt} "s@^#*registry.servers=.*@registry.servers=${registryServers}@g" conf/registry.properties
# 2.create directory
echo "2.create directory"
diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh
index 81af5fd..4de812b 100755
--- a/script/dolphinscheduler-daemon.sh
+++ b/script/dolphinscheduler-daemon.sh
@@ -36,6 +36,7 @@ BIN_DIR=`cd "$BIN_DIR"; pwd`
DOLPHINSCHEDULER_HOME=`cd "$BIN_DIR/.."; pwd`
source /etc/profile
+source "${DOLPHINSCHEDULER_HOME}/conf/env/dolphinscheduler_env.sh"
export JAVA_HOME=$JAVA_HOME
#export JAVA_HOME=/opt/soft/jdk
diff --git a/script/remove-zk-node.sh b/script/remove-zk-node.sh
index 1129ca6..2cb8a2b 100755
--- a/script/remove-zk-node.sh
+++ b/script/remove-zk-node.sh
@@ -32,6 +32,7 @@ BIN_DIR=`cd "$BIN_DIR"; pwd`
DOLPHINSCHEDULER_HOME=$BIN_DIR/..
source ${BIN_DIR}/../conf/config/install_config.conf
+source ${BIN_DIR}/../conf/env/dolphinscheduler_env.sh
export JAVA_HOME=$JAVA_HOME
@@ -44,7 +45,7 @@ export STOP_TIMEOUT=5
CLASS=org.apache.zookeeper.ZooKeeperMain
-exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS -server $zkQuorum rmr $rootNode"
+exec_command="$DOLPHINSCHEDULER_OPTS -classpath $DOLPHINSCHEDULER_CONF_DIR:$DOLPHINSCHEDULER_LIB_JARS $CLASS -server $registryServers rmr $rootNode"
cd $DOLPHINSCHEDULER_HOME
$JAVA_HOME/bin/java $exec_command
diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh
index bdcdc73..8def4d1 100755
--- a/script/scp-hosts.sh
+++ b/script/scp-hosts.sh
@@ -57,7 +57,8 @@ do
fi
echo "start to scp $dsDir to $host/$installPath"
- scp -P $sshPort -r $workDir/../$dsDir $host:$installPath
+ # Use quiet mode to reduce command line output
+ scp -q -P $sshPort -r $workDir/../$dsDir $host:$installPath
done
echo "scp dirs to $host/$installPath complete"