You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zh...@apache.org on 2022/05/02 15:18:19 UTC
[dolphinscheduler] branch 3.0.0-alpha-release updated: [cherry-pick][doc] Update 3.0.0-alpha docs from dev (#9865)
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.0.0-alpha-release
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/3.0.0-alpha-release by this push:
new 0453a52e40 [cherry-pick][doc] Update 3.0.0-alpha docs from dev (#9865)
0453a52e40 is described below
commit 0453a52e40533a595fbc253e3b8fd7d3199be4cf
Author: Jiajie Zhong <zh...@hotmail.com>
AuthorDate: Mon May 2 23:18:14 2022 +0800
[cherry-pick][doc] Update 3.0.0-alpha docs from dev (#9865)
---
docs/configs/docsdev.js | 24 +
docs/docs/en/architecture/configuration.md | 48 +-
docs/docs/en/faq.md | 35 +-
.../docs/en/guide/alert/alert_plugin_user_guide.md | 8 +-
docs/docs/en/guide/alert/enterprise-wechat.md | 65 +-
docs/docs/en/guide/alert/http.md | 34 +
docs/docs/en/guide/alert/script.md | 17 +
docs/docs/en/guide/data-quality.md | 310 ++++++
docs/docs/en/guide/datasource/hive.md | 4 +
docs/docs/en/guide/datasource/introduction.md | 22 +-
docs/docs/en/guide/datasource/mysql.md | 4 +
docs/docs/en/guide/datasource/postgresql.md | 4 +
docs/docs/en/guide/datasource/spark.md | 4 +
docs/docs/en/guide/expansion-reduction.md | 4 +-
docs/docs/en/guide/howto/general-setting.md | 14 +-
docs/docs/en/guide/installation/cluster.md | 6 +-
docs/docs/en/guide/installation/pseudo-cluster.md | 127 ++-
docs/docs/en/guide/installation/standalone.md | 20 +-
docs/docs/en/guide/start/docker.md | 1101 ++------------------
docs/docs/en/guide/task/datax.md | 2 +-
docs/docs/en/guide/task/flink.md | 4 +-
docs/docs/en/guide/task/map-reduce.md | 2 +-
docs/docs/en/guide/task/shell.md | 5 +
docs/docs/en/guide/task/spark.md | 2 +-
docs/docs/en/guide/upgrade.md | 24 +-
docs/docs/zh/architecture/configuration.md | 49 +-
docs/docs/zh/faq.md | 41 +-
.../docs/zh/guide/alert/alert_plugin_user_guide.md | 8 +-
docs/docs/zh/guide/alert/enterprise-wechat.md | 64 +-
docs/docs/zh/guide/alert/http.md | 34 +
docs/docs/zh/guide/alert/script.md | 17 +
docs/docs/zh/guide/data-quality.md | 313 ++++++
docs/docs/zh/guide/datasource/hive.md | 3 +
docs/docs/zh/guide/datasource/introduction.md | 19 +-
docs/docs/zh/guide/datasource/mysql.md | 4 +
docs/docs/zh/guide/datasource/postgresql.md | 4 +
docs/docs/zh/guide/datasource/spark.md | 4 +
docs/docs/zh/guide/expansion-reduction.md | 2 +-
docs/docs/zh/guide/howto/general-setting.md | 14 +-
docs/docs/zh/guide/installation/cluster.md | 4 +-
docs/docs/zh/guide/installation/pseudo-cluster.md | 132 ++-
docs/docs/zh/guide/installation/standalone.md | 20 +-
docs/docs/zh/guide/start/docker.md | 1091 ++-----------------
docs/docs/zh/guide/task/datax.md | 2 +-
docs/docs/zh/guide/task/flink.md | 2 +-
docs/docs/zh/guide/task/map-reduce.md | 2 +-
docs/docs/zh/guide/task/shell.md | 2 +-
docs/docs/zh/guide/task/spark.md | 2 +-
docs/docs/zh/guide/upgrade.md | 25 +-
docs/img/alert/alert_scenarios_en.png | Bin 0 -> 33920 bytes
docs/img/alert/alert_scenarios_zh.png | Bin 0 -> 36607 bytes
docs/img/alert/alert_step_2.png | Bin 34429 -> 159166 bytes
docs/img/alert/alert_step_3.png | Bin 44253 -> 240057 bytes
docs/img/alert/alert_step_4.png | Bin 78671 -> 144842 bytes
docs/img/alert/enterprise-wechat-app-msg-md.png | Bin 0 -> 54994 bytes
docs/img/alert/enterprise-wechat-app-msg.png | Bin 0 -> 18009 bytes
docs/img/alert/enterprise-wechat-create-group.png | Bin 0 -> 32570 bytes
docs/img/alert/enterprise-wechat-group-msg-md.png | Bin 0 -> 52086 bytes
docs/img/alert/enterprise-wechat-group-msg.png | Bin 0 -> 43359 bytes
docs/img/alert/enterprise-wechat-query-userid.png | Bin 0 -> 18747 bytes
docs/img/alert/http-get-example.png | Bin 0 -> 15623 bytes
docs/img/alert/http-post-example.png | Bin 0 -> 17889 bytes
docs/img/alert/script-plugin.png | Bin 0 -> 12752 bytes
docs/img/alert/wechat-app-form-example.png | Bin 0 -> 22056 bytes
docs/img/alert/wechat-group-form-example.png | Bin 0 -> 23285 bytes
docs/img/tasks/demo/custom_sql_check.png | Bin 0 -> 20259 bytes
docs/img/tasks/demo/enumeration_check.png | Bin 0 -> 20145 bytes
docs/img/tasks/demo/field_length_check.png | Bin 0 -> 18951 bytes
docs/img/tasks/demo/multi_table_accuracy_check.png | Bin 0 -> 21355 bytes
.../tasks/demo/multi_table_comparison_check.png | Bin 0 -> 20590 bytes
docs/img/tasks/demo/null_check.png | Bin 0 -> 21491 bytes
docs/img/tasks/demo/regexp_check.png | Bin 0 -> 20379 bytes
docs/img/tasks/demo/result.png | Bin 0 -> 63652 bytes
docs/img/tasks/demo/rule_detail.png | Bin 0 -> 74110 bytes
docs/img/tasks/demo/rule_list.png | Bin 0 -> 55713 bytes
docs/img/tasks/demo/shell.jpg | Bin 632738 -> 436873 bytes
docs/img/tasks/demo/shell_custom_param.jpg | Bin 556773 -> 404207 bytes
docs/img/tasks/demo/table_count_check.png | Bin 0 -> 20701 bytes
docs/img/tasks/demo/timeliness_check.png | Bin 0 -> 21350 bytes
docs/img/tasks/demo/uniqueness_check.png | Bin 0 -> 20438 bytes
80 files changed, 1562 insertions(+), 2186 deletions(-)
diff --git a/docs/configs/docsdev.js b/docs/configs/docsdev.js
index 5a887314c3..9240175028 100644
--- a/docs/configs/docsdev.js
+++ b/docs/configs/docsdev.js
@@ -241,8 +241,20 @@ export default {
title: 'Enterprise Webexteams',
link: '/en-us/docs/3.0.0/user_doc/guide/alert/enterprise-webexteams.html',
},
+ {
+ title: 'Script',
+ link: '/en-us/docs/3.0.0/user_doc/guide/alert/script.html',
+ },
+ {
+ title: 'Http',
+ link: '/en-us/docs/3.0.0/user_doc/guide/alert/http.html',
+ },
],
},
+ {
+ title: 'Data Quality',
+ link: '/en-us/docs/3.0.0/user_doc/guide/data-quality.html',
+ },
{
title: 'Resource',
link: '/en-us/docs/3.0.0/user_doc/guide/resource.html',
@@ -555,8 +567,20 @@ export default {
title: 'Webexteams',
link: '/zh-cn/docs/3.0.0/user_doc/guide/alert/enterprise-webexteams.html',
},
+ {
+ title: '脚本告警',
+ link: '/zh-cn/docs/3.0.0/user_doc/guide/alert/script.html',
+ },
+ {
+ title: 'Http告警',
+ link: '/zh-cn/docs/3.0.0/user_doc/guide/alert/http.html',
+ },
],
},
+ {
+ title: '数据质量',
+ link: '/zh-cn/docs/3.0.0/user_doc/guide /data-quality.html',
+ },
{
title: '资源中心',
link: '/zh-cn/docs/3.0.0/user_doc/guide/resource.html',
diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md
index 5caf1fc9d5..b56ed1b119 100644
--- a/docs/docs/en/architecture/configuration.md
+++ b/docs/docs/en/architecture/configuration.md
@@ -397,21 +397,41 @@ apiServers="ds1"
### dolphinscheduler_env.sh [load environment variables configs]
-When using shell to commit tasks, DS will load environment variables inside dolphinscheduler_env.sh into the host.
-Types of tasks involved are: Shell, Python, Spark, Flink, DataX, etc.
-```bash
-export HADOOP_HOME=/opt/soft/hadoop
-export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
-export SPARK_HOME1=/opt/soft/spark1
-export SPARK_HOME2=/opt/soft/spark2
-export PYTHON_HOME=/opt/soft/python
-export JAVA_HOME=/opt/soft/java
-export HIVE_HOME=/opt/soft/hive
-export FLINK_HOME=/opt/soft/flink
-export DATAX_HOME=/opt/soft/datax/bin/datax.py
-
-export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
+When using shell to commit tasks, DolphinScheduler will export environment variables from `bin/env/dolphinscheduler_env.sh`. The
+mainly configuration including `JAVA_HOME`, mata database, registry center, and task configuration.
+```bash
+# JAVA_HOME, will use it to start DolphinScheduler server
+export JAVA_HOME=${JAVA_HOME:-/opt/soft/java}
+
+# Database related configuration, set database type, username and password
+export DATABASE=${DATABASE:-postgresql}
+export SPRING_PROFILES_ACTIVE=${DATABASE}
+export SPRING_DATASOURCE_DRIVER_CLASS_NAME
+export SPRING_DATASOURCE_URL
+export SPRING_DATASOURCE_USERNAME
+export SPRING_DATASOURCE_PASSWORD
+
+# DolphinScheduler server related configuration
+export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
+export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
+export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
+
+# Registry center configuration, determines the type and link of the registry center
+export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
+export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-localhost:2181}
+
+# Tasks related configurations, need to change the configuration if you use the related tasks.
+export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
+export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1}
+export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
+export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
+export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
+export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
+export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
+
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
```
### Services logback configs
diff --git a/docs/docs/en/faq.md b/docs/docs/en/faq.md
index 05d8be7dc1..be3a969a4f 100644
--- a/docs/docs/en/faq.md
+++ b/docs/docs/en/faq.md
@@ -218,7 +218,7 @@ A: 1, in **the process definition list**, click the **Start** button.
## Q : Python task setting Python version
-A: 1,**for the version after 1.0.3** only need to modify PYTHON_HOME in conf/env/.dolphinscheduler_env.sh
+A: 1,**for the version after 1.0.3** only need to modify PYTHON_HOME in `worker/bin/dolphinscheduler_env.sh`
```
export PYTHON_HOME=/bin/python
@@ -306,11 +306,10 @@ A: 1, Create deployment user and hosts mapping, please refer 1.3 part of [cluste
3, Copy the deployment directory from worker server that has already deployed
- 4, Go to bin dir, then start worker server and logger server
+ 4, Go to bin dir, then start worker server
```
./dolphinscheduler-daemon.sh start worker-server
- ./dolphinscheduler-daemon.sh start logger-server
```
---
@@ -602,7 +601,7 @@ A:By deploying different worker in different yarn clusters,the steps are as
2. Changing `yarn.application.status.address` to current emr's yarn url in the `conf/common.properties`
- 3. Execute command `bin/dolphinscheduler-daemon.sh start worker-server` and `bin/dolphinscheduler-daemon.sh start logger-server` to start worker-server and logger-server
+ 3. Execute command `bin/dolphinscheduler-daemon.sh start worker-server` to start worker-server
---
@@ -707,6 +706,34 @@ After version 3.0.0-alpha, Python gateway server integrate into API server, and
start API server. If you want disabled when Python gateway service you could change API server configuration in path
`api-server/conf/application.yaml` and change attribute `python-gateway.enabled : false`.
+## 如果构建自定义的 Docker 镜像
+
+DolphinScheduler 每次发版都会同时发布 Docker 镜像,你可以在 docker hub 中找到这些镜像,如果你因为个性化需求想要自己打包 docker 镜像,最佳实践是基于 DolphinScheduler 对应镜像编写 Dockerfile 文件
+
+```Dockerfile
+FROM dolphinscheduler-standalone-server
+RUN apt update ; \
+ apt install -y <YOUR-CUSTOM-DEPENDENCE> ; \
+```
+
+如果你想基于源码进行改造,打包并分发你的镜像,可以在代码改造完成后运行
+
+```shell
+./mvnw -B clean deploy \
+ -Dmaven.test.skip \
+ -Dmaven.javadoc.skip \
+ -Dmaven.checkstyle.skip \
+ -Dmaven.deploy.skip \
+ -Ddocker.tag=latest \
+ -Pdocker,release
+```
+
+如果你不仅需要改造源码,还想要自定义 Docker 镜像打包的依赖,可以在修改源码的同时修改 Dockerfile 的定义,你可以在源码项目根目录中运行以下命令找到所有的 Dockerfile 文件
+
+```shell
+find . -iname 'Dockerfile'
+```
+
---
## We will collect more FAQ later
\ No newline at end of file
diff --git a/docs/docs/en/guide/alert/alert_plugin_user_guide.md b/docs/docs/en/guide/alert/alert_plugin_user_guide.md
index 8ea72f8889..aeaba871e4 100644
--- a/docs/docs/en/guide/alert/alert_plugin_user_guide.md
+++ b/docs/docs/en/guide/alert/alert_plugin_user_guide.md
@@ -2,7 +2,11 @@
## How to Create Alert Plugins and Alert Groups
-In version 2.0.0, users need to create alert instances, and then associate them with alert groups. Alert group can use multiple alert instances and notify them one by one.
+In version 2.0.0, users need to create alert instances, and needs to choose an alarm policy when defining an alarm instance, there are three options: send if the task succeeds, send on failure, and send on both success and failure. when the workflow or task is executed, if an alarm is triggered, calling the alarm instance send method needs a logical judgment, which matches the alarm instance with the task status, executes the alarm instance sending logic if it matches, and filters if it [...]
+The alarm module supports the following scenarios:
+<img src="/img/alert/alert_scenarios_en.png">
+
+The steps to use are as follows:
First, go to the Security Center page. Select Alarm Group Management, click Alarm Instance Management on the left and create an alarm instance. Select the corresponding alarm plug-in and fill in the relevant alarm parameters.
@@ -11,4 +15,4 @@ Then select Alarm Group Management, create an alarm group, and choose the corres
<img src="/img/alert/alert_step_1.png">
<img src="/img/alert/alert_step_2.png">
<img src="/img/alert/alert_step_3.png">
-<img src="/img/alert/alert_step_4.png">
\ No newline at end of file
+<img src="/img/alert/alert_step_4.png">
diff --git a/docs/docs/en/guide/alert/enterprise-wechat.md b/docs/docs/en/guide/alert/enterprise-wechat.md
index 750c10c583..41970143df 100644
--- a/docs/docs/en/guide/alert/enterprise-wechat.md
+++ b/docs/docs/en/guide/alert/enterprise-wechat.md
@@ -1,14 +1,71 @@
# Enterprise WeChat
-If you need to use `Enterprise WeChat` to alert, create an alert instance in the alert instance management, and choose the WeChat plugin.
+If you need to use `Enterprise WeChat` to alert, create an alert instance in the alert instance management, and choose the `WeChat` plugin.
The following is the `WeChat` configuration example:

-The parameter `send.type` corresponds to app and group chat respectively:
+## Send Type
+
+The parameter `send.type` corresponds to sending messages to Enterprise WeChat customized APP and group chat created by API respectively.
+
+### APP
+
+The APP sends type means to notify the alert results via Enterprise WeChat customized APPs, supports sending messages to both specified users and all members. Currently, send to specified enterprise department and tags are not supported, a new PR to contribute is welcomed.
+The following is the `APP` alert config example:
+
+
+
+The following is the `APP` `MARKDOWN` alert message example:
+
+
+
+The following is the `APP` `TEXT` alert message example:
+
+
+
+#### Prerequisites
+
+Need to create a new customized APP in Enterprise WeChat before sending messages to APP, create at the [APP Page](https://work.weixin.qq.com/wework_admin/frame#apps) and acquire the APP `AgentId` and set its visible scope to the root of the hierarchy.
+
+#### Send Messages to Specified Users
+
+The Enterprise WeChat APPs support sending messages to both specified users and all members, using `|` to separate multiple `userIds` and using `@all` to send messages to everyone.
+To acquire user `userId` refer to [Official Doc](https://developer.work.weixin.qq.com/document/path/95402), acquire `userId` by user phone number.
+The following is the `query userId` API example:
+
+
+
+#### References
APP: https://work.weixin.qq.com/api/doc/90000/90135/90236
-Group Chat: https://work.weixin.qq.com/api/doc/90000/90135/90248
-The parameter `user.send.msg` corresponds to the `content` in the document, and the corresponding variable is `{msg}`.
\ No newline at end of file
+### Group Chat
+
+The Group Chat send type means to notify the alert results via group chat created by Enterprise WeChat API, sending messages to all members of the group and specified users are not supported.
+The following is the `Group Chat` alert config example:
+
+
+
+The following is the `APP` `MARKDOWN` alert message example:
+
+
+
+The following is the `Group Chat` `TEXT` alert message example:
+
+
+
+#### Prerequisites
+
+Before sending messages to group chat, create a new group chat by Enterprise WeChat API, refer to [Official Doc](https://developer.work.weixin.qq.com/document/path/90245) to create a new one and acquire `chatid`.
+To acquire user `userId` refer to [Official Doc](https://developer.work.weixin.qq.com/document/path/95402), acquire `userId` by user phone number.
+The following is the `create new group chat` API and `query userId` API example:
+
+
+
+
+
+#### References
+
+Group Chat:https://work.weixin.qq.com/api/doc/90000/90135/90248
\ No newline at end of file
diff --git a/docs/docs/en/guide/alert/http.md b/docs/docs/en/guide/alert/http.md
new file mode 100644
index 0000000000..9174fafbd8
--- /dev/null
+++ b/docs/docs/en/guide/alert/http.md
@@ -0,0 +1,34 @@
+# HTTP
+
+If you need to use `Http script` for alerting, create an alert instance in the alert instance management and select the `Http` plugin.
+
+## Parameter Configuration
+
+* URL
+ > The `Http` request URL needs to contain protocol, host, path and parameters if the method is `GET`
+* Request Type
+ > Select the request type from `POST` or `GET`
+* Headers
+ > The headers of the `Http` request in JSON format
+* Body
+ > The request body of the `Http` request in JSON format, when using `POST` method to alert
+* Content Field
+ > The field name to place the alert information
+
+## Send Type
+
+Using `POST` and `GET` method to send `Http` request in the `Request Type`.
+
+### GET Http
+
+Send alert information by `Http` GET method.
+The following shows the `GET` configuration example:
+
+
+
+### POST Http
+
+Send alert information inside `Http` body by `Http` POST method.
+The following shows the `POST` configuration example:
+
+
\ No newline at end of file
diff --git a/docs/docs/en/guide/alert/script.md b/docs/docs/en/guide/alert/script.md
new file mode 100644
index 0000000000..fb7a34b883
--- /dev/null
+++ b/docs/docs/en/guide/alert/script.md
@@ -0,0 +1,17 @@
+# DingTalk
+
+If you need to use `Shell script` for alerting, create an alert instance in the alert instance management and select the `Script` plugin.
+The following shows the `Script` configuration example:
+
+
+
+## Parameter Configuration
+
+* User Params
+ > User defined parameters will pass to the script.
+* Script Path
+ > The file location path in the server.
+* Type
+ > Support `Shell` script.
+
+**_Notice:_** please consider the script file access privileges with the executing tenant.
\ No newline at end of file
diff --git a/docs/docs/en/guide/data-quality.md b/docs/docs/en/guide/data-quality.md
new file mode 100644
index 0000000000..ed3ee03cc9
--- /dev/null
+++ b/docs/docs/en/guide/data-quality.md
@@ -0,0 +1,310 @@
+# Overview
+## Introduction
+
+The data quality task is used to check the data accuracy during the integration and processing of data. Data quality tasks in this release include single-table checking, single-table custom SQL checking, multi-table accuracy, and two-table value comparisons. The running environment of the data quality task is Spark 2.4.0, and other versions have not been verified, and users can verify by themselves.
+- The execution flow of the data quality task is as follows:
+
+> The user defines the task in the interface, and the user input value is stored in `TaskParam`
+When running a task, `Master` will parse `TaskParam`, encapsulate the parameters required by `DataQualityTask` and send it to `Worker`.
+Worker runs the data quality task. After the data quality task finishes running, it writes the statistical results to the specified storage engine. The current data quality task result is stored in the `t_ds_dq_execute_result` table of `dolphinscheduler`
+`Worker` sends the task result to `Master`, after `Master` receives `TaskResponse`, it will judge whether the task type is `DataQualityTask`, if so, it will read the corresponding result from `t_ds_dq_execute_result` according to `taskInstanceId`, and then The result is judged according to the check mode, operator and threshold configured by the user. If the result is a failure, the corresponding operation, alarm or interruption will be performed according to the failure policy configure [...]
+
+Add config : `<server-name>/conf/common.properties`
+
+```properties
+data-quality.jar.name=dolphinscheduler-data-quality-dev-SNAPSHOT.jar
+```
+
+Please fill in `data-quality.jar.name` according to the actual package name,
+If you package `data-quality` separately, remember to modify the package name to be consistent with `data-quality.jar.name`.
+If the old version is upgraded and used, you need to execute the `sql` update script to initialize the database before running.
+If you want to use `MySQL` data, you need to comment out the `scope` of `MySQL` in `pom.xml`
+Currently only `MySQL`, `PostgreSQL` and `HIVE` data sources have been tested, other data sources have not been tested yet
+`Spark` needs to be configured to read `Hive` metadata, `Spark` does not use `jdbc` to read `Hive`
+
+## Detail
+
+- CheckMethod: [CheckFormula][Operator][Threshold], if the result is true, it indicates that the data does not meet expectations, and the failure strategy is executed.
+- CheckFormula:
+ - Expected-Actual
+ - Actual-Expected
+ - (Actual/Expected)x100%
+ - (Expected-Actual)/Expected x100%
+- Operator:=、>、>=、<、<=、!=
+- ExpectedValue
+ - FixValue
+ - DailyAvg
+ - WeeklyAvg
+ - MonthlyAvg
+ - Last7DayAvg
+ - Last30DayAvg
+ - SrcTableTotalRows
+ - TargetTableTotalRows
+
+- example
+ - CheckFormula:Expected-Actual
+ - Operator:>
+ - Threshold:0
+ - ExpectedValue:FixValue=9。
+
+Assuming that the actual value is 10, the operator is >, and the expected value is 9, then the result 10 -9 > 0 is true, which means that the row data in the empty column has exceeded the threshold, and the task is judged to fail
+# Guide
+## NullCheck
+### Introduction
+The goal of the null value check is to check the number of empty rows in the specified column. The number of empty rows can be compared with the total number of rows or a specified threshold. If it is greater than a certain threshold, it will be judged as failure.
+- Calculate the SQL statement that the specified column is empty as follows:
+
+ ```sql
+ SELECT COUNT(*) AS miss FROM ${src_table} WHERE (${src_field} is null or ${src_field} = '') AND (${src_filter})
+ ```
+
+- The SQL to calculate the total number of rows in the table is as follows:
+
+ ```sql
+ SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})
+ ```
+
+### UI Guide
+
+- Source data type: select MySQL, PostgreSQL, etc.
+- Source data source: the corresponding data source under the source data type
+- Source data table: drop-down to select the table where the validation data is located
+- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
+- Src table check column: drop-down to select the check column name
+- Check method:
+ - [Expected-Actual]
+ - [Actual-Expected]
+ - [Actual/Expected]x100%
+ - [(Expected-Actual)/Expected]x100%
+- Check operators: =, >, >=, <, <=, ! =
+- Threshold: The value used in the formula for comparison
+- Failure strategy
+ - Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
+ - Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
+- Expected value type: select the desired type from the drop-down menu
+
+## Timeliness Check
+### Introduction
+The timeliness check is used to check whether the data is processed within the expected time. The start time and end time can be specified to define the time range. If the amount of data within the time range does not reach the set threshold, the check task will be judged as fail
+### UI Guide
+
+- Source data type: select MySQL, PostgreSQL, etc.
+- Source data source: the corresponding data source under the source data type
+- Source data table: drop-down to select the table where the validation data is located
+- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
+- Src table check column: drop-down to select check column name
+- start time: the start time of a time range
+- end time: the end time of a time range
+- Time Format: Set the corresponding time format
+- Check method:
+ - [Expected-Actual]
+ - [Actual-Expected]
+ - [Actual/Expected]x100%
+ - [(Expected-Actual)/Expected]x100%
+- Check operators: =, >, >=, <, <=, ! =
+- Threshold: The value used in the formula for comparison
+- Failure strategy
+ - Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
+ - Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
+- Expected value type: select the desired type from the drop-down menu
+
+## Field Length Check
+### Introduction
+The goal of field length verification is to check whether the length of the selected field meets the expectations. If there is data that does not meet the requirements, and the number of rows exceeds the threshold, the task will be judged to fail
+### UI Guide
+
+- Source data type: select MySQL, PostgreSQL, etc.
+- Source data source: the corresponding data source under the source data type
+- Source data table: drop-down to select the table where the validation data is located
+- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
+- Src table check column: drop-down to select the check column name
+- Logical operators: =, >, >=, <, <=, ! =
+- Field length limit: like the title
+- Check method:
+ - [Expected-Actual]
+ - [Actual-Expected]
+ - [Actual/Expected]x100%
+ - [(Expected-Actual)/Expected]x100%
+- Check operators: =, >, >=, <, <=, ! =
+- Threshold: The value used in the formula for comparison
+- Failure strategy
+ - Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
+ - Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
+- Expected value type: select the desired type from the drop-down menu
+
+## Uniqueness Check
+### Introduction
+The goal of the uniqueness check is to check whether the field is duplicated. It is generally used to check whether the primary key is duplicated. If there is duplication and the threshold is reached, the check task will be judged to be failed.
+### UI Guide
+
+- Source data type: select MySQL, PostgreSQL, etc.
+- Source data source: the corresponding data source under the source data type
+- Source data table: drop-down to select the table where the validation data is located
+- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
+- Src table check column: drop-down to select the check column name
+- Check method:
+ - [Expected-Actual]
+ - [Actual-Expected]
+ - [Actual/Expected]x100%
+ - [(Expected-Actual)/Expected]x100%
+- Check operators: =, >, >=, <, <=, ! =
+- Threshold: The value used in the formula for comparison
+- Failure strategy
+ - Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
+ - Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
+- Expected value type: select the desired type from the drop-down menu
+
+## Regular Expression Check
+### Introduction
+The goal of regular expression verification is to check whether the format of the value of a field meets the requirements, such as time format, email format, ID card format, etc. If there is data that does not meet the format and exceeds the threshold, the task will be judged as failed.
+### UI Guide
+
+- Source data type: select MySQL, PostgreSQL, etc.
+- Source data source: the corresponding data source under the source data type
+- Source data table: drop-down to select the table where the validation data is located
+- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
+- Src table check column: drop-down to select check column name
+- Regular expression: as title
+- Check method:
+ - [Expected-Actual]
+ - [Actual-Expected]
+ - [Actual/Expected]x100%
+ - [(Expected-Actual)/Expected]x100%
+- Check operators: =, >, >=, <, <=, ! =
+- Threshold: The value used in the formula for comparison
+- Failure strategy
+ - Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
+ - Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
+- Expected value type: select the desired type from the drop-down menu
+
+## Enumeration Check
+### Introduction
+The goal of enumeration value verification is to check whether the value of a field is within the range of enumeration values. If there is data that is not in the range of enumeration values and exceeds the threshold, the task will be judged to fail
+### UI Guide
+
+- Source data type: select MySQL, PostgreSQL, etc.
+- Source data source: the corresponding data source under the source data type
+- Source data table: drop-down to select the table where the validation data is located
+- Src table filter conditions: such as title, also used when counting the total number of rows in the table, optional
+- Src table check column: drop-down to select the check column name
+- List of enumeration values: separated by commas
+- Check method:
+ - [Expected-Actual]
+ - [Actual-Expected]
+ - [Actual/Expected]x100%
+ - [(Expected-Actual)/Expected]x100%
+- Check operators: =, >, >=, <, <=, ! =
+- Threshold: The value used in the formula for comparison
+- Failure strategy
+ - Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
+ - Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
+- Expected value type: select the desired type from the drop-down menu
+## Table Count Check
+### Introduction
+The goal of table row number verification is to check whether the number of rows in the table reaches the expected value. If the number of rows does not meet the standard, the task will be judged as failed.
+### UI Guide
+
+- Source data type: select MySQL, PostgreSQL, etc.
+- Source data source: the corresponding data source under the source data type
+- Source data table: drop-down to select the table where the validation data is located
+- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
+- Src table check column: drop-down to select the check column name
+- Check method:
+ - [Expected-Actual]
+ - [Actual-Expected]
+ - [Actual/Expected]x100%
+ - [(Expected-Actual)/Expected]x100%
+- Check operators: =, >, >=, <, <=, ! =
+- Threshold: The value used in the formula for comparison
+- Failure strategy
+ - Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
+ - Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
+- Expected value type: select the desired type from the drop-down menu
+
+## Custom SQL Check
+### Introduction
+### UI Guide
+
+- Source data type: select MySQL, PostgreSQL, etc.
+- Source data source: the corresponding data source under the source data type
+- Source data table: drop-down to select the table where the data to be verified is located
+- Actual value name: alias in SQL for statistical value calculation, such as max_num
+- Actual value calculation SQL: SQL for outputting actual values,
+ - Note: The SQL must be statistical SQL, such as counting the number of rows, calculating the maximum value, minimum value, etc.
+ - select max(a) as max_num from ${src_table}, the table name must be filled like this
+- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
+- Check method:
+- Check operators: =, >, >=, <, <=, ! =
+- Threshold: The value used in the formula for comparison
+- Failure strategy
+ - Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
+ - Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
+- Expected value type: select the desired type from the drop-down menu
+
+## Accuracy check of multi-table
+### Introduction
+Accuracy checks are performed by comparing the accuracy differences of data records for selected fields between two tables, examples are as follows
+- table test1
+
+| c1 | c2 |
+| :---: | :---: |
+| a | 1 |
+|b|2|
+
+- table test2
+
+| c21 | c22 |
+| :---: | :---: |
+| a | 1 |
+|b|3|
+If you compare the data in c1 and c21, the tables test1 and test2 are exactly the same. If you compare c2 and c22, the data in table test1 and table test2 are inconsistent.
+### UI Guide
+
+- Source data type: select MySQL, PostgreSQL, etc.
+- Source data source: the corresponding data source under the source data type
+- Source data table: drop-down to select the table where the data to be verified is located
+- Src filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
+- Target data type: choose MySQL, PostgreSQL, etc.
+- Target data source: the corresponding data source under the source data type
+- Target data table: drop-down to select the table where the data to be verified is located
+- Target filter conditions: such as the title, it will also be used when counting the total number of rows in the table, optional
+- Check column:
+ - Fill in the source data column, operator and target data column respectively
+- Verification method: select the desired verification method
+- Operators: =, >, >=, <, <=, ! =
+- Failure strategy
+ - Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
+ - Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
+- Expected value type: select the desired type in the drop-down menu, only SrcTableTotalRow, TargetTableTotalRow and fixed value are suitable for selection here
+## Comparison of the values checked by the two tables
+### Introduction
+Two-table value comparison allows users to customize different SQL statistics for two tables and compare the corresponding values. For example, for the source table A, the total amount of a certain column is calculated, and for the target table, the total amount of a certain column is calculated. value sum2, compare sum1 and sum2 to determine the check result
+### UI Guide
+
+- Source data type: select MySQL, PostgreSQL, etc.
+- Source data source: the corresponding data source under the source data type
+- Source data table: the table where the data is to be verified
+- Actual value name: Calculate the alias in SQL for the actual value, such as max_age1
+- Actual value calculation SQL: SQL for outputting actual values,
+ - Note: The SQL must be statistical SQL, such as counting the number of rows, calculating the maximum value, minimum value, etc.
+ - select max(age) as max_age1 from ${src_table} The table name must be filled like this
+- Target data type: choose MySQL, PostgreSQL, etc.
+- Target data source: the corresponding data source under the source data type
+- Target data table: the table where the data is to be verified
+- Expected value name: Calculate the alias in SQL for the expected value, such as max_age2
+- Expected value calculation SQL: SQL for outputting expected value,
+ - Note: The SQL must be statistical SQL, such as counting the number of rows, calculating the maximum value, minimum value, etc.
+ - select max(age) as max_age2 from ${target_table} The table name must be filled like this
+- Verification method: select the desired verification method
+- Operators: =, >, >=, <, <=, ! =
+- Failure strategy
+ - Alert: The data quality task failed, the DolphinScheduler task result is successful, and an alert is sent
+ - Blocking: The data quality task fails, the DolphinScheduler task result is failed, and an alarm is sent
+
+## Task result view
+
+## Rule View
+### List of rules
+
+### Rules Details
+
\ No newline at end of file
diff --git a/docs/docs/en/guide/datasource/hive.md b/docs/docs/en/guide/datasource/hive.md
index 9eff6013f6..6ac14fa479 100644
--- a/docs/docs/en/guide/datasource/hive.md
+++ b/docs/docs/en/guide/datasource/hive.md
@@ -37,3 +37,7 @@ login.user.keytab.username=hdfs-mycluster@ESZ.COM
# login user from keytab path
login.user.keytab.path=/opt/hdfs.headless.keytab
```
+
+## Native Supported
+
+Yes, could use this datasource by default.
diff --git a/docs/docs/en/guide/datasource/introduction.md b/docs/docs/en/guide/datasource/introduction.md
index 5095cd685b..1f8121ca16 100644
--- a/docs/docs/en/guide/datasource/introduction.md
+++ b/docs/docs/en/guide/datasource/introduction.md
@@ -2,5 +2,23 @@
DataSource supports MySQL, PostgreSQL, Hive/Impala, Spark, ClickHouse, Oracle, SQL Server and other DataSources.
-- Click "Data Source Center -> Create Data Source" to create different types of DataSources according to requirements.
-- Click "Test Connection" to test whether the DataSource can connect successfully.
\ No newline at end of file
+- Click bottom "Data Source Center -> Create Data Source" to create a new datasource.
+- Click "Test Connection" to test whether the DataSource can connect successfully(datasource can be saved only if passed the
+ connection test).
+
+## Using datasource incompatible to Apache LICENSE V2 LICENSE
+
+Some of datasource are native supported to DolphinScheduler while others need users download JDBC driver package manually,
+because those JDBC driver incompatible to Apache LICENSE V2 LICENSE. For this reason we have to release DolphinScheduler's
+distribute package without those packages, even if this will make more complicated for users. Datasource such as MySQL,
+Oracle, SQL Server as the examples, but we have the solution to solve this
+
+### Example
+
+For example, if you want to use MySQL datasource, you need to download the correct JDBC driver from [mysql maven repository](https://repo1.maven.org/maven2/mysql/mysql-connector-java),
+and move it into directory `api-server/libs` and `worker-server/libs`. After that, you could activate MySQL datasource by
+restarting `api-server` and `worker-server`. Mount to container volume in the same path and restart it if you use container
+like Docker.
+
+> Note: If you only want to use MySQL in the datasource center, there is no requirement for the version of MySQL JDBC driver.
+> But if you want to use MySQL as the metabase of DolphinScheduler, it only supports [8.0.16 and above](https:/ /repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar) version.
diff --git a/docs/docs/en/guide/datasource/mysql.md b/docs/docs/en/guide/datasource/mysql.md
index 0ac7cda7a1..1648204db4 100644
--- a/docs/docs/en/guide/datasource/mysql.md
+++ b/docs/docs/en/guide/datasource/mysql.md
@@ -12,3 +12,7 @@
- Database name: enter the database name of the MYSQL connection
- Jdbc connection parameters: parameter settings for MYSQL connection, in JSON format
+## Native Supported
+
+No, read section example in [introduction](introduction.md) to activate this datasource.
+
diff --git a/docs/docs/en/guide/datasource/postgresql.md b/docs/docs/en/guide/datasource/postgresql.md
index fe05169d5f..3083e859c4 100644
--- a/docs/docs/en/guide/datasource/postgresql.md
+++ b/docs/docs/en/guide/datasource/postgresql.md
@@ -11,3 +11,7 @@
- Password: set the password for PostgreSQL connection
- Database name: enter the database name of the PostgreSQL connection
- Jdbc connection parameters: parameter settings for PostgreSQL connection, in JSON format
+
+## Native Supported
+
+Yes, could use this datasource by default.
diff --git a/docs/docs/en/guide/datasource/spark.md b/docs/docs/en/guide/datasource/spark.md
index 61e2c7dfed..f481966fc0 100644
--- a/docs/docs/en/guide/datasource/spark.md
+++ b/docs/docs/en/guide/datasource/spark.md
@@ -11,3 +11,7 @@
- Password: set the password for Spark connection
- Database name: enter the database name of the Spark connection
- Jdbc connection parameters: parameter settings for Spark connection, in JSON format
+
+## Native Supported
+
+Yes, could use this datasource by default.
diff --git a/docs/docs/en/guide/expansion-reduction.md b/docs/docs/en/guide/expansion-reduction.md
index 65db52b222..c91d7646cf 100644
--- a/docs/docs/en/guide/expansion-reduction.md
+++ b/docs/docs/en/guide/expansion-reduction.md
@@ -73,10 +73,10 @@ sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
datasource.properties: database connection information
zookeeper.properties: information for connecting zk
common.properties: Configuration information about the resource store (if hadoop is set up, please check if the core-site.xml and hdfs-site.xml configuration files exist).
- env/dolphinscheduler_env.sh: environment Variables
+ dolphinscheduler_env.sh: environment Variables
````
-- Modify the `dolphinscheduler_env.sh` environment variable in the `conf/env` directory according to the machine configuration (the following is the example that all the used software install under `/opt/soft`)
+- Modify the `dolphinscheduler_env.sh` environment variable in the `bin/dolphinscheduler_env.sh` directory according to the machine configuration (the following is the example that all the used software install under `/opt/soft`)
```shell
export HADOOP_HOME=/opt/soft/hadoop
diff --git a/docs/docs/en/guide/howto/general-setting.md b/docs/docs/en/guide/howto/general-setting.md
index fc2588bba4..00fa912bbe 100644
--- a/docs/docs/en/guide/howto/general-setting.md
+++ b/docs/docs/en/guide/howto/general-setting.md
@@ -14,9 +14,17 @@ of to [language](#language) control button.
## Time Zone
-DolphinScheduler support time zone setting. The build-in time zone is based on the server you run DolphinScheduler. You could
+DolphinScheduler support time zone setting.
+
+Server Time Zone
+
+The default time zone is UTC when using `bin/dolphinshceduler_daemon.sh` to start the server, you could update `SPRING_JACKSON_TIME_ZONE` in `bin/env/dolphinscheduler_env.sh`, such as `export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-Asia/Shanghai}`.<br>
+If you start server in IDEA, the default time zone is your local time zone, you could add the JVM parameter to update server time zone, such as `-Duser.timezone=UTC`. Time zone list refer to [List of tz database time zones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)
+
+User Time zone
+
+The user's default time zone is based on the time zone which you run the DolphinScheduler service.You could
click the button on the right of the [language](#language) button and then click `Choose timeZone` to choose the time zone
you want to switch. All time related components will adjust their time zone according to the time zone setting you select.
-DolphinScheduler uses UTC time for the internal communication, the time zone you choose only changes the display time format
-base on the UTC time. When you choose time zone `TIME ZONE`, we just convert the UTC time to the zone you prefer.
+
diff --git a/docs/docs/en/guide/installation/cluster.md b/docs/docs/en/guide/installation/cluster.md
index 4f1ead96a2..37032b85a0 100644
--- a/docs/docs/en/guide/installation/cluster.md
+++ b/docs/docs/en/guide/installation/cluster.md
@@ -6,7 +6,7 @@ If you are a new hand and want to experience DolphinScheduler functions, we reco
## Deployment Steps
-Cluster deployment uses the same scripts and configuration files as [pseudo-cluster deployment](pseudo-cluster.md), so the preparation and deployment steps are the same as pseudo-cluster deployment. The difference is that [pseudo-cluster deployment](pseudo-cluster.md) is for one machine, while cluster deployment (Cluster) is for multiple machines. And steps of "Modify Configuration" are quite different between pseudo-cluster deployment and cluster deployment.
+Cluster deployment uses the same scripts and configuration files as [pseudo-cluster deployment](pseudo-cluster.md), so the preparation and deployment steps are the same as pseudo-cluster deployment. The difference is that pseudo-cluster deployment is for one machine, while cluster deployment (Cluster) is for multiple machines. And steps of "Modify Configuration" are quite different between pseudo-cluster deployment and cluster deployment.
### Prerequisites and DolphinScheduler Startup Environment Preparations
@@ -32,8 +32,8 @@ apiServers="ds5"
## Start and Login DolphinScheduler
-Same as pseudo-cluster.md](pseudo-cluster.md)
+Same as [pseudo-cluster](pseudo-cluster.md)
## Start and Stop Server
-Same as pseudo-cluster.md](pseudo-cluster.md)
\ No newline at end of file
+Same as [pseudo-cluster](pseudo-cluster.md)
\ No newline at end of file
diff --git a/docs/docs/en/guide/installation/pseudo-cluster.md b/docs/docs/en/guide/installation/pseudo-cluster.md
index 333aecc762..1a3ed534eb 100644
--- a/docs/docs/en/guide/installation/pseudo-cluster.md
+++ b/docs/docs/en/guide/installation/pseudo-cluster.md
@@ -67,27 +67,16 @@ Go to the ZooKeeper installation directory, copy configure file `zoo_sample.cfg`
./bin/zkServer.sh start
```
-<!--
-Modify the database configuration and initialize
-
-```properties
-spring.datasource.driver-class-name=com.mysql.jdbc.Driver
-spring.datasource.url=jdbc:mysql://localhost:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true
-# Modify it if you are not using dolphinscheduler/dolphinscheduler as your username and password
-spring.datasource.username=dolphinscheduler
-spring.datasource.password=dolphinscheduler
-```
+## Modify Configuration
-After modifying and saving, execute the following command to create database tables and init basic data.
+After completing the preparation of the basic environment, you need to modify the configuration file according to the
+environment you used. The configuration files in directory `bin/dolphinscheduler_env.sh` and `bin/env/install_env.sh` and
+`<api-server|alert-server|master-server|worker-server|tools>/bin/dolphinscheduler_env.sh`.
-```shell
-sh script/create-dolphinscheduler.sh
-```
--->
+### Modify `install_env.sh`
-## Modify Configuration
-
-After completing the preparation of the basic environment, you need to modify the configuration file according to your environment. The configuration file is in the path of `conf/config/install_config.conf`. Generally, you just need to modify the **INSTALL MACHINE, DolphinScheduler ENV, Database, Registry Server** part to complete the deployment, the following describes the parameters that must be modified:
+File `install_env.sh` describes which machines will be installed DolphinScheduler and what server will be installed on
+each machine. You could find this file in the path `bin/env/install_env.sh` and the detail of the configuration as below.
```shell
# ---------------------------------------------------------
@@ -105,51 +94,103 @@ installPath="~/dolphinscheduler"
# Deploy user, use the user you create in section **Configure machine SSH password-free login**
deployUser="dolphinscheduler"
+```
-# ---------------------------------------------------------
-# DolphinScheduler ENV
-# ---------------------------------------------------------
-# The path of JAVA_HOME, which JDK install path in section **Preparation**
-javaHome="/your/java/home/here"
+### Modify `dolphinscheduler_env.sh`
-# ---------------------------------------------------------
-# Database
-# ---------------------------------------------------------
-# Database type, username, password, IP, port, metadata. For now `dbtype` supports `mysql` and `postgresql`
-dbtype="mysql"
-dbhost="localhost:3306"
-# Need to modify if you are not using `dolphinscheduler/dolphinscheduler` as your username and password
-username="dolphinscheduler"
-password="dolphinscheduler"
-dbname="dolphinscheduler"
+File `dolphinscheduler_env.sh` should be change in both directory `<bin|api-server|alert-server|master-server|worker-server|tools>`
+describes the following configurations:
-# ---------------------------------------------------------
-# Registry Server
-# ---------------------------------------------------------
-# Registration center address, the address of ZooKeeper service
-registryServers="localhost:2181"
+* Database configuration of DolphinScheduler, see [Initialize the Database](#initialize-the-database) for detailed instructions.
+* Some tasks which need external dependencies or libraries such as `JAVA_HOME` and `SPARK_HOME`.
+* Registry center `zookeeper`.
+* Server related configuration, such as cache type, timezone, etc.
+
+You could ignore the task external dependencies if you do not use those tasks, but you have to change `JAVA_HOME`, registry center and database
+related configurations based on your environment.
+
+```shell
+# JAVA_HOME, will use it to start DolphinScheduler server
+export JAVA_HOME=${JAVA_HOME:-/opt/soft/java}
+
+# Database related configuration, set database type, username and password
+export DATABASE=${DATABASE:-postgresql}
+export SPRING_PROFILES_ACTIVE=${DATABASE}
+export SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
+export SPRING_DATASOURCE_URL=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
+export SPRING_DATASOURCE_USERNAME={user}
+export SPRING_DATASOURCE_PASSWORD={password}
+
+# DolphinScheduler server related configuration
+export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
+export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
+export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
+
+# Registry center configuration, determines the type and link of the registry center
+export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
+export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-localhost:2181}
+
+# Tasks related configurations, need to change the configuration if you use the related tasks.
+export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
+export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1}
+export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
+export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
+export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
+export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
+export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
+
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
```
## Initialize the Database
-DolphinScheduler metadata is stored in the relational database. Currently, supports PostgreSQL and MySQL. If you use MySQL, you need to manually download [mysql-connector-java driver][mysql] (8.0.16) and move it to the lib directory of DolphinScheduler. Let's take MySQL as an example for how to initialize the database:
+DolphinScheduler metadata is stored in the relational database. Currently, supports PostgreSQL and MySQL. If you use MySQL, you need to manually download [mysql-connector-java driver][mysql] (8.0.16) and move it to the lib directory of DolphinScheduler, which is `tools/libs/`. Let's take MySQL as an example for how to initialize the database:
+
+For mysql 5.6 / 5.7
```shell
mysql -uroot -p
mysql> CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
-# Change {user} and {password} by requests
+# Replace {user} and {password} with your username and password
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'%' IDENTIFIED BY '{password}';
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'localhost' IDENTIFIED BY '{password}';
mysql> flush privileges;
```
-After the above steps done you would create a new database for DolphinScheduler, then run Shell scripts to init database:
+For mysql 8:
+
+```shell
+mysql -uroot -p
+
+mysql> CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
+
+# Replace {user} and {password} with your username and password
+mysql> CREATE USER '{user}'@'%' IDENTIFIED BY '{password}';
+mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'%';
+mysql> CREATE USER '{user}'@'localhost' IDENTIFIED BY '{password}';
+mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'localhost';
+mysql> FLUSH PRIVILEGES;
+```
+
+Then, modify `tools/bin/dolphinscheduler_env.sh` to use mysql, change {user} and {password} to what you set in the previous step.
+
+```shell
+export DATABASE=${DATABASE:-mysql}
+export SPRING_PROFILES_ACTIVE=${DATABASE}
+export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver
+export SPRING_DATASOURCE_URL=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
+export SPRING_DATASOURCE_USERNAME={user}
+export SPRING_DATASOURCE_PASSWORD={password}
+```
+
+After the above steps done you would create a new database for DolphinScheduler, then run the Shell script to init database:
```shell
-sh script/create-dolphinscheduler.sh
+sh tools/bin/create-schema.sh
```
## Start DolphinScheduler
@@ -157,7 +198,7 @@ sh script/create-dolphinscheduler.sh
Use **deployment user** you created above, running the following command to complete the deployment, and the server log will be stored in the logs folder.
```shell
-sh install.sh
+sh ./bin/install.sh
```
> **_Note:_** For the first time deployment, there maybe occur five times of `sh: bin/dolphinscheduler-daemon.sh: No such file or directory` in the terminal,
diff --git a/docs/docs/en/guide/installation/standalone.md b/docs/docs/en/guide/installation/standalone.md
index c9e37db7f1..80406bd30d 100644
--- a/docs/docs/en/guide/installation/standalone.md
+++ b/docs/docs/en/guide/installation/standalone.md
@@ -4,7 +4,8 @@ Standalone only for quick experience for DolphinScheduler.
If you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow [Standalone deployment](standalone.md). If you want to experience more complete functions and schedule massive tasks, we recommend you install follow [pseudo-cluster deployment](pseudo-cluster.md). If you want to deploy DolphinScheduler in production, we recommend you follow [cluster deployment](cluster.md) or [Kubernetes deployment](kubernetes.md).
-> **_Note:_** Standalone only recommends the usage of fewer than 20 workflows, because it uses H2 Database, ZooKeeper Testing Server, too many tasks may cause instability.
+> **_Note:_** Standalone only recommends the usage of fewer than 20 workflows, because it uses in-memory H2 Database in default, ZooKeeper Testing Server, too many tasks may cause instability.
+> When Standalone stops or restarts, in-memory H2 database will clear up. To use Standalone with external databases like mysql or postgresql, please see [`Database Configuration`](#database-configuration).
## Preparation
@@ -44,3 +45,20 @@ sh ./bin/dolphinscheduler-daemon.sh stop standalone-server
> path `api-server/conf/application.yaml`
[jdk]: https://www.oracle.com/technetwork/java/javase/downloads/index.html
+
+### Database Configuration
+
+* Use mysql as an example to illustrate how to configure an external database:
+* First of all, follow the instructions in [pseudo-cluster deployment](pseudo-cluster.md) `Initialize the Database` section to create and initialize database
+* Set the following environment variables in your terminal with your database username and password for {user} and {password}:
+
+```shell
+export DATABASE=${DATABASE:-mysql}
+export SPRING_PROFILES_ACTIVE=${DATABASE}
+export SPRING_DATASOURCE_USERNAME={user}
+export SPRING_DATASOURCE_PASSWORD={password}
+```
+
+* Add mysql-connector-java driver to `./standalone-server/libs/standalone-server/`, see [pseudo-cluster deployment](pseudo-cluster.md) `Initialize the Database` section about where to download
+
+* Start standalone-server, now you are using mysql as database and it will not clear up your data when you stop or restart standalone-server.
diff --git a/docs/docs/en/guide/start/docker.md b/docs/docs/en/guide/start/docker.md
index b6df8e9304..2b7c596405 100644
--- a/docs/docs/en/guide/start/docker.md
+++ b/docs/docs/en/guide/start/docker.md
@@ -1,1024 +1,135 @@
-# Quick Trial Docker Deployment
+# Docker Quick Start
-## Pre-conditions
+There are three ways to start DolphinScheduler with Docker, [Standalone-server](#using-standalone-server-docker-image) is the way you
+find if you just want to start and try DolphinScheduler as a beginner. [docker-compose](#using-docker-compose-to-start-server) is for
+some who want to deploy DolphinScheduler in small or event middle scale workflows in their daily work.
+[Using exists postgresql and zookeeper server](#using-exists-postgresql-zookeeper) is for users who want to reuse the database
+or zookeeper server already exists.
+
+## Prepare
- [Docker](https://docs.docker.com/engine/install/) 1.13.1+
- [Docker Compose](https://docs.docker.com/compose/) 1.11.0+
-## How to use docker image?
-
-There are 3 ways to quickly try DolphinScheduler.
-
-### I. Start DolphinScheduler as docker-compose (recommended)
-
-This method requires the installation of [docker-compose](https://docs.docker.com/compose/). The installation of docker-compose is widely available online, so please install it yourself.
-
-For Windows 7-10 versions, you can install [Docker Toolbox](https://github.com/docker/toolbox/releases). For Windows 10 64-bit, you can install [Docker Desktop](https://docs.docker.com/docker-for-windows/install/) and note the [system requirements](https://docs.docker.com/ docker-for-windows/install/#system-requirements).
-
-#### 0. Please allocate at least 4GB of memory
-
-For Mac users, click on `Docker Desktop -> Preferences -> Resources -> Memory`.
-
-For Windows Docker Toolbox users, there are two items that need to be configured.
+## Start Server
-- **Memory**: Open Oracle VirtualBox Manager and if you double click on Docker Quickstart Terminal and run Docker Toolbox successfully, you will see a virtual machine named `default`. Click on `Settings -> System -> Motherboard -> Memory Size`.
-- **Port Forwarding**: Click `Settings -> Network -> Advanced -> Port Forwarding -> Add`. `Name`, fill in `12345` for both `Host Port` and `Subsystem Port`, leave out the `Host IP` and `Subsystem IP`.
+### Using standalone-server Docker Image
-For Windows Docker Desktop users.
-- **Hyper-V Mode**: Click `Docker Desktop -> Settings -> Resources -> Memory`
-- **WSL 2 Mode**: Reference [WSL 2 utility VM](https://docs.microsoft.com/zh-cn/windows/wsl/wsl-config#configure-global-options-with-wslconfig)
-
-#### 1.Download the source code package
-
-Please download the source package apache-dolphinscheduler-x.x.x-src.tar.gz from: [download](/en-us/download/download.html)
-
-#### 2.Pull the image and start the service
-
-> For Mac and Linux user, open **Terminal**
-> For Windows Docker Toolbox user, open **Docker Quickstart Terminal**
-> For Windows Docker Desktop user, open **Windows PowerShell**
+Start DolphinScheduler with standalone-server Docker images is the easiest way to experience and explode it. In this way,
+you can learn DolphinScheduler's concepts and usage, with minimal cost.
+```shell
+$ DOLPHINSCHEDULER_VERSION=3.0.0-alpha
+$ docker run --name dolphinscheduler-standalone-server -p 12345:12345 -p 25333:25333 -d apache/dolphinscheduler-standalone-server:"${DOLPHINSCHEDULER_VERSION}"
```
-$ tar -zxvf apache-dolphinscheduler-3.0.0-alpha-src.tar.gz
-$ cd apache-dolphinscheduler-3.0.0-alpha-src/docker/docker-swarm
-$ docker pull dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:3.0.0-alpha
-$ docker tag apache/dolphinscheduler:3.0.0-alpha apache/dolphinscheduler:latest
-$ docker-compose up -d
-```
-> PowerShell should use `cd apache-dolphinscheduler-3.0.0-alpha-src\docker\docker-swarm`
-
-**PostgreSQL** (user `root`, password `root`, database `dolphinscheduler`) and **ZooKeeper** services will be started by default
-#### 3.Login system
+> Note: Do not use apache/dolphinscheduler-standalone-server Docker image in production, it should only for you to taste
+> DolphinScheduler at the first time. Not only because it runs all services in one single process, but also it uses H2 as
+> its database which will lose metadata after it stops (could be changed to another database to avoid it). In addition,
+> apache/dolphinscheduler-standalone-server only contains DolphinScheduler core services, some tasks such as Spark and Flink,
+> require external components or environment to run it.
-To access the front-end page: http://localhost:12345/dolphinscheduler, please change to the corresponding IP address if necessary
+### Using docker-compose to Start Server
-The default user is `admin` and the default password is `dolphinscheduler123`.
+The difference between start services by docker-compose and standalone-server are servers running in one single process
+or not. Services start with docker-compose running in separate containers, as well as different processes. Metadata could
+be stored on disks after you change docker-compose configuration, and it is robust and stable for someone who wants to run
+DolphinScheduler in a long term. You have to install [docker-compose](https://docs.docker.com/compose/install/) before you
+start servers.
-
-
-Please refer to the user manual chapter [Quick Start] (... /start/quick-start.md) to see how to use DolphinScheduler.
-
-### II. By specifying the existing PostgreSQL and ZooKeeper services
-
-This method requires the installation of [docker](https://docs.docker.com/engine/install/). The installation of docker is well documented on the web, so please install it yourself.
-
-#### 1.Basic software installation (please install it yourself)
-
-- [PostgreSQL](https://www.postgresql.org/download/) (8.2.15+)
-- [ZooKeeper](https://zookeeper.apache.org/releases.html) (3.4.6+)
-- [Docker](https://docs.docker.com/engine/install/) (1.13.1+)
-
-#### 2. Please login to the PostgreSQL database and create a database named `dolphinscheduler`.
-
-#### 3. Initialize the database and import `sql/dolphinscheduler_postgre.sql` to create tables and import the base data
+After installed docker-compose, it is recommended to modify some configurations for better experience. We highly recommended
+modify docker-compose's memory up to 4 GB.
-#### 4. Download the DolphinScheduler image
+- Mac:Click `Docker Desktop -> Preferences -> Resources -> Memory` modified it
+- Windows Docker Desktop:
+ - Hyper-V mode: Click `Docker Desktop -> Settings -> Resources -> Memory` modified it
+ - WSL 2 mode: see [WSL 2 utility VM](https://docs.microsoft.com/zh-cn/windows/wsl/wsl-config#configure-global-options-with-wslconfig) for more detail.
-We have uploaded the DolphinScheduler images for users to the docker repository. Instead of building the image locally, users can pull the image from the docker repository by running the following command.
+After complete the configuration, we can get the `docker-compose.yaml` file from [download page](/en-us/download/download.html)
+form its source package, and make sure you get the right version. After download the package, you can run the commands as below.
+```shell
+$ DOLPHINSCHEDULER_VERSION=3.0.0-alpha
+$ tar -zxf apache-dolphinscheduler-"${DOLPHINSCHEDULER_VERSION}"-src.tar.gz
+# Going to docker-compose's location
+# For Mac or Linux users
+$ cd apache-dolphinscheduler-"${DOLPHINSCHEDULER_VERSION}"-src/deploy/docker
+# For Windows users
+$ cd apache-dolphinscheduler-"${DOLPHINSCHEDULER_VERSION}"-src\deploy\docker
+$ docker-compose up -d
```
-docker pull dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:3.0.0-alpha
-```
-
-#### 5. Run a DolphinScheduler instance
-
-```
-$ docker run -d --name dolphinscheduler \
--e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
--e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
--e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
--p 12345:12345 \
-apache/dolphinscheduler:3.0.0-alpha all
-```
-
-Note: The database user test and password test need to be replaced with the actual PostgreSQL user and password. 192.168.x.x needs to be replaced with the host IP of PostgreSQL and ZooKeeper.
-#### 6. Login system
+> NOTES: It will not only start DolphinScheduler servers but also some others necessary services like PostgreSQL(with `root`
+> as user, `root` as password and `dolphinscheduler` as database) and ZooKeeper when starting with docker-compose.
-As above.
+### Using Exists PostgreSQL ZooKeeper
-### III. Running a standalone service in DolphinScheduler
+[Using docker-compose to start server](#using-docker-compose-to-start-server) will create new a database and the ZooKeeper
+container when it up. You could start DolphinScheduler server separately if you want to reuse your exists services.
-When the container is started, the following services are automatically started.
-```
- MasterServer ----- master service
- WorkerServer ----- worker service
- ApiApplicationServer ----- api service
- AlertServer ----- alert service
-```
-
-If you just want to run some of the services in dolphinscheduler. You can run some of the services in dolphinscheduler by executing the following command.
-
-* Start a **master server**, as follows:
-
-```
+```shell
+$ DOLPHINSCHEDULER_VERSION=3.0.0-alpha
+# Initialize the database, make sure database <DATABASE> already exists
+$ docker run -d --name dolphinscheduler-tools \
+ -e DATABASE="postgresql" \
+ # Use "com.mysql.cj.jdbc.driver" if you use MySQL
+ -e SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/<DATABASE>" \
+ -e SPRING_DATASOURCE_USERNAME="<USER>" \
+ -e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
+ apache/dolphinscheduler-tools:"${DOLPHINSCHEDULER_VERSION}" bin/create-schema.sh
+# Starting DolphinScheduler service
$ docker run -d --name dolphinscheduler-master \
--e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
--e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
--e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
-apache/dolphinscheduler:3.0.0-alpha master-server
-```
-
-* Start a **worker server**, as follows:
-
-```
+ -e DATABASE="postgresql" \
+ -e SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_USERNAME="<USER>" \
+ -e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
+ -e REGISTRY_ZOOKEEPER_CONNECT_STRING="localhost:2181" \
+ -d apache/dolphinscheduler-master:"${DOLPHINSCHEDULER_VERSION}"
$ docker run -d --name dolphinscheduler-worker \
--e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
--e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
--e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
-apache/dolphinscheduler:3.0.0-alpha worker-server
-```
-
-* Start a **api server**, as follows:
-
-```
+ -e DATABASE="postgresql" \
+ -e SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_USERNAME="<USER>" \
+ -e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
+ -e REGISTRY_ZOOKEEPER_CONNECT_STRING="localhost:2181" \
+ -d apache/dolphinscheduler-worker:"${DOLPHINSCHEDULER_VERSION}"
$ docker run -d --name dolphinscheduler-api \
--e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
--e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
--e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
--p 12345:12345 \
-apache/dolphinscheduler:3.0.0-alpha api-server
-```
-
-* Start a **alter server**, as follows:
-
-```
-$ docker run -d --name dolphinscheduler-alert \
--e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
--e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-apache/dolphinscheduler:3.0.0-alpha alert-server
-```
-
-**NOTE**: When you run some of the services in dolphinscheduler, you must specify these environment variables `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_DATABASE`, `DATABASE_USERNAME`, `DATABASE_ PASSWORD`, `ZOOKEEPER_QUORUM`.
-
-## Environment variables
-
-Docker containers are configured via environment variables, appendix-environment-variables lists the configurable environment variables for DolphinScheduler and their default values <! -- markdown-link-check-disable-line -->
-
-In particular, in Docker Compose and Docker Swarm, this can be configured via the environment variable configuration file `config.env.sh`.
-
-## Support Matrix
-
-| Type | Support | Notes |
-| ------------------------------------------------------------ | ------- | --------------------- |
-| Shell | Yes | Yes
-| Python2 | Yes | Yes
-| Python3 | Indirect support | See FAQ |
-| Hadoop2 | Indirect support | See FAQ |
-| Hadoop3 | Not yet determined | Not yet tested |
-| Spark-Local(client) | Indirect support | See FAQ |
-| Spark-YARN(cluster) | Indirect support | See FAQ |
-| Spark-Standalone(cluster) | not yet | |
-| Spark-Kubernetes(cluster) | not yet |
-| Flink-Local(local>=1.11) | Not yet | Generic CLI mode is not yet supported
-| Flink-YARN(yarn-cluster) | indirectly supported | see FAQ |
-| Flink-YARN(yarn-session/yarn-per-job/yarn-application>=1.11) | Not yet | Generic CLI mode is not yet supported |
-| Flink-Standalone(default) | not yet |
-| Flink-Standalone(remote>=1.11) | Not yet | Generic CLI mode is not yet supported |
-| Flink-Kubernetes(default) | not yet | | Flink-Kubernetes(default) | not yet |
-| Flink-Kubernetes(remote>=1.11) | Not yet | Generic CLI mode is not yet supported |
-| Flink-NativeKubernetes(kubernetes-session/application>=1.11) | Not yet | Generic CLI mode is not yet supported |
-| MapReduce | Indirectly supported | See FAQ |
-| Kerberos | Indirectly supported | See FAQ |
-| HTTP | Yes | Yes
-| DataX | Indirect support | See FAQ | Yes
-| Sqoop | Indirect Support | See FAQ |
-| SQL-MySQL | Indirect Support | See FAQ |
-| SQL-PostgreSQL | Yes | Yes
-| SQL-Hive | Indirect Support | See FAQ |
-| SQL-Spark | Indirect support | See FAQ |
-| SQL-ClickHouse | Indirect Support | See FAQ |
-| SQL-Oracle | Indirect Support | See FAQ |
-| SQL-SQLServer | Indirect Support | See FAQ |
-| SQL-DB2 | Indirect Support | See FAQ |
-
-## FAQ
-
-### How to manage DolphinScheduler via docker-compose?
-
-Start, restart, stop or list all containers:
-
-```
-docker-compose start
-docker-compose restart
-docker-compose stop
-docker-compose ps
-```
-
-Stop all containers and remove all containers, networks:
-
-```
-docker-compose down
-```
-
-Stop all containers and remove all containers, networks and storage volumes:
-
-```
-docker-compose down -v
-```
-
-### How do I check the logs of a container?
-
-Lists all running containers:
-
-```
-docker ps
-docker ps --format "{{.Names}}" # 只打印名字
-```
-
-View the logs of the container named docker-swarm_dolphinscheduler-api_1:
-
-```
-docker logs docker-swarm_dolphinscheduler-api_1
-docker logs -f docker-swarm_dolphinscheduler-api_1 # 跟随日志输出
-docker logs --tail 10 docker-swarm_dolphinscheduler-api_1 # 显示倒数10行日志
-```
-
-### How to scale master and worker with docker-compose?
-
-Scaling master to 2 instances:
-
-```
-docker-compose up -d --scale dolphinscheduler-master=2 dolphinscheduler-master
-```
-
-Scaling worker to 3 instances:
-
-```
-docker-compose up -d --scale dolphinscheduler-worker=3 dolphinscheduler-worker
-```
-
-### How to deploy DolphinScheduler on Docker Swarm?
-
-Assuming the Docker Swarm cluster has been deployed (see [create-swarm](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/) if a Docker Swarm cluster has not been created yet).
-
-Start a stack called dolphinscheduler:
-
-```
-docker stack deploy -c docker-stack.yml dolphinscheduler
-```
-
-List all services of the stack named dolphinscheduler:
-
-```
-docker stack services dolphinscheduler
-```
-
-Stop and remove the stack named dolphinscheduler:
-
-```
-docker stack rm dolphinscheduler
-```
-
-Remove all storage volumes from the stack named dolphinscheduler:
-
-```
-docker volume rm -f $(docker volume ls --format "{{.Name}}" | grep -e "^dolphinscheduler")
-```
-
-### How to scale up and down master and worker on Docker Swarm?
-
-Scaling up the master of a stack named dolphinscheduler to 2 instances:
-
-```
-docker service scale dolphinscheduler_dolphinscheduler-master=2
-```
-
-Scaling up the workers of the stack named dolphinscheduler to 3 instances:
-
-```
-docker service scale dolphinscheduler_dolphinscheduler-worker=3
-```
-
-### How to build a Docker image?
-
-#### Build from source (requires Maven 3.3+ & JDK 1.8+)
-
-In a Unix-like system, execute in Terminal:
-
-```bash
-$ bash ./docker/build/hooks/build
-```
-
-In a Windows system, execute in cmd or PowerShell:
-
-```bat
-C:\dolphinscheduler-src>.\docker\build\hooks\build.bat
-```
-
-If you don't understand `. /docker/build/hooks/build` `. /docker/build/hooks/build.bat` these scripts, please read the contents.
-
-#### Build from binary packages (Maven 3.3+ & JDK 1.8+ not required)
-
-Please download the binary package apache-dolphinscheduler-3.0.0-alpha-bin.tar.gz from: [download](/zh-cn/download/download.html). Then put apache-dolphinscheduler-3.0.0-alpha-bin.tar.gz into the `apache-dolphinscheduler-3.0.0-alpha-src/docker/build` directory and execute it in Terminal or PowerShell:
-
-```
-$ cd apache-dolphinscheduler-3.0.0-alpha-src/docker/build
-$ docker build --build-arg VERSION=3.0.0-alpha -t apache/dolphinscheduler:3.0.0-alpha .
-```
-
-> PowerShell should use `cd apache-dolphinscheduler-3.0.0-alpha-src/docker/build`
-
-#### Building images for multi-platform architectures
-
-Currently supports building images for `linux/amd64` and `linux/arm64` platform architectures, requiring
-
-1. support for [docker buildx](https://docs.docker.com/engine/reference/commandline/buildx/)
-2. have push permissions for https://hub.docker.com/r/apache/dolphinscheduler (**Be careful**: the build command automatically pushes multi-platform architecture images to the apache/dolphinscheduler docker hub by default)
-
-Execute :
-
-```bash
-$ docker login # login, use to push apache/dolphinscheduler
-$ bash ./docker/build/hooks/build x
-```
-
-### How to add an environment variable to Docker?
-
-If you want to add additional actions and environment variables at compile time or run time, you can do so in the `/root/start-init-conf.sh` file, and if you need to change the configuration file, please change the corresponding configuration file in `/opt/dolphinscheduler/conf/*.tpl`. configuration file
-
-For example, add an environment variable `SECURITY_AUTHENTICATION_TYPE` to `/root/start-init-conf.sh`.
-
-```
-export SECURITY_AUTHENTICATION_TYPE=PASSWORD
-```
-
-After adding the above environment variables, you should add this environment variable configuration to the corresponding template file `application-api.properties.tpl`:
-
-```
-security.authentication.type=${SECURITY_AUTHENTICATION_TYPE}
-```
-
-`/root/start-init-conf.sh` will dynamically generate a configuration file based on the template file.
-
-```sh
-echo "generate dolphinscheduler config"
-ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
-eval "cat << EOF
-$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
-EOF
-" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
-done
-```
-
-### How to replace PostgreSQL with MySQL as the database for DolphinScheduler?
-
-> Due to commercial licensing, we cannot use MySQL driver packages directly.
->
-> If you want to use MySQL, you can build it based on the official image `apache/dolphinscheduler`.
-
-1. Download the MySQL driver package [mysql-connector-java-8.0.16.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar)
-
-2. Create a new `Dockerfile` to add the MySQL driver package:
-
-```
-FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:3.0.0-alpha
-COPY mysql-connector-java-8.0.16.jar /opt/dolphinscheduler/lib
-```
-
-3. Build a new image containing the MySQL driver package:
-
-```
-docker build -t apache/dolphinscheduler:mysql-driver
-```
-
-4. Modify all the image fields in the `docker-compose.yml` file to `apache/dolphinscheduler:mysql-driver`.
-
-> If you want to deploy dolphinscheduler on Docker Swarm, you need to modify `docker-stack.yml`.
-
-5. Comment out the `dolphinscheduler-postgresql` block in the `docker-compose.yml` file.
-
-6. Add the `dolphinscheduler-mysql` service to the `docker-compose.yml` file (**optional**, you can use an external MySQL database directly).
-
-7. Modify the DATABASE environment variable in the `config.env.sh` file.
-
-```
-DATABASE_TYPE=mysql
-DATABASE_DRIVER=com.mysql.jdbc.Driver
-DATABASE_HOST=dolphinscheduler-mysql
-DATABASE_PORT=3306
-DATABASE_USERNAME=root
-DATABASE_PASSWORD=root
-DATABASE_DATABASE=dolphinscheduler
-DATABASE_PARAMS=useUnicode=true&characterEncoding=UTF-8
-```
-
-> If you have already added the `dolphinscheduler-mysql` service, set `DATABASE_HOST` to `dolphinscheduler-mysql`.
-
-8. Run dolphinscheduler (see **How to use a docker image** for details).
-
-### How do I support MySQL data sources in the Data Source Centre?
-
-> Due to commercial licensing, we cannot use MySQL's driver packages directly.
->
-> If you want to add a MySQL datasource, you can build it based on the official image `apache/dolphinscheduler`.
-
-1. Download MySQL driver package [mysql-connector-java-8.0.16.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar)
-
-2. Create a new `Dockerfile` to add the MySQL driver package:
-
-```
-FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:3.0.0-alpha
-COPY mysql-connector-java-8.0.16.jar /opt/dolphinscheduler/lib
-```
-
-3. Build a new image containing the MySQL driver package:
-
-```
-docker build -t apache/dolphinscheduler:mysql-driver .
-```
-
-4. Change all `image` fields in the `docker-compose.yml` file to `apache/dolphinscheduler:mysql-driver`.
-
-> If you want to deploy dolphinscheduler on Docker Swarm, you will need to modify `docker-stack.yml`.
-
-5. Run dolphinscheduler (see **How to use a docker image** for details).
-
-6. Add a MySQL data source to the data source centre.
-
-### How to support Oracle data sources in the Data Source Centre?
-
-> Due to commercial licensing, we cannot use Oracle's driver packages directly.
->
-> If you want to add an Oracle datasource, you can build it based on the official image `apache/dolphinscheduler`.
-
-1. Download the Oracle driver package [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`).
-
-2. Create a new `Dockerfile` to add the Oracle driver package:
-
-```
-FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:3.0.0-alpha
-COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib
-```
-
-3. Build a new image containing the Oracle driver package:
-
-```
-docker build -t apache/dolphinscheduler:oracle-driver .
-```
-
-4. Change all `image` fields in the `docker-compose.yml` file to `apache/dolphinscheduler:oracle-driver`.
-
-> If you want to deploy dolphinscheduler on Docker Swarm, you will need to modify `docker-stack.yml`.
-
-5. Run dolphinscheduler (see **How to use a docker image** for details).
-
-6. Add an Oracle data source to the data source centre.
-
-### How to support Python 2 pip and custom requirements.txt?
-
-1. Create a new `Dockerfile` for installing pip:
-
-```
-FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:3.0.0-alpha
-COPY requirements.txt /tmp
-RUN apt-get update && \
- apt-get install -y --no-install-recommends python-pip && \
- pip install --no-cache-dir -r /tmp/requirements.txt && \
- rm -rf /var/lib/apt/lists/*
-```
-
-This command will install the default **pip 18.1**. If you want to upgrade pip, just add a line.
-
-
-```
- pip install --no-cache-dir -U pip && \
-```
-
-2. Build a new image containing pip.
-
-```
-docker build -t apache/dolphinscheduler:pip .
-```
-
-3. Change all `image` fields in the `docker-compose.yml` file to `apache/dolphinscheduler:pip`.
-
-> If you want to deploy dolphinscheduler on Docker Swarm, you will need to modify `docker-stack.yml`.
-
-4. Run dolphinscheduler (see **How to use docker images** for details).
-
-5. Verify pip under a new Python task.
-
-### How do I support Python 3?
-
-1. Create a new `Dockerfile` for installing Python 3:
-
-```
-FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:3.0.0-alpha
-RUN apt-get update && \
- apt-get install -y --no-install-recommends python3 && \
- rm -rf /var/lib/apt/lists/*
-```
-
-This command will install the default **Python 3.7.3**. If you also want to install **pip3**, replace `python3` with `python3-pip` and you're done.
-
-```
- apt-get install -y --no-install-recommends python3-pip && \
-```
-
-2. Build a new image containing Python 3:
-
-```
-docker build -t apache/dolphinscheduler:python3 .
-```
-
-3. Change all `image` fields in the `docker-compose.yml` file to `apache/dolphinscheduler:python3`.
-
-> If you want to deploy dolphinscheduler on Docker Swarm, you will need to modify `docker-stack.yml`.
-
-4. Modify `PYTHON_HOME` to `/usr/bin/python3` in the `config.env.sh` file.
+ -e DATABASE="postgresql" \
+ -e SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_USERNAME="<USER>" \
+ -e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
+ -e REGISTRY_ZOOKEEPER_CONNECT_STRING="localhost:2181" \
+ -d apache/dolphinscheduler-api:"${DOLPHINSCHEDULER_VERSION}"
+$ docker run -d --name dolphinscheduler-alert-server \
+ -e DATABASE="postgresql" \
+ -e SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_USERNAME="<USER>" \
+ -e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
+ -e REGISTRY_ZOOKEEPER_CONNECT_STRING="localhost:2181" \
+ -d apache/dolphinscheduler-alert-server:"${DOLPHINSCHEDULER_VERSION}"
+```
+
+> Note: You should install and start [PostgreSQL](https://www.postgresql.org/download/)(8.2.15+) and [ZooKeeper](https://zookeeper.apache.org/releases.html)(3.4.6+)
+> by yourself if you want to use this way to start Dolphinscheduler, but you do not have those services
+
+## Login DolphinScheduler
+
+You could access DolphinScheduler web UI by click [http://localhost:12345/dolphinscheduler/ui](http://localhost:12345/dolphinscheduler/ui)
+and use `admin` and `dolphinscheduler123` as default username and password in the login page.
-5. Run dolphinscheduler (see **How to use docker images** for details).
-
-6. Verify Python 3 under a new Python task.
-
-### How to support Hadoop, Spark, Flink, Hive or DataX?
-
-Take Spark 2.4.7 as an example:
-
-1. Download the Spark 2.4.7 release binary package `spark-2.4.7-bin-hadoop2.7.tgz`.
-
-2. Run dolphinscheduler (see **How to use a docker image** for details).
-
-3. Copy the Spark 2.4.7 binary package to the Docker container.
-
-```bash
-docker cp spark-2.4.7-bin-hadoop2.7.tgz docker-swarm_dolphinscheduler-worker_1:/opt/soft
-```
-
-Because the storage volume `dolphinscheduler-shared-local` is mounted to `/opt/soft`, all files in `/opt/soft` will not be lost.
-
-4. Login to the container and make sure `SPARK_HOME2` exists.
-
-```bash
-docker exec -it docker-swarm_dolphinscheduler-worker_1 bash
-cd /opt/soft
-tar zxf spark-2.4.7-bin-hadoop2.7.tgz
-rm -f spark-2.4.7-bin-hadoop2.7.tgz
-ln -s spark-2.4.7-bin-hadoop2.7 spark2 # or mv
-$SPARK_HOME2/bin/spark-submit --version
-```
-
-If everything executes correctly, the last command will print the Spark version information.
-
-5. Verify Spark in a shell task.
-
-```
-$SPARK_HOME2/bin/spark-submit --class org.apache.spark.examples.SparkPi $SPARK_HOME2/examples/jars/spark-examples_2.11-2.4.7.jar
-```
-
-Check if the task log contains the output `Pi is roughly 3.146015`.
-
-6. Verifying Spark in a Spark Task
-
-The file `spark-examples_2.11-2.4.7.jar` needs to be uploaded to the Resource Center first, then a Spark task created and set up:
-
-- Spark version: `SPARK2`
-- Class of main function: `org.apache.spark.examples.SparkPi`
-- Main package: `spark-examples_2.11-2.4.7.jar`
-- Deployment method: `local`
-
-Similarly, check if the task log contains output `Pi is roughly 3.146015`
-
-7. Verifying Spark on YARN
-
-Spark on YARN (deployed as a `cluster` or `client`) requires Hadoop support. Similar to Spark support, supporting Hadoop is almost identical to the previous steps.
-
-Make sure `$HADOOP_HOME` and `$HADOOP_CONF_DIR` are present.
-
-### How is Spark 3 supported?
-
-In fact, submitting an application using `spark-submit` is the same, whether it is Spark 1, 2 or 3. In other words, the semantics of `SPARK_HOME2` is a second `SPARK_HOME`, not the `HOME` of `SPARK2`, so simply setting `SPARK_HOME2=/path/to/ spark3`.
-
-Let's take Spark 3.1.1 as an example:
-
-1. Download the Spark 3.1.1 release binary package `spark-3.1.1-bin-hadoop2.7.tgz`.
-
-2. Run dolphinscheduler (see **How to use a docker image** for details).
-
-3. Copy the Spark 3.1.1 binary package to the Docker container
-
-```bash
-docker cp spark-3.1.1-bin-hadoop2.7.tgz docker-swarm_dolphinscheduler-worker_1:/opt/soft
-```
-
-4. log in to the container and ensure that `SPARK_HOME2` exists
-
-```bash
-docker exec -it docker-swarm_dolphinscheduler-worker_1 bash
-cd /opt/soft
-tar zxf spark-3.1.1-bin-hadoop2.7.tgz
-rm -f spark-3.1.1-bin-hadoop2.7.tgz
-ln -s spark-3.1.1-bin-hadoop2.7 spark2 # or mv
-$SPARK_HOME2/bin/spark-submit --version
-```
-
-If everything executes correctly, the last command will print the Spark version information.
-
-5. Verify Spark in a shell task.
-
-```
-$SPARK_HOME2/bin/spark-submit --class org.apache.spark.examples.SparkPi $SPARK_HOME2/examples/jars/spark-examples_2.12-3.1.1.jar
-```
-
-Check if the task log contains the output `Pi is roughly 3.146015`.
-
-### How to support shared storage between Master, Worker and Api services?
-
-> **Note**: If you are deploying via docker-compose on a single machine, steps 1 and 2 can be skipped and you can execute commands like `docker cp hadoop-3.2.2.tar.gz docker-swarm_dolphinscheduler-worker_1:/opt/soft ' Place Hadoop in the container under the shared directory /opt/soft.
-
-For example, the Master, Worker and Api services may use Hadoop at the same time.
-
-1. Modify the `dolphinscheduler-shared-local` storage volume in the `docker-compose.yml` file to support nfs.
-
-> If you want to deploy dolphinscheduler on Docker Swarm, you need to modify `docker-stack.yml`.
-
-```yaml
-volumes:
- dolphinscheduler-shared-local:
- driver_opts:
- type: "nfs"
- o: "addr=10.40.0.199,nolock,soft,rw"
- device: ":/path/to/shared/dir"
-```
-
-
-2. Put Hadoop into nfs.
-
-3. Make sure `$HADOOP_HOME` and `$HADOOP_CONF_DIR` are correct.
-
-### How to support local file storage instead of HDFS and S3?
-
-> **Note**: If you are deploying on a standalone machine via docker-compose, you can skip step 2.
-
-1. modify the following environment variables in the `config.env.sh` file:
-
-```
-RESOURCE_STORAGE_TYPE=HDFS
-FS_DEFAULT_FS=file:///
-```
-
-2. Modify the `dolphinscheduler-resource-local` storage volume in the `docker-compose.yml` file to support nfs.
-
-> If you want to deploy dolphinscheduler on Docker Swarm, you need to modify `docker-stack.yml`.
-
-```yaml
-volumes:
- dolphinscheduler-resource-local:
- driver_opts:
- type: "nfs"
- o: "addr=10.40.0.199,nolock,soft,rw"
- device: ":/path/to/resource/dir"
-```
-
-### How do I support S3 resource stores such as MinIO?
-
-Take MinIO as an example: Modify the following environment variables in the `config.env.sh` file.
-
-```
-RESOURCE_STORAGE_TYPE=S3
-RESOURCE_UPLOAD_PATH=/dolphinscheduler
-FS_DEFAULT_FS=s3a://BUCKET_NAME
-FS_S3A_ENDPOINT=http://MINIO_IP:9000
-FS_S3A_ACCESS_KEY=MINIO_ACCESS_KEY
-FS_S3A_SECRET_KEY=MINIO_SECRET_KEY
-```
-
-`BUCKET_NAME`, `MINIO_IP`, `MINIO_ACCESS_KEY` and `MINIO_SECRET_KEY` need to be changed to actual values.
-
-> **NOTE**: `MINIO_IP` can only use IPs and not domain names, as DolphinScheduler does not yet support S3 path style access.
-
-### How to configure SkyWalking?
-
-Modify the SKYWALKING environment variables in the `config.env.sh` file.
-
-```
-SKYWALKING_ENABLE=true
-SW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11800
-SW_GRPC_LOG_SERVER_HOST=127.0.0.1
-SW_GRPC_LOG_SERVER_PORT=11800
-```
-
-## Appendix - Environment Variables
-
-### Database
-
-**`DATABASE_TYPE`**
-
-Configure the `TYPE` of the `database`, default value `postgresql`.
-
-**NOTE**: This environment variable must be specified when running the `master-server`, `worker-server`, `api-server`, and `alert-server` services in `dolphinscheduler`, so that you can build distributed services better.
-
-**`DATABASE_DRIVER`**
-
-Configure `DRIVER` for `database`, default value `org.postgresql.Driver`.
-
-**NOTE**: This environment variable must be specified when running the `master-server`, `worker-server`, `api-server`, and `alert-server` services in `dolphinscheduler`, so that you can better build distributed services.
-
-**`DATABASE_HOST`**
-
-Configure the `HOST` of `database`, default value `127.0.0.1`.
-
-**NOTE**: This environment variable must be specified when running `master-server`, `worker-server`, `api-server`, `alert-server` services in `dolphinscheduler` so that you can build distributed services better.
-
-**`DATABASE_PORT`**
-
-Configure `PORT` for `database`, default value `5432`.
-
-**NOTE**: This environment variable must be specified when running `master-server`, `worker-server`, `api-server`, `alert-server` services in `dolphinscheduler` so that you can build distributed services better.
-
-**`DATABASE_USERNAME`**
-
-Configure the `USERNAME` of `database`, default value `root`.
-
-**NOTE**: This environment variable must be specified when running `master-server`, `worker-server`, `api-server`, `alert-server` services in `dolphinscheduler`, so that you can build distributed services better.
-
-**`DATABASE_PASSWORD`**
-
-Configure `PASSWORD` for `database`, default value `root`.
-
-**NOTE**: This environment variable must be specified when running `master-server`, `worker-server`, `api-server`, `alert-server` services in `dolphinscheduler`, so that you can better build distributed services.
-
-**`DATABASE_DATABASE`**
-
-Configure `DATABASE` for `database`, default value `dolphinscheduler`.
-
-**NOTE**: This environment variable must be specified when running the `master-server`, `worker-server`, `api-server`, and `alert-server` services in `dolphinscheduler`, so that you can better build distributed services.
-
-**`DATABASE_PARAMS`**
-
-Configure `PARAMS` for `database`, default value `characterEncoding=utf8`.
-
-**NOTE**: This environment variable must be specified when running `master-server`, `worker-server`, `api-server`, `alert-server` services in `dolphinscheduler`, so that you can build distributed services better.
-
-### ZooKeeper
-
-**`ZOOKEEPER_QUORUM`**
-
-Configure the `Zookeeper` address for `dolphinscheduler`, default value `127.0.0.1:2181`.
-
-**NOTE**: This environment variable must be specified when running the `master-server`, `worker-server`, `api-server` services in `dolphinscheduler`, so that you can build distributed services better.
-
-**`ZOOKEEPER_ROOT`**
-
-Configure `dolphinscheduler` as the root directory for data storage in `zookeeper`, default value `/dolphinscheduler`.
-
-### General
-
-**`DOLPHINSCHEDULER_OPTS`**
-
-Configure `jvm options` for `dolphinscheduler`, for `master-server`, `worker-server`, `api-server`, `alert-server`, default `""`,
-
-**`DATA_BASEDIR_PATH`**
-
-User data directory, user configured, make sure it exists and user read/write access, default value `/tmp/dolphinscheduler`.
-
-**`RESOURCE_STORAGE_TYPE`**
-
-Configure the resource storage type for `dolphinscheduler`, options are `HDFS`, `S3`, `NONE`, default `HDFS`.
-
-**`RESOURCE_UPLOAD_PATH`**
-
-Configure the resource storage path on `HDFS/S3`, default value `/dolphinscheduler`.
-
-**`FS_DEFAULT_FS`**
-
-Configure the file system protocol for the resource store, e.g. `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler`, default value `file:///`.
-
-**`FS_S3A_ENDPOINT`**
-
-When `RESOURCE_STORAGE_TYPE=S3`, the access path to `S3` needs to be configured, default value `s3.xxx.amazonaws.com`.
-
-**`FS_S3A_ACCESS_KEY`**
-
-When `RESOURCE_STORAGE_TYPE=S3`, you need to configure the `s3 access key` of `S3`, default value `xxxxxxx`.
-
-**`FS_S3A_SECRET_KEY`**
-
-When `RESOURCE_STORAGE_TYPE=S3`, you need to configure `s3 secret key` for `S3`, default value `xxxxxxx`.
-
-**`HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE`**
-
-Configure whether `dolphinscheduler` is kerberos enabled, default value `false`.
-
-**`JAVA_SECURITY_KRB5_CONF_PATH`**
-
-Configure the path to java.security.krb5.conf for `dolphinscheduler`, default value `/opt/krb5.conf`.
-
-**`LOGIN_USER_KEYTAB_USERNAME`**
-
-Configure the keytab username for the `dolphinscheduler` login user, default value `hdfs@HADOOP.COM`.
-
-**`LOGIN_USER_KEYTAB_PATH`**
-
-Configure the keytab path for the `dolphinscheduler` login user, default value `/opt/hdfs.keytab`.
-
-**`KERBEROS_EXPIRE_TIME`**
-
-Configure the kerberos expiration time for `dolphinscheduler`, in hours, default value `2`.
-
-**`HDFS_ROOT_USER`**
-
-Configure the root user name of hdfs for `dolphinscheduler` when `RESOURCE_STORAGE_TYPE=HDFS`, default value `hdfs`.
-
-**`RESOURCE_MANAGER_HTTPADDRESS_PORT`**
-
-Configure the resource manager httpaddress port for `dolphinscheduler`, default value `8088`.
-
-**`YARN_RESOURCEMANAGER_HA_RM_IDS`**
-
-Configure `dolphinscheduler`'s yarn resourcemanager ha rm ids, default value `null`.
-
-**`YARN_APPLICATION_STATUS_ADDRESS`**
-
-Configure the yarn application status address for `dolphinscheduler`, default value `http://ds1:%s/ws/v1/cluster/apps/%s`.
-
-**`SKYWALKING_ENABLE`**
-
-Configure whether `skywalking` is enabled or not. Default value `false`.
-
-**`SW_AGENT_COLLECTOR_BACKEND_SERVICES`**
-
-Configure the collector back-end address for `skywalking`. Default value `127.0.0.1:11800`.
-
-**`SW_GRPC_LOG_SERVER_HOST`**
-
-Configure the grpc service host or IP for `skywalking`. Default value `127.0.0.1`.
-
-**`SW_GRPC_LOG_SERVER_PORT`**
-
-Configure the grpc service port for `skywalking`. Default value `11800`.
-
-**`HADOOP_HOME`**
-
-Configure `HADOOP_HOME` for `dolphinscheduler`, default value `/opt/soft/hadoop`.
-
-**`HADOOP_CONF_DIR`**
-
-Configure `HADOOP_CONF_DIR` for `dolphinscheduler`, default value `/opt/soft/hadoop/etc/hadoop`.
-
-**`SPARK_HOME1`**
-
-Configure `SPARK_HOME1` for `dolphinscheduler`, default value `/opt/soft/spark1`.
-
-**`SPARK_HOME2`**
-
-Configure `SPARK_HOME2` for `dolphinscheduler`, default value `/opt/soft/spark2`.
-
-**`PYTHON_HOME`**
-
-Configure `PYTHON_HOME` for `dolphinscheduler`, default value `/usr/bin/python`.
-
-**`JAVA_HOME`**
-
-Configure `JAVA_HOME` for `dolphinscheduler`, default value `/usr/local/openjdk-8`.
-
-**`HIVE_HOME`**
-
-Configure `HIVE_HOME` for `dolphinscheduler`, default value `/opt/soft/hive`.
-
-**`FLINK_HOME`**
-
-Configure `FLINK_HOME` for `dolphinscheduler`, default value `/opt/soft/flink`.
-
-**`DATAX_HOME`**
-
-Configure `DATAX_HOME` for `dolphinscheduler`, default value `/opt/soft/datax`.
-
-### Master Server
-
-**`MASTER_SERVER_OPTS`**
-
-Configure `jvm options` for `master-server`, default value `-Xms1g -Xmx1g -Xmn512m`.
-
-**`MASTER_EXEC_THREADS`**
-
-Configure the number of threads to be executed in `master-server`, default value `100`.
-
-**`MASTER_EXEC_TASK_NUM`**
-
-Configure the number of tasks to be executed in `master-server`, default value `20`.
-
-**`MASTER_DISPATCH_TASK_NUM`**
-
-Configure the number of tasks to be dispatched in `master-server`, default value `3`.
-
-**`MASTER_HOST_SELECTOR`**
-
-Configure the selector for the worker host when dispatching tasks in `master-server`, optional values are `Random`, `RoundRobin` and `LowerWeight`, default value `LowerWeight`.
-
-**`MASTER_HEARTBEAT_INTERVAL'**
-
-Configure the heartbeat interaction time in `master-server`, default value `10`.
-
-**`MASTER_TASK_COMMIT_RETRYTIMES`**
-
-Configure the number of task commit retries in `master-server`, default value `5`.
-
-**`MASTER_TASK_COMMIT_INTERVAL`**
-
-Configure the task commit interaction time in `master-server`, default value `1`.
-
-**`MASTER_MAX_CPULOAD_AVG`**
-
-Configure the `load average` value in the CPU in `master-server`, default value `-1`.
-
-**`MASTER_RESERVED_MEMORY`**
-
-Configure the reserved memory in G for `master-server`, default value `0.3`.
-
-### Worker Server
-
-**`WORKER_SERVER_OPTS`**
-
-Configure `jvm options` for `worker-server`, default value `-Xms1g -Xmx1g -Xmn512m`.
-
-**`WORKER_EXEC_THREADS`**
-
-Configure the number of threads to be executed in `worker-server`, default value `100`.
-
-**`WORKER_HEARTBEAT_INTERVAL`**
-
-Configure the heartbeat interaction time in `worker-server`, default value `10`.
-
-**`WORKER_MAX_CPULOAD_AVG`**
-
-Configure the maximum `load average` value in the CPU in `worker-server`, default value `-1`.
-
-**`WORKER_RESERVED_MEMORY`**
-
-Configure the reserved memory in G for `worker-server`, default value `0.3`.
-
-**`WORKER_GROUPS`**
-
-Configure the grouping of `worker-server`, default value `default`.
-
-### Alert Server
-
-**`ALERT_SERVER_OPTS`**
-
-Configure `jvm options` for `alert-server`, default value `-Xms512m -Xmx512m -Xmn256m`.
-
-**`XLS_FILE_PATH`**
-
-Configure the path to store `XLS` files for the `alert-server`, default value `/tmp/xls`.
-
-**`MAIL_SERVER_HOST`**
-
-Configure the mail service address for `alert-server`, default value `empty`.
-
-**`MAIL_SERVER_PORT`**
-
-Configure the mail service port for `alert-server`, default value `empty`.
-
-**`MAIL_SENDER`**
-
-Configure the mail sender for `alert-server`, default value `empty`.
-
-**`MAIL_USER=`**
-
-Configure the user name of the mail service for `alert-server`, default value `empty`.
-
-**`MAIL_PASSWD`**
-
-Configure the mail service user password for `alert-server`, default value `empty`.
-
-**`MAIL_SMTP_STARTTLS_ENABLE`**
-
-Configure whether TLS is enabled for `alert-server`'s mail service, default value `true`.
-
-**`MAIL_SMTP_SSL_ENABLE`**
-
-Configure whether the mail service of `alert-server` is SSL enabled or not, default value `false`.
-
-**`MAIL_SMTP_SSL_TRUST`**
-
-Configure the trusted address for SSL for `alert-server`'s mail service, default value `null`.
-
-**`ENTERPRISE_WECHAT_ENABLE`**
-
-Configure whether the mail service of `alert-server` has Enterprise Wechat enabled, default value `false`.
-
-**`ENTERPRISE_WECHAT_CORP_ID`**
-
-Configures the Enterprise Wechat `ID` of the mail service for `alert-server`, default value `null`.
-
-**`ENTERPRISE_WECHAT_SECRET`**
-
-Configure the mail service enterprise wechat `SECRET` for `alert-server`, default value `Empty`.
-
-**`ENTERPRISE_WECHAT_AGENT_ID`**
-
-Configure `AGENT_ID` of the mail service enterprise wechat for `alert-server`, default value `Empty`.
-
-**`ENTERPRISE_WECHAT_USERS`**
-
-Configure `USERS` for the mail service enterprise microsoft for `alert-server`, default value `empty`.
+
-### Api Server
+> Note: If you start the services by the way [using exists PostgreSQL ZooKeeper](#using-exists-postgresql-zookeeper), and
+> strating with multiple machine, you should change URL domain from `localhost` to IP or hostname the api server running.
-**`API_SERVER_OPTS`**
+## Change Environment Variable
-Configure `jvm options` for `api-server`, default value `-Xms512m -Xmx512m -Xmn256m`.
+You can modify some environment variables to change configurations when you are starting servers through Docker. We have
+an example in [using exists PostgreSQL ZooKeeper](#using-exists-postgresql-zookeeper) to change database and ZooKeeper configurations,
+and you could find all environment variables in [all environment variables](https://github.com/apache/dolphinscheduler/blob/3.0.0-alpha/script/env/dolphinscheduler_env.sh) <!-- markdown-link-check-disable-line -->
+and change them if you want.
diff --git a/docs/docs/en/guide/task/datax.md b/docs/docs/en/guide/task/datax.md
index 8f43510e20..bbe538b43b 100644
--- a/docs/docs/en/guide/task/datax.md
+++ b/docs/docs/en/guide/task/datax.md
@@ -40,7 +40,7 @@ This example demonstrates how to import data from Hive into MySQL.
### Configure the DataX environment in DolphinScheduler
-If you are using the DataX task type in a production environment, it is necessary to configure the required environment first. The following is the configuration file: `/dolphinscheduler/conf/env/dolphinscheduler_env.sh`.
+If you are using the DataX task type in a production environment, it is necessary to configure the required environment first. The following is the configuration file: `worker/bin/dolphinscheduler_env.sh`.

diff --git a/docs/docs/en/guide/task/flink.md b/docs/docs/en/guide/task/flink.md
index ab4f76b266..169e574a3d 100644
--- a/docs/docs/en/guide/task/flink.md
+++ b/docs/docs/en/guide/task/flink.md
@@ -46,13 +46,13 @@ This is a common introductory case in the big data ecosystem, which often apply
#### Configure the flink environment in DolphinScheduler
-If you are using the flink task type in a production environment, it is necessary to configure the required environment first. The following is the configuration file: `/dolphinscheduler/conf/env/dolphinscheduler_env.sh`.
+If you are using the flink task type in a production environment, it is necessary to configure the required environment first. The following is the configuration file: `worker/bin/dolphinscheduler_env.sh`.

#### Upload the Main Package
-When using the Flink task node, you need to upload the jar package to the Resource Centre for the execution, refer to the [resource center](../resource.md).
+When using the Flink task node, you need to upload the jar package to the Resource Center for the execution, refer to the [resource center](../resource.md).
After finish the Resource Centre configuration, upload the required target files directly by dragging and dropping.
diff --git a/docs/docs/en/guide/task/map-reduce.md b/docs/docs/en/guide/task/map-reduce.md
index 5ea69bab90..ed4eb48dfe 100644
--- a/docs/docs/en/guide/task/map-reduce.md
+++ b/docs/docs/en/guide/task/map-reduce.md
@@ -54,7 +54,7 @@ This example is a common introductory type of MapReduce application, which used
#### Configure the MapReduce Environment in DolphinScheduler
-If you are using the MapReduce task type in a production environment, it is necessary to configure the required environment first. The following is the configuration file: `/dolphinscheduler/conf/env/dolphinscheduler_env.sh`.
+If you are using the MapReduce task type in a production environment, it is necessary to configure the required environment first. The following is the configuration file: `worker/bin/dolphinscheduler_env.sh`.

diff --git a/docs/docs/en/guide/task/shell.md b/docs/docs/en/guide/task/shell.md
index a026d2f2a1..5f6a6411f5 100644
--- a/docs/docs/en/guide/task/shell.md
+++ b/docs/docs/en/guide/task/shell.md
@@ -41,3 +41,8 @@ we declare a custom parameter named "param_key", with the value "param_val". The
After running this example, we would see "param_val" print in the log.

+
+## Attention
+The shell task type resolves whether the task log contains ```application_xxx_xxx``` to determine whether is the yarn task. If so, the corresponding application
+will be use to judge the running state of the current shell node. At this time, if stops the operation of the workflow, the corresponding ```application_id```
+will be killed.
\ No newline at end of file
diff --git a/docs/docs/en/guide/task/spark.md b/docs/docs/en/guide/task/spark.md
index 1e9f31524f..0be36403d4 100644
--- a/docs/docs/en/guide/task/spark.md
+++ b/docs/docs/en/guide/task/spark.md
@@ -45,7 +45,7 @@ This is a common introductory case in the big data ecosystem, which often apply
#### Configure the Spark Environment in DolphinScheduler
-If you are using the Spark task type in a production environment, it is necessary to configure the required environment first. The following is the configuration file: `/dolphinscheduler/conf/env/dolphinscheduler_env.sh`.
+If you are using the Spark task type in a production environment, it is necessary to configure the required environment first. The following is the configuration file: `worker/bin/dolphinscheduler_env.sh`.

diff --git a/docs/docs/en/guide/upgrade.md b/docs/docs/en/guide/upgrade.md
index 38f80d8c73..8cd9c2c6dc 100644
--- a/docs/docs/en/guide/upgrade.md
+++ b/docs/docs/en/guide/upgrade.md
@@ -13,24 +13,20 @@
## Database Upgrade
-- Modify the following properties in `conf/datasource.properties`.
-
-- If using MySQL as the database to run DolphinScheduler, please comment out PostgreSQL related configurations, and add MYSQL connector jar into lib dir, here we download `mysql-connector-java-8.0.16.jar`, and then correctly configure database connection information. You can download MYSQL connector jar from [here](https://downloads.MySQL.com/archives/c-j/). Alternatively, if you use PostgreSQL as the database, you just need to comment out Mysql related configurations and correctly confi [...]
-
- ```properties
- # postgre
- #spring.datasource.driver-class-name=org.postgresql.Driver
- #spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler
- # mysql
- spring.datasource.driver-class-name=com.mysql.jdbc.Driver
- spring.datasource.url=jdbc:mysql://xxx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true
- spring.datasource.username=xxx
- spring.datasource.password=xxx
+- If using MySQL as the database to run DolphinScheduler, please config it in `tools/bin/dolphinscheduler_env.sh`, change username and password to yours, and add MYSQL connector jar into lib dir `./tools/libs`, here we download `mysql-connector-java-8.0.16.jar`, and then correctly configure database connection information. You can download MYSQL connector jar from [here](https://downloads.MySQL.com/archives/c-j/). Otherwise, PostgreSQL is the default database.
+
+ ```shell
+ export DATABASE=${DATABASE:-mysql}
+ export SPRING_PROFILES_ACTIVE=${DATABASE}
+ export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver
+ export SPRING_DATASOURCE_URL=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
+ export SPRING_DATASOURCE_USERNAME={user}
+ export SPRING_DATASOURCE_PASSWORD={password}
```
- Execute database upgrade script:
- `sh ./script/upgrade-dolphinscheduler.sh`
+ `sh ./tools/bin/upgrade-schema.sh`
## Backend Service Upgrade
diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md
index ceb99a21b8..7eba76ea2c 100644
--- a/docs/docs/zh/architecture/configuration.md
+++ b/docs/docs/zh/architecture/configuration.md
@@ -380,21 +380,42 @@ apiServers="ds1"
```
## 11.dolphinscheduler_env.sh [环境变量配置]
-通过类似shell方式提交任务的的时候,会加载该配置文件中的环境变量到主机中.
-涉及到的任务类型有: Shell任务、Python任务、Spark任务、Flink任务、Datax任务等等
-```bash
-export HADOOP_HOME=/opt/soft/hadoop
-export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
-export SPARK_HOME1=/opt/soft/spark1
-export SPARK_HOME2=/opt/soft/spark2
-export PYTHON_HOME=/opt/soft/python
-export JAVA_HOME=/opt/soft/java
-export HIVE_HOME=/opt/soft/hive
-export FLINK_HOME=/opt/soft/flink
-export DATAX_HOME=/opt/soft/datax/bin/datax.py
-
-export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
+通过类似shell方式提交任务的的时候,会加载该配置文件中的环境变量到主机中. 涉及到的 `JAVA_HOME`、元数据库、注册中心和任务类型配置,其中任务
+类型主要有: Shell任务、Python任务、Spark任务、Flink任务、Datax任务等等
+
+```bash
+# JAVA_HOME, will use it to start DolphinScheduler server
+export JAVA_HOME=${JAVA_HOME:-/opt/soft/java}
+
+# Database related configuration, set database type, username and password
+export DATABASE=${DATABASE:-postgresql}
+export SPRING_PROFILES_ACTIVE=${DATABASE}
+export SPRING_DATASOURCE_DRIVER_CLASS_NAME
+export SPRING_DATASOURCE_URL
+export SPRING_DATASOURCE_USERNAME
+export SPRING_DATASOURCE_PASSWORD
+
+# DolphinScheduler server related configuration
+export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
+export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
+export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
+
+# Registry center configuration, determines the type and link of the registry center
+export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
+export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-localhost:2181}
+
+# Tasks related configurations, need to change the configuration if you use the related tasks.
+export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
+export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1}
+export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
+export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
+export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
+export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
+export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
+
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
```
## 12.各服务日志配置文件
diff --git a/docs/docs/zh/faq.md b/docs/docs/zh/faq.md
index 8d152aae40..ca7fea50c1 100644
--- a/docs/docs/zh/faq.md
+++ b/docs/docs/zh/faq.md
@@ -203,7 +203,7 @@ A: 1,在 **流程定义列表**,点击 **启动** 按钮
## Q:Python 任务设置 Python 版本
-A: 只需要修改 conf/env/dolphinscheduler_env.sh 中的 PYTHON_HOME
+A: 只需要修改 `bin/env/dolphinscheduler_env.sh` 中的 PYTHON_HOME
```
export PYTHON_HOME=/bin/python
@@ -290,10 +290,9 @@ A: 1,参考官网[部署文档](https://dolphinscheduler.apache.org/zh-cn/do
3,复制正在运行的服务器上的部署目录到新机器的同样的部署目录下
- 4,到 bin 下,启动 worker server 和 logger server
+ 4,到 bin 下,启动 worker server
```
./dolphinscheduler-daemon.sh start worker-server
- ./dolphinscheduler-daemon.sh start logger-server
```
---
@@ -582,7 +581,7 @@ A:将Worker节点分别部署至多个Yarn集群,步骤如下(例如AWS EM
2. 将 `conf/common.properties` 中的 `yarn.application.status.address` 修改为当前集群的 Yarn 的信息
- 3. 通过 `bin/dolphinscheduler-daemon.sh start worker-server` 和 `bin/dolphinscheduler-daemon.sh start logger-server` 分别启动 worker-server 和 logger-server
+ 3. 通过 `bin/dolphinscheduler-daemon.sh start worker-server` 启动 worker-server
---
@@ -683,4 +682,38 @@ update t_ds_version set version='2.0.1';
---
+## 在二进制分发包中找不到 python-gateway-server 文件夹
+
+在 3.0.0-alpha 版本之后,Python gateway server 集成到 api server 中,当您启动 api server 后,Python gateway server 将启动。
+如果您不想在 api server 启动的时候启动 Python gateway server,您可以修改 api server 中的配置文件 `api-server/conf/application.yaml`
+并更改可选项 `python-gateway.enabled` 中的值设置为 `false`。
+
+## 如果构建自定义的 Docker 镜像
+
+DolphinScheduler 每次发版都会同时发布 Docker 镜像,你可以在 docker hub 中找到这些镜像,如果你因为个性化需求想要自己打包 docker 镜像,最佳实践是基于 DolphinScheduler 对应镜像编写 Dockerfile 文件
+
+```Dockerfile
+FROM dolphinscheduler-standalone-server
+RUN apt update ; \
+ apt install -y <YOUR-CUSTOM-DEPENDENCE> ; \
+```
+
+如果你想基于源码进行改造,打包并分发你的镜像,可以在代码改造完成后运行
+
+```shell
+./mvnw -B clean deploy \
+ -Dmaven.test.skip \
+ -Dmaven.javadoc.skip \
+ -Dmaven.checkstyle.skip \
+ -Dmaven.deploy.skip \
+ -Ddocker.tag=latest \
+ -Pdocker,release
+```
+
+如果你不仅需要改造源码,还想要自定义 Docker 镜像打包的依赖,可以在修改源码的同时修改 Dockerfile 的定义,你可以在源码项目根目录中运行以下命令找到所有的 Dockerfile 文件
+
+```shell
+find . -iname 'Dockerfile'
+```
+
我们会持续收集更多的 FAQ。
\ No newline at end of file
diff --git a/docs/docs/zh/guide/alert/alert_plugin_user_guide.md b/docs/docs/zh/guide/alert/alert_plugin_user_guide.md
index 03e00f35f0..065adc4dd0 100644
--- a/docs/docs/zh/guide/alert/alert_plugin_user_guide.md
+++ b/docs/docs/zh/guide/alert/alert_plugin_user_guide.md
@@ -1,6 +1,10 @@
## 如何创建告警插件以及告警组
-在2.0.0版本中,用户需要创建告警实例,然后同告警组进行关联,一个告警组可以使用多个告警实例,我们会逐一进行进行告警通知。
+在2.0.0版本中,用户需要创建告警实例,在创建告警实例时,需要选择告警策略,有三个选项,成功发、失败发,以及成功和失败都发。在执行完工作流或任务时,如果触发告警,调用告警实例发送方法会进行逻辑判断,将告警实例与任务状态进行匹配,匹配则执行该告警实例发送逻辑,不匹配则过滤。创建完告警实例后,需要同告警组进行关联,一个告警组可以使用多个告警实例。
+告警模块支持场景如下:
+<img src="/img/alert/alert_scenarios_zh.png">
+
+使用步骤如下:
首先需要进入到安全中心,选择告警组管理,然后点击左侧的告警实例管理,然后创建一个告警实例,然后选择对应的告警插件,填写相关告警参数。
@@ -9,4 +13,4 @@
<img src="/img/alert/alert_step_1.png">
<img src="/img/alert/alert_step_2.png">
<img src="/img/alert/alert_step_3.png">
-<img src="/img/alert/alert_step_4.png">
\ No newline at end of file
+<img src="/img/alert/alert_step_4.png">
diff --git a/docs/docs/zh/guide/alert/enterprise-wechat.md b/docs/docs/zh/guide/alert/enterprise-wechat.md
index e79c322afe..263a44a257 100644
--- a/docs/docs/zh/guide/alert/enterprise-wechat.md
+++ b/docs/docs/zh/guide/alert/enterprise-wechat.md
@@ -1,13 +1,69 @@
# 企业微信
-如果您需要使用到企业微信进行告警,请在告警实例管理里创建告警实例,选择 WeChat 插件。企业微信的配置样例如下
+如果您需要使用到企业微信进行告警,请在告警实例管理里创建告警实例,选择 WeChat 插件。企业微信的配置样例如下:

-其中 send.type 分别对应企微文档:
+## 发送类型
+
+其中`send.type`分别对应向企业微信自定义应用发送和向企业微信API创建的群聊发送消息。
+
+### 应用
+
+应用指将告警结果通过企业微信的自定义应用进行通知,支持向特定用户发送消息和对所有人发送消息。目前还不支持部门和标签,欢迎提PR贡献代码。
+下图是应用告警配置的示例:
+
+
+
+下图是`应用``MARKDOWN`告警消息的示例:
+
+
+
+下图是`应用``TEXT`告警消息的示例:
+
+
+
+#### 前置
+
+向企业微信应用发送消息之前需要在企业微信中创建自定义应用,请在[应用页面](https://work.weixin.qq.com/wework_admin/frame#apps) 进行创建,获取应用的`AgentId`并将可见范围设为根。
+
+#### 向指定用户发消息
+
+企业微信应用支持向特定用户发送消息和对所有人发送消息,分别为使用`|`分隔多个userId和使用`@all`向所有人发送信息。
+获取用户的userId请参考[官方文档](https://developer.work.weixin.qq.com/document/path/95402)根据手机号获取userId。
+下图是获取userId接口的示例:
+
+
+
+#### 参考文档
应用:https://work.weixin.qq.com/api/doc/90000/90135/90236
-群聊:https://work.weixin.qq.com/api/doc/90000/90135/90248
+### 群聊
+
+群聊指将告警结果通过企业微信API创建的群聊进行通知,会向该群聊下的所有人发送消息,不支持向特定用户发送消息。
+下图是群聊告警配置的示例:
+
+
+
+下图是`群聊``MARKDOWN`告警消息的示例:
+
+
+
+下图是`群聊``TEXT`告警消息的示例:
+
+
+
+#### 前置
+
+向企业微信群聊发送消息之前需要通过企业微信的API创建群聊,请参考[官方文档](https://developer.work.weixin.qq.com/document/path/90245) 进行创建群聊并获取`chatid`。
+其中获取用户的userId请参考[官方文档](https://developer.work.weixin.qq.com/document/path/95402)根据手机号获取userId。
+下图是创建新聊天群组和获取userId接口的示例:
+
+
+
+
+
+#### 参考文档
-user.send.msg 对应文档中的 content,与此相对应的值的变量为 {msg}
\ No newline at end of file
+群聊:https://work.weixin.qq.com/api/doc/90000/90135/90248
\ No newline at end of file
diff --git a/docs/docs/zh/guide/alert/http.md b/docs/docs/zh/guide/alert/http.md
new file mode 100644
index 0000000000..dd124df95b
--- /dev/null
+++ b/docs/docs/zh/guide/alert/http.md
@@ -0,0 +1,34 @@
+# HTTP告警
+
+如果您需要使用到`Http`(GET或POST)进行告警,请在告警实例管理里创建告警实例,选择`Http`插件。
+
+## 参数配置
+
+* URL
+ > 访问的`Http`连接URL,需要包含协议、Host、路径,如果是GET方法可以添加参数
+* 请求方式
+ > 选择该请求为POST或GET方法
+* 请求头
+ > `Http`请求的完整请求头,以JSON为格式
+* 请求体
+ > `Http`请求的完整请求体,以JSON为格式,GET方法不需要写该参数
+* 内容字段
+ > 放置本次告警告警信息的字段名称
+
+## 发送类型
+
+其中`Request Type`分别对应使用`POST`方法和`GET`方法进行`Http`告警。
+
+### GET Http告警
+
+GET `Http`告警指将告警结果作为参数通过`Http` GET方法进行请求。
+下图是GET告警配置的示例:
+
+
+
+### POST Http告警
+
+POST `Http`告警指将告警结果作为`BODY`参数通过`Http`POST方法进行请求。
+下图是POST告警配置的示例:
+
+
diff --git a/docs/docs/zh/guide/alert/script.md b/docs/docs/zh/guide/alert/script.md
new file mode 100644
index 0000000000..41be752ed4
--- /dev/null
+++ b/docs/docs/zh/guide/alert/script.md
@@ -0,0 +1,17 @@
+# Script脚本
+
+如果您需要使用到`Shell`脚本进行告警,请在告警实例管理里创建告警实例,选择`Script`插件。`Script`的配置样例如下:
+
+
+
+参数配置
+
+* 自定义参数
+ > 用户自定义的参数将被传入脚本执行
+* 脚本路径
+ > 脚本在服务器上的文件位置
+* 脚本类型
+ > 支持`Shell`脚本
+
+**_注意:_** 请注意脚本的读写权限与执行租户的关系
+
diff --git a/docs/docs/zh/guide/data-quality.md b/docs/docs/zh/guide/data-quality.md
new file mode 100644
index 0000000000..89fc4e855f
--- /dev/null
+++ b/docs/docs/zh/guide/data-quality.md
@@ -0,0 +1,313 @@
+# 概述
+## 任务类型介绍
+
+数据质量任务是用于检查数据在集成、处理过程中的数据准确性。本版本的数据质量任务包括单表检查、单表自定义SQL检查、多表准确性以及两表值比对。数据质量任务的运行环境为Spark2.4.0,其他版本尚未进行过验证,用户可自行验证。
+
+- 数据质量任务的执行逻辑如下:
+
+> 用户在界面定义任务,用户输入值保存在`TaskParam`中
+运行任务时,`Master`会解析`TaskParam`,封装`DataQualityTask`所需要的参数下发至`Worker。
+Worker`运行数据质量任务,数据质量任务在运行结束之后将统计结果写入到指定的存储引擎中,当前数据质量任务结果存储在`dolphinscheduler`的`t_ds_dq_execute_result`表中
+`Worker`发送任务结果给`Master`,`Master`收到`TaskResponse`之后会判断任务类型是否为`DataQualityTask`,如果是的话会根据`taskInstanceId`从`t_ds_dq_execute_result`中读取相应的结果,然后根据用户配置好的检查方式,操作符和阈值进行结果判断,如果结果为失败的话,会根据用户配置好的的失败策略进行相应的操作,告警或者中断
+## 注意事项
+
+添加配置信息:`<server-name>/conf/common.properties`
+
+```properties
+data-quality.jar.name=dolphinscheduler-data-quality-dev-SNAPSHOT.jar
+```
+
+这里的`data-quality.jar.name`请根据实际打包的名称来填写,
+如果单独打包`data-quality`的话,记得修改包名和`data-quality.jar.name`一致。
+如果是老版本升级使用,运行之前需要先执行`sql`更新脚本进行数据库初始化。
+如果要用到`MySQL`数据,需要将`pom.xml`中`MySQL`的`scope`注释掉
+当前只测试了`MySQL`、`PostgreSQL`和`HIVE`数据源,其他数据源暂时未测试过
+`Spark`需要配置好读取`Hive`元数据,`Spark`不是采用`jdbc`的方式读取`Hive`
+
+## 检查逻辑详解
+
+- 校验公式:[校验方式][操作符][阈值],如果结果为真,则表明数据不符合期望,执行失败策略
+- 校验方式:
+ - [Expected-Actual][期望值-实际值]
+ - [Actual-Expected][实际值-期望值]
+ - [Actual/Expected][实际值/期望值]x100%
+ - [(Expected-Actual)/Expected][(期望值-实际值)/期望值]x100%
+- 操作符:=、>、>=、<、<=、!=
+- 期望值类型
+ - 固定值
+ - 日均值
+ - 周均值
+ - 月均值
+ - 最近7天均值
+ - 最近30天均值
+ - 源表总行数
+ - 目标表总行数
+
+- 例子
+ - 校验方式为:[Expected-Actual][期望值-实际值]
+ - [操作符]:>
+ - [阈值]:0
+ - 期望值类型:固定值=9。
+
+ 假设实际值为10,操作符为 >, 期望值为9,那么结果 10 -9 > 0 为真,那就意味列为空的行数据已经超过阈值,任务被判定为失败
+
+# 任务操作指南
+## 单表检查之空值检查
+### 检查介绍
+空值检查的目标是检查出指定列为空的行数,可将为空的行数与总行数或者指定阈值进行比较,如果大于某个阈值则判定为失败
+- 计算指定列为空的SQL语句如下:
+
+ ```sql
+ SELECT COUNT(*) AS miss FROM ${src_table} WHERE (${src_field} is null or ${src_field} = '') AND (${src_filter})
+ ```
+
+- 计算表总行数的SQL如下:
+
+ ```sql
+ SELECT COUNT(*) AS total FROM ${src_table} WHERE (${src_filter})
+ ```
+
+### 界面操作指南
+
+- 源数据类型:选择MySQL、PostgreSQL等
+- 源数据源:源数据类型下对应的数据源
+- 源数据表:下拉选择验证数据所在表
+- 源过滤条件:如标题,统计表总行数的时候也会用到,选填
+- 源表检查列:下拉选择检查列名
+- 校验方式:
+ - [Expected-Actual][期望值-实际值]
+ - [Actual-Expected][实际值-期望值]
+ - [Actual/Expected][实际值/期望值]x100%
+ - [(Expected-Actual)/Expected][(期望值-实际值)/期望值]x100%
+- 校验操作符:=,>、>=、<、<=、!=
+- 阈值:公式中用于比较的值
+- 失败策略
+ - 告警:数据质量任务失败了,DolphinScheduler任务结果为成功,发送告警
+ - 阻断:数据质量任务失败了,DolphinScheduler任务结果为失败,发送告警
+- 期望值类型:在下拉菜单中选择所要的类型
+
+## 单表检查之及时性检查
+### 检查介绍
+及时性检查用于检查数据是否在预期时间内处理完成,可指定开始时间、结束时间来界定时间范围,如果在该时间范围内的数据量没有达到设定的阈值,那么会判断该检查任务为失败
+### 界面操作指南
+
+- 源数据类型:选择MySQL、PostgreSQL等
+- 源数据源:源数据类型下对应的数据源
+- 源数据表:下拉选择验证数据所在表
+- 源过滤条件:如标题,统计表总行数的时候也会用到,选填
+- 源表检查列:下拉选择检查列名
+- 起始时间:某个时间范围的开始时间
+- 结束时间:某个时间范围的结束时间
+- 时间格式:设置对应的时间格式
+- 校验方式:
+ - [Expected-Actual][期望值-实际值]
+ - [Actual-Expected][实际值-期望值]
+ - [Actual/Expected][实际值/期望值]x100%
+ - [(Expected-Actual)/Expected][(期望值-实际值)/期望值]x100%
+- 校验操作符:=,>、>=、<、<=、!=
+- 阈值:公式中用于比较的值
+- 失败策略
+ - 告警:数据质量任务失败了,DolphinScheduler任务结果为成功,发送告警
+ - 阻断:数据质量任务失败了,DolphinScheduler任务结果为失败,发送告警
+- 期望值类型:在下拉菜单中选择所要的类型
+
+## 单表检查之字段长度校验
+### 检查介绍
+字段长度校验的目标是检查所选字段的长度是否满足预期,如果有存在不满足要求的数据,并且行数超过阈值则会判断任务为失败
+### 界面操作指南
+
+- 源数据类型:选择MySQL、PostgreSQL等
+- 源数据源:源数据类型下对应的数据源
+- 源数据表:下拉选择验证数据所在表
+- 源过滤条件:如标题,统计表总行数的时候也会用到,选填
+- 源表检查列:下拉选择检查列名
+- 逻辑操作符:=,>、>=、<、<=、!=
+- 字段长度限制:如标题
+- 校验方式:
+ - [Expected-Actual][期望值-实际值]
+ - [Actual-Expected][实际值-期望值]
+ - [Actual/Expected][实际值/期望值]x100%
+ - [(Expected-Actual)/Expected][(期望值-实际值)/期望值]x100%
+- 校验操作符:=,>、>=、<、<=、!=
+- 阈值:公式中用于比较的值
+- 失败策略
+ - 告警:数据质量任务失败了,DolphinScheduler任务结果为成功,发送告警
+ - 阻断:数据质量任务失败了,DolphinScheduler任务结果为失败,发送告警
+- 期望值类型:在下拉菜单中选择所要的类型
+
+## 单表检查之唯一性校验
+### 检查介绍
+唯一性校验的目标是检查字段是否存在重复的情况,一般用于检验primary key是否有重复,如果存在重复且达到阈值,则会判断检查任务为失败
+### 界面操作指南
+
+- 源数据类型:选择MySQL、PostgreSQL等
+- 源数据源:源数据类型下对应的数据源
+- 源数据表:下拉选择验证数据所在表
+- 源过滤条件:如标题,统计表总行数的时候也会用到,选填
+- 源表检查列:下拉选择检查列名
+- 校验方式:
+ - [Expected-Actual][期望值-实际值]
+ - [Actual-Expected][实际值-期望值]
+ - [Actual/Expected][实际值/期望值]x100%
+ - [(Expected-Actual)/Expected][(期望值-实际值)/期望值]x100%
+- 校验操作符:=,>、>=、<、<=、!=
+- 阈值:公式中用于比较的值
+- 失败策略
+ - 告警:数据质量任务失败了,DolphinScheduler任务结果为成功,发送告警
+ - 阻断:数据质量任务失败了,DolphinScheduler任务结果为失败,发送告警
+- 期望值类型:在下拉菜单中选择所要的类型
+
+## 单表检查之正则表达式校验
+### 检查介绍
+正则表达式校验的目标是检查某字段的值的格式是否符合要求,例如时间格式、邮箱格式、身份证格式等等,如果存在不符合格式的数据并超过阈值,则会判断任务为失败
+### 界面操作指南
+
+- 源数据类型:选择MySQL、PostgreSQL等
+- 源数据源:源数据类型下对应的数据源
+- 源数据表:下拉选择验证数据所在表
+- 源过滤条件:如标题,统计表总行数的时候也会用到,选填
+- 源表检查列:下拉选择检查列名
+- 正则表达式:如标题
+- 校验方式:
+ - [Expected-Actual][期望值-实际值]
+ - [Actual-Expected][实际值-期望值]
+ - [Actual/Expected][实际值/期望值]x100%
+ - [(Expected-Actual)/Expected][(期望值-实际值)/期望值]x100%
+- 校验操作符:=,>、>=、<、<=、!=
+- 阈值:公式中用于比较的值
+- 失败策略
+ - 告警:数据质量任务失败了,DolphinScheduler任务结果为成功,发送告警
+ - 阻断:数据质量任务失败了,DolphinScheduler任务结果为失败,发送告警
+- 期望值类型:在下拉菜单中选择所要的类型
+
+## 单表检查之枚举值校验
+### 检查介绍
+枚举值校验的目标是检查某字段的值是否在枚举值的范围内,如果存在不在枚举值范围里的数据并超过阈值,则会判断任务为失败
+### 界面操作指南
+
+- 源数据类型:选择MySQL、PostgreSQL等
+- 源数据源:源数据类型下对应的数据源
+- 源数据表:下拉选择验证数据所在表
+- 源表过滤条件:如标题,统计表总行数的时候也会用到,选填
+- 源表检查列:下拉选择检查列名
+- 枚举值列表:用英文逗号,隔开
+- 校验方式:
+ - [Expected-Actual][期望值-实际值]
+ - [Actual-Expected][实际值-期望值]
+ - [Actual/Expected][实际值/期望值]x100%
+ - [(Expected-Actual)/Expected][(期望值-实际值)/期望值]x100%
+- 校验操作符:=,>、>=、<、<=、!=
+- 阈值:公式中用于比较的值
+- 失败策略
+ - 告警:数据质量任务失败了,DolphinScheduler任务结果为成功,发送告警
+ - 阻断:数据质量任务失败了,DolphinScheduler任务结果为失败,发送告警
+- 期望值类型:在下拉菜单中选择所要的类型
+## 单表检查之表行数校验
+### 检查介绍
+表行数校验的目标是检查表的行数是否达到预期的值,如果行数未达标,则会判断任务为失败
+### 界面操作指南
+
+- 源数据类型:选择MySQL、PostgreSQL等
+- 源数据源:源数据类型下对应的数据源
+- 源数据表:下拉选择验证数据所在表
+- 源过滤条件:如标题,统计表总行数的时候也会用到,选填
+- 源表检查列:下拉选择检查列名
+- 校验方式:
+ - [Expected-Actual][期望值-实际值]
+ - [Actual-Expected][实际值-期望值]
+ - [Actual/Expected][实际值/期望值]x100%
+ - [(Expected-Actual)/Expected][(期望值-实际值)/期望值]x100%
+- 校验操作符:=,>、>=、<、<=、!=
+- 阈值:公式中用于比较的值
+- 失败策略
+ - 告警:数据质量任务失败了,DolphinScheduler任务结果为成功,发送告警
+ - 阻断:数据质量任务失败了,DolphinScheduler任务结果为失败,发送告警
+- 期望值类型:在下拉菜单中选择所要的类型
+
+## 单表检查之自定义SQL检查
+### 检查介绍
+### 界面操作指南
+
+- 源数据类型:选择MySQL、PostgreSQL等
+- 源数据源:源数据类型下对应的数据源
+- 源数据表:下拉选择要验证数据所在表
+- 实际值名:为统计值计算SQL中的别名,如max_num
+- 实际值计算SQL: 用于输出实际值的SQL、
+ - 注意点:该SQL必须为统计SQL,例如统计行数,计算最大值、最小值等
+ - select max(a) as max_num from ${src_table},表名必须这么填
+- 源过滤条件:如标题,统计表总行数的时候也会用到,选填
+- 校验方式:
+- 校验操作符:=,>、>=、<、<=、!=
+- 阈值:公式中用于比较的值
+- 失败策略
+ - 告警:数据质量任务失败了,DolphinScheduler任务结果为成功,发送告警
+ - 阻断:数据质量任务失败了,DolphinScheduler任务结果为失败,发送告警
+- 期望值类型:在下拉菜单中选择所要的类型
+
+## 多表检查之准确性检查
+### 检查介绍
+准确性检查是通过比较两个表之间所选字段的数据记录的准确性差异,例子如下
+- 表test1
+
+| c1 | c2 |
+| :---: | :---: |
+| a | 1 |
+| b | 2|
+
+- 表test2
+
+| c21 | c22 |
+| :---: | :---: |
+| a | 1 |
+| b | 3|
+如果对比c1和c21中的数据,则表test1和test2完全一致。 如果对比c2和c22则表test1和表test2中的数据则存在不一致了。
+### 界面操作指南
+
+- 源数据类型:选择MySQL、PostgreSQL等
+- 源数据源:源数据类型下对应的数据源
+- 源数据表:下拉选择要验证数据所在表
+- 源过滤条件:如标题,统计表总行数的时候也会用到,选填
+- 目标数据类型:选择MySQL、PostgreSQL等
+- 目标数据源:源数据类型下对应的数据源
+- 目标数据表:下拉选择要验证数据所在表
+- 目标过滤条件:如标题,统计表总行数的时候也会用到,选填
+- 检查列:
+ - 分别填写 源数据列,操作符,目标数据列
+- 校验方式:选择想要的校验方式
+- 操作符:=,>、>=、<、<=、!=
+- 失败策略
+ - 告警:数据质量任务失败了,DolphinScheduler任务结果为成功,发送告警
+ - 阻断:数据质量任务失败了,DolphinScheduler任务结果为失败,发送告警
+- 期望值类型:在下拉菜单中选择所要的类型,这里只适合选择SrcTableTotalRow、TargetTableTotalRow和固定值
+## 两表检查之值比对
+### 检查介绍
+两表值比对允许用户对两张表自定义不同的SQL统计出相应的值进行比对,例如针对源表A统计出某一列的金额总值sum1,针对目标表统计出某一列的金额总值sum2,将sum1和sum2进行比较来判定检查结果
+### 界面操作指南
+
+- 源数据类型:选择MySQL、PostgreSQL等
+- 源数据源:源数据类型下对应的数据源
+- 源数据表:要验证数据所在表
+- 实际值名:为实际值计算SQL中的别名,如max_age1
+- 实际值计算SQL: 用于输出实际值的SQL、
+ - 注意点:该SQL必须为统计SQL,例如统计行数,计算最大值、最小值等
+ - select max(age) as max_age1 from ${src_table} 表名必须这么填
+- 目标数据类型:选择MySQL、PostgreSQL等
+- 目标数据源:源数据类型下对应的数据源
+- 目标数据表:要验证数据所在表
+- 期望值名:为期望值计算SQL中的别名,如max_age2
+- 期望值计算SQL: 用于输出期望值的SQL、
+ - 注意点:该SQL必须为统计SQL,例如统计行数,计算最大值、最小值等
+ - select max(age) as max_age2 from ${target_table} 表名必须这么填
+- 校验方式:选择想要的校验方式
+- 操作符:=,>、>=、<、<=、!=
+- 失败策略
+ - 告警:数据质量任务失败了,DolphinScheduler任务结果为成功,发送告警
+ - 阻断:数据质量任务失败了,DolphinScheduler任务结果为失败,发送告警
+
+## 任务结果查看
+
+## 规则查看
+### 规则列表
+
+### 规则详情
+
\ No newline at end of file
diff --git a/docs/docs/zh/guide/datasource/hive.md b/docs/docs/zh/guide/datasource/hive.md
index c7df33542e..6598ba81b5 100644
--- a/docs/docs/zh/guide/datasource/hive.md
+++ b/docs/docs/zh/guide/datasource/hive.md
@@ -38,3 +38,6 @@ login.user.keytab.username=hdfs-mycluster@ESZ.COM
login.user.keytab.path=/opt/hdfs.headless.keytab
```
+## 是否原生支持
+
+是,数据源不需要任务附加操作即可使用。
diff --git a/docs/docs/zh/guide/datasource/introduction.md b/docs/docs/zh/guide/datasource/introduction.md
index 77e91bc55c..f5cd2d6db0 100644
--- a/docs/docs/zh/guide/datasource/introduction.md
+++ b/docs/docs/zh/guide/datasource/introduction.md
@@ -2,5 +2,20 @@
数据源中心支持MySQL、POSTGRESQL、HIVE/IMPALA、SPARK、CLICKHOUSE、ORACLE、SQLSERVER等数据源。
-- 点击“数据源中心->创建数据源”,根据需求创建不同类型的数据源。
-- 点击“测试连接”,测试数据源是否可以连接成功。
\ No newline at end of file
+- 点击"数据源中心->创建数据源",根据需求创建不同类型的数据源
+- 点击"测试连接",测试数据源是否可以连接成功(只有当数据源通过连接性测试后才能保存数据源)。
+
+## 使用不兼容 Apache LICENSE V2 许可的数据库
+
+数据源中心里,DolphinScheduler 对部分数据源有原生的支持,但是部分数据源需要用户下载对应的 JDBC 驱动包并放置到正确的位置才能正常使用。
+这对用户会增加用户的使用成本,但是我们不得不这么做,因为这部分数据源的 JDBC 驱动和 Apache LICENSE V2 不兼容,所以我们无法在
+DolphinScheduler 分发的二进制包中包含他们。这部分数据源主要包括 MySQL,Oracle,SQL Server 等,幸运的是我们为这部分数据源的支持给出了解决方案。
+
+### 样例
+
+我们以 MySQL 为例,如果你想要使用 MySQL 数据源,你需要先在 [mysql maven 仓库](https://repo1.maven.org/maven2/mysql/mysql-connector-java)
+中下载对应版本的 JDBC 驱动,将其移入 `api-server/libs` 以及 `worker-server/libs` 文件夹中,最后重启 `api-server` 和 `worker-server`
+服务,即可使用 MySQL 数据源。如果你使用容器启动 DolphinScheduler,同样也是将 JDBC 驱动挂载放到以上两个服务的对应路径下后,重启驱动即可。
+
+> 注意:如果你只是想要在数据源中心使用 MySQL,则对 MySQL JDBC 驱动的版本没有要求,如果你想要将 MySQL 作为 DolphinScheduler 的元数据库,
+> 则仅支持 [8.0.16 及以上](https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar)的版本。
diff --git a/docs/docs/zh/guide/datasource/mysql.md b/docs/docs/zh/guide/datasource/mysql.md
index 0e9ef4a07f..3727dac78e 100644
--- a/docs/docs/zh/guide/datasource/mysql.md
+++ b/docs/docs/zh/guide/datasource/mysql.md
@@ -11,3 +11,7 @@
- 密码:设置连接 MySQL 的密码
- 数据库名:输入连接 MySQL 的数据库名称
- Jdbc 连接参数:用于 MySQL 连接的参数设置,以 JSON 形式填写
+
+## 是否原生支持
+
+否,使用前需请参考[简介](introduction.md)中的 "样例" 章节激活数据源。
diff --git a/docs/docs/zh/guide/datasource/postgresql.md b/docs/docs/zh/guide/datasource/postgresql.md
index ec3fd11a31..a0a8143641 100644
--- a/docs/docs/zh/guide/datasource/postgresql.md
+++ b/docs/docs/zh/guide/datasource/postgresql.md
@@ -11,3 +11,7 @@
- 密码:设置连接 POSTGRESQL 的密码
- 数据库名:输入连接 POSTGRESQL 的数据库名称
- Jdbc 连接参数:用于 POSTGRESQL 连接的参数设置,以 JSON 形式填写
+
+## 是否原生支持
+
+是,数据源不需要任务附加操作即可使用。
diff --git a/docs/docs/zh/guide/datasource/spark.md b/docs/docs/zh/guide/datasource/spark.md
index 9b9702890d..946ff01101 100644
--- a/docs/docs/zh/guide/datasource/spark.md
+++ b/docs/docs/zh/guide/datasource/spark.md
@@ -17,3 +17,7 @@
<p align="center">
<img src="/img/sparksql_kerberos.png" width="80%" />
</p>
+
+## 是否原生支持
+
+是,数据源不需要任务附加操作即可使用。
diff --git a/docs/docs/zh/guide/expansion-reduction.md b/docs/docs/zh/guide/expansion-reduction.md
index f3dae4dd64..8f92ef4d4d 100644
--- a/docs/docs/zh/guide/expansion-reduction.md
+++ b/docs/docs/zh/guide/expansion-reduction.md
@@ -71,7 +71,7 @@ sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
datasource.properties 中的数据库连接信息.
zookeeper.properties 中的连接zk的信息.
common.properties 中关于资源存储的配置信息(如果设置了hadoop,请检查是否存在core-site.xml和hdfs-site.xml配置文件).
- env/dolphinscheduler_env.sh 中的环境变量
+ dolphinscheduler_env.sh 中的环境变量
````
- 根据机器配置,修改 conf/env 目录下的 `dolphinscheduler_env.sh` 环境变量(以相关用到的软件都安装在/opt/soft下为例)
diff --git a/docs/docs/zh/guide/howto/general-setting.md b/docs/docs/zh/guide/howto/general-setting.md
index 1b7744b3f9..1533575723 100644
--- a/docs/docs/zh/guide/howto/general-setting.md
+++ b/docs/docs/zh/guide/howto/general-setting.md
@@ -12,8 +12,14 @@ DolphinScheduler 支持两种类型的内置主题,包括 `Dark` 和 `Light`
## 时区
-DolphinScheduler 支持时区设置。默认时区基于您运行 DolphinScheduler 服务器的时区。如果你想要切换时区,可以点击 [语言](#语言) 按钮右侧的时区按钮,
-然后点击 `请选择时区` 进行时区选择。当切换完成后,所有与时间相关的组件都将更改。
+DolphinScheduler 支持时区设置。
+
+服务时区
+
+使用脚本 `bin/dolphinshceduler_daemon.sh`启动服务, 服务的默认时区为UTC, 可以在`bin/env/dolphinscheduler_env.sh`中进行修改, 如`export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-Asia/Shanghai}`。<br>
+IDEA 启动服务默认时区为本地时区,可以加jvm参数如`-Duser.timezone=UTC`来修改时区。 时区选择详见[List of tz database time zones](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones)
-DolphinScheduler 内部通讯,以及任务主流程中使用 UTC 时间,在界面上显示的时间均是在 UTC 时间基础上进行时区格式化的,意味着当你切换了时区后,
-只是将格式化的时间从一个时区切换到另一个时区。
+用户时区
+
+用户的默认时区基于您运行 DolphinScheduler 服务的时区。如果你想要切换时区,可以点击 [语言](#语言) 按钮右侧的时区按钮,
+然后点击 `请选择时区` 进行时区选择。当切换完成后,所有与时间相关的组件都将更改。
diff --git a/docs/docs/zh/guide/installation/cluster.md b/docs/docs/zh/guide/installation/cluster.md
index 276a330043..194a411ca0 100644
--- a/docs/docs/zh/guide/installation/cluster.md
+++ b/docs/docs/zh/guide/installation/cluster.md
@@ -6,7 +6,7 @@
## 部署步骤
-集群部署(Cluster)使用的脚本和配置文件与[伪集群部署](pseudo-cluster.md)中的配置一样,所以所需要的步骤也与[伪集群部署](pseudo-cluster.md)大致一样。区别就是[伪集群部署](pseudo-cluster.md)针对的是一台机器,而集群部署(Cluster)需要针对多台机器,且两者“修改相关配置”步骤区别较大
+集群部署(Cluster)使用的脚本和配置文件与[伪集群部署](pseudo-cluster.md)中的配置一样,所以所需要的步骤也与伪集群部署大致一样。区别就是伪集群部署针对的是一台机器,而集群部署(Cluster)需要针对多台机器,且两者“修改相关配置”步骤区别较大
### 前置准备工作 && 准备 DolphinScheduler 启动环境
@@ -14,7 +14,7 @@
### 修改相关配置
-这个是与[伪集群部署](pseudo-cluster.md)差异较大的一步,因为部署脚本会通过 `scp` 的方式将安装需要的资源传输到各个机器上,所以这一步我们仅需要修改运行`install.sh`脚本的所在机器的配置即可。配置文件在路径在`conf/config/install_config.conf`下,此处我们仅需修改**INSTALL MACHINE**,**DolphinScheduler ENV、Database、Registry Server**与[伪集群部署](pseudo-cluster.md)保持一致,下面对必须修改参数进行说明
+这个是与[伪集群部署](pseudo-cluster.md)差异较大的一步,因为部署脚本会通过 `scp` 的方式将安装需要的资源传输到各个机器上,所以这一步我们仅需要修改运行`install.sh`脚本的所在机器的配置即可。配置文件在路径在`conf/config/install_config.conf`下,此处我们仅需修改**INSTALL MACHINE**,**DolphinScheduler ENV、Database、Registry Server**与伪集群部署保持一致,下面对必须修改参数进行说明
```shell
# ---------------------------------------------------------
diff --git a/docs/docs/zh/guide/installation/pseudo-cluster.md b/docs/docs/zh/guide/installation/pseudo-cluster.md
index e153e44373..63adda7e8f 100644
--- a/docs/docs/zh/guide/installation/pseudo-cluster.md
+++ b/docs/docs/zh/guide/installation/pseudo-cluster.md
@@ -67,72 +67,83 @@ chmod 600 ~/.ssh/authorized_keys
./bin/zkServer.sh start
```
-<!--
-修改数据库配置,并初始化
-
-```properties
-spring.datasource.driver-class-name=com.mysql.jdbc.Driver
-spring.datasource.url=jdbc:mysql://localhost:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true
-# 如果你不是以 dolphinscheduler/dolphinscheduler 作为用户名和密码的,需要进行修改
-spring.datasource.username=dolphinscheduler
-spring.datasource.password=dolphinscheduler
-```
+## 修改相关配置
-修改并保存完后,执行 script 目录下的创建表及导入基础数据脚本
+完成基础环境的准备后,需要根据你的机器环境修改配置文件。配置文件可以在目录 `<bin|alert-server|api-server|master-server|worker-server|tools>/dolphinscheduler_env.sh`
+六个文件以及 `bin/env/install_env.sh` 找到他们。
-```shell
-sh script/create-dolphinscheduler.sh
-```
--->
+### 修改 `install_env.sh` 文件
-## 修改相关配置
-
-完成了基础环境的准备后,在运行部署命令前,还需要根据环境修改配置文件。配置文件在路径在`conf/config/install_config.conf`下,一般部署只需要修改**INSTALL MACHINE、DolphinScheduler ENV、Database、Registry Server**部分即可完成部署,下面对必须修改参数进行说明
+文件 `install_env.sh` 描述了哪些机器将被安装 DolphinScheduler 以及每台机器对应安装哪些服务。您可以在路径 `bin/env/install_env.sh` 中找到此文件,配置详情如下。
```shell
# ---------------------------------------------------------
# INSTALL MACHINE
# ---------------------------------------------------------
-# 因为是在单节点上部署master、worker、API server,所以服务器的IP均为机器IP或者localhost
+# Due to the master, worker, and API server being deployed on a single node, the IP of the server is the machine IP or localhost
ips="localhost"
masters="localhost"
workers="localhost:default"
alertServer="localhost"
apiServers="localhost"
-# DolphinScheduler安装路径,如果不存在会创建
+# DolphinScheduler installation path, it will auto-create if not exists
installPath="~/dolphinscheduler"
-# 部署用户,填写在 **配置用户免密及权限** 中创建的用户
+# Deploy user, use the user you create in section **Configure machine SSH password-free login**
deployUser="dolphinscheduler"
+```
-# ---------------------------------------------------------
-# DolphinScheduler ENV
-# ---------------------------------------------------------
-# JAVA_HOME 的路径,是在 **前置准备工作** 安装的JDK中 JAVA_HOME 所在的位置
-javaHome="/your/java/home/here"
-
-# ---------------------------------------------------------
-# Database
-# ---------------------------------------------------------
-# 数据库的类型,用户名,密码,IP,端口,元数据库db。其中dbtype目前支持 mysql 和 postgresql
-dbtype="mysql"
-dbhost="localhost:3306"
-# 如果你不是以 dolphinscheduler/dolphinscheduler 作为用户名和密码的,需要进行修改
-username="dolphinscheduler"
-password="dolphinscheduler"
-dbname="dolphinscheduler"
-
-# ---------------------------------------------------------
-# Registry Server
-# ---------------------------------------------------------
-# 注册中心地址,zookeeper服务的地址
-registryServers="localhost:2181"
+### 修改 `dolphinscheduler_env.sh` 文件
+
+拆分服务后,配置文件在 `<bin|alert-server|api-server|master-server|worker-server|tools>/dolphinscheduler_env.sh` 六个文件中可以找到,描述了下列配置:
+
+* DolphinScheduler 的数据库配置,详细配置方法见[初始化数据库](#初始化数据库)
+* 一些任务类型外部依赖路径或库文件,如 `JAVA_HOME` 和 `SPARK_HOME`都是在这里定义的
+* 注册中心`zookeeper`
+* 服务端相关配置,比如缓存,时区设置等
+
+如果您不使用某些任务类型,您可以忽略任务外部依赖项,但您必须根据您的环境更改 `JAVA_HOME`、注册中心和数据库相关配置。
+
+```sh
+# JAVA_HOME, will use it to start DolphinScheduler server
+export JAVA_HOME=${JAVA_HOME:-/opt/soft/java}
+
+# Database related configuration, set database type, username and password
+export DATABASE=${DATABASE:-postgresql}
+export SPRING_PROFILES_ACTIVE=${DATABASE}
+export SPRING_DATASOURCE_DRIVER_CLASS_NAME=org.postgresql.Driver
+export SPRING_DATASOURCE_URL=jdbc:postgresql://127.0.0.1:5432/dolphinscheduler
+export SPRING_DATASOURCE_USERNAME={user}
+export SPRING_DATASOURCE_PASSWORD={password}
+
+# DolphinScheduler server related configuration
+export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
+export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
+export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}
+
+# Registry center configuration, determines the type and link of the registry center
+export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
+export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-localhost:2181}
+
+# Tasks related configurations, need to change the configuration if you use the related tasks.
+export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
+export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
+export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1}
+export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
+export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python}
+export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
+export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
+export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax}
+
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
```
## 初始化数据库
-DolphinScheduler 元数据存储在关系型数据库中,目前支持 PostgreSQL 和 MySQL,如果使用 MySQL 则需要手动下载 [mysql-connector-java 驱动][mysql] (8.0.16) 并移动到 DolphinScheduler 的 lib目录下。下面以 MySQL 为例,说明如何初始化数据库
+DolphinScheduler 元数据存储在关系型数据库中,目前支持 PostgreSQL 和 MySQL,如果使用 MySQL 则需要手动下载 [mysql-connector-java 驱动][mysql] (8.0.16) 并移动到 DolphinScheduler 的 lib目录下(`tools/libs/`)。下面以 MySQL 为例,说明如何初始化数据库
+
+对于mysql 5.6 / 5.7:
```shell
mysql -uroot -p
@@ -146,10 +157,36 @@ mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'localhost' IDENTI
mysql> flush privileges;
```
+对于mysql 8:
+
+```shell
+mysql -uroot -p
+
+mysql> CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
+
+# 修改 {user} 和 {password} 为你希望的用户名和密码
+mysql> CREATE USER '{user}'@'%' IDENTIFIED BY '{password}';
+mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'%';
+mysql> CREATE USER '{user}'@'localhost' IDENTIFIED BY '{password}';
+mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO '{user}'@'localhost';
+mysql> FLUSH PRIVILEGES;
+```
+
+然后修改`./bin/dolphinscheduler_env.sh`,将username和password改成你在上一步中设置的用户名{user}和密码{password}
+
+```shell
+export DATABASE=${DATABASE:-mysql}
+export SPRING_PROFILES_ACTIVE=${DATABASE}
+export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver
+export SPRING_DATASOURCE_URL=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
+export SPRING_DATASOURCE_USERNAME={user}
+export SPRING_DATASOURCE_PASSWORD={password}
+```
+
完成上述步骤后,您已经为 DolphinScheduler 创建一个新数据库,现在你可以通过快速的 Shell 脚本来初始化数据库
```shell
-sh script/create-dolphinscheduler.sh
+sh tools/bin/create-schema.sh
```
## 启动 DolphinScheduler
@@ -157,7 +194,7 @@ sh script/create-dolphinscheduler.sh
使用上面创建的**部署用户**运行以下命令完成部署,部署后的运行日志将存放在 logs 文件夹内
```shell
-sh install.sh
+sh ./bin/install.sh
```
> **_注意:_** 第一次部署的话,可能出现 5 次`sh: bin/dolphinscheduler-daemon.sh: No such file or directory`相关信息,次为非重要信息直接忽略即可
@@ -192,7 +229,8 @@ sh ./bin/dolphinscheduler-daemon.sh start alert-server
sh ./bin/dolphinscheduler-daemon.sh stop alert-server
```
-> **_注意:_**:服务用途请具体参见《系统架构设计》小节
+> **_注意:_**:服务用途请具体参见《系统架构设计》小节。Python gateway service 默认与 api-server 一起启动,如果您不想启动 Python gateway service
+> 请通过更改 api-server 配置文件 `api-server/conf/application.yaml` 中的 `python-gateway.enabled : false` 来禁用它。
[jdk]: https://www.oracle.com/technetwork/java/javase/downloads/index.html
[zookeeper]: https://zookeeper.apache.org/releases.html
diff --git a/docs/docs/zh/guide/installation/standalone.md b/docs/docs/zh/guide/installation/standalone.md
index 1c3ca7f2fe..ac511e8b80 100644
--- a/docs/docs/zh/guide/installation/standalone.md
+++ b/docs/docs/zh/guide/installation/standalone.md
@@ -4,7 +4,8 @@ Standalone 仅适用于 DolphinScheduler 的快速体验.
如果你是新手,想要体验 DolphinScheduler 的功能,推荐使用[Standalone](standalone.md)方式体检。如果你想体验更完整的功能,或者更大的任务量,推荐使用[伪集群部署](pseudo-cluster.md)。如果你是在生产中使用,推荐使用[集群部署](cluster.md)或者[kubernetes](kubernetes.md)
-> **_注意:_** Standalone仅建议20个以下工作流使用,因为其采用 H2 Database, Zookeeper Testing Server,任务过多可能导致不稳定
+> **_注意:_** Standalone仅建议20个以下工作流使用,因为其采用内存式的H2 Database, Zookeeper Testing Server,任务过多可能导致不稳定,并且如果重启或者停止standalone-server会导致内存中数据库里的数据清空。
+> 如果您要连接外部数据库,比如mysql或者postgresql,请看[配置数据库](#配置数据库)
## 前置准备工作
@@ -40,3 +41,20 @@ sh ./bin/dolphinscheduler-daemon.sh stop standalone-server
```
[jdk]: https://www.oracle.com/technetwork/java/javase/downloads/index.html
+
+### 配置数据库
+
+* 这里以mysql为例介绍如何连接外部数据库
+* 首先,参照 [伪集群部署](pseudo-cluster.md) `初始化数据库` 创建并初始化数据库
+* 在你的命令行设定下列环境变量,将{user}和{password}改为你数据库的用户名和密码
+
+```shell
+export DATABASE=${DATABASE:-mysql}
+export SPRING_PROFILES_ACTIVE=${DATABASE}
+export SPRING_DATASOURCE_USERNAME={user}
+export SPRING_DATASOURCE_PASSWORD={password}
+```
+
+* 将mysql-connector-java驱动加到`./standalone-server/libs/standalone-server/`目录下, 下载方法见 [伪集群部署](pseudo-cluster.md) `初始化数据库` 一栏
+
+* 启动standalone-server,此时你已经连接上mysql,重启或者停止standalone-server并不会清空您数据库里的数据
diff --git a/docs/docs/zh/guide/start/docker.md b/docs/docs/zh/guide/start/docker.md
index e1274e9d4a..72c5ca61bf 100644
--- a/docs/docs/zh/guide/start/docker.md
+++ b/docs/docs/zh/guide/start/docker.md
@@ -1,1023 +1,126 @@
-# 快速试用 Docker 部署
+# Docker 快速使用教程
-## 先决条件
+本教程使用三种不同的方式通过 Docker 完成 DolphinScheduler 的部署,如果你想要快速体验,推荐使用 standalone-server 镜像,
+如果你想要体验比较完成的服务,推荐使用 docker-compose 启动服务。如果你已经有自己的数据库或者 Zookeeper 服务
+你想要沿用这些基础服务,你可以参考沿用已有的 PostgreSQL 和 ZooKeeper 服务完成部署。
+
+## 前置条件
- [Docker](https://docs.docker.com/engine/install/) 1.13.1+
- [Docker Compose](https://docs.docker.com/compose/) 1.11.0+
-## 如何使用 Docker 镜像
-
-有 3 种方式可以快速试用 DolphinScheduler
-
-### 一、以 docker-compose 的方式启动 DolphinScheduler (推荐)
-
-这种方式需要先安装 [docker-compose](https://docs.docker.com/compose/), docker-compose 的安装网上已经有非常多的资料,请自行安装即可
-
-对于 Windows 7-10,你可以安装 [Docker Toolbox](https://github.com/docker/toolbox/releases)。对于 Windows 10 64-bit,你可以安装 [Docker Desktop](https://docs.docker.com/docker-for-windows/install/),并且注意[系统要求](https://docs.docker.com/docker-for-windows/install/#system-requirements)
-
-#### 0、请配置内存不少于 4GB
-
-对于 Mac 用户,点击 `Docker Desktop -> Preferences -> Resources -> Memory`
-
-对于 Windows Docker Toolbox 用户,有两项需要配置:
+## 启动服务
-- **内存**:打开 Oracle VirtualBox Manager,如果你双击 Docker Quickstart Terminal 并成功运行 Docker Toolbox,你将会看到一个名为 `default` 的虚拟机. 点击 `设置 -> 系统 -> 主板 -> 内存大小`
-- **端口转发**:点击 `设置 -> 网络 -> 高级 -> 端口转发 -> 添加`. `名称`,`主机端口` 和 `子系统端口` 都填写 `12345`,不填 `主机IP` 和 `子系统IP`
+### 使用 standalone-server 镜像
-对于 Windows Docker Desktop 用户
-- **Hyper-V 模式**:点击 `Docker Desktop -> Settings -> Resources -> Memory`
-- **WSL 2 模式**:参考 [WSL 2 utility VM](https://docs.microsoft.com/zh-cn/windows/wsl/wsl-config#configure-global-options-with-wslconfig)
-
-#### 1、下载源码包
-
-请下载源码包 apache-dolphinscheduler-3.0.0-alpha-src.tar.gz,下载地址: [下载](/zh-cn/download/download.html)
-
-#### 2、拉取镜像并启动服务
-
-> 对于 Mac 和 Linux 用户,打开 **Terminal**
-> 对于 Windows Docker Toolbox 用户,打开 **Docker Quickstart Terminal**
-> 对于 Windows Docker Desktop 用户,打开 **Windows PowerShell**
+使用 standalone-server 镜像启动一个 DolphinScheduler standalone-server 容器应该是最快体验 DolphinScheduler 的方法。通过这个方式
+你可以最快速的体验到 DolphinScheduler 的大部分功能,了解主要和概念和内容。
+```shell
+$ DOLPHINSCHEDULER_VERSION=3.0.0-alpha
+$ docker run --name dolphinscheduler-standalone-server -p 12345:12345 -p 25333:25333 -d apache/dolphinscheduler-standalone-server:"${DOLPHINSCHEDULER_VERSION}"
```
-$ tar -zxvf apache-dolphinscheduler-3.0.0-alpha-src.tar.gz
-$ cd apache-dolphinscheduler-3.0.0-alpha-src/docker/docker-swarm
-$ docker pull dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:3.0.0-alpha
-$ docker tag apache/dolphinscheduler:3.0.0-alpha apache/dolphinscheduler:latest
-$ docker-compose up -d
-```
-
-> PowerShell 应该使用 `cd apache-dolphinscheduler-3.0.0-alpha-src\docker\docker-swarm`
-**PostgreSQL** (用户 `root`, 密码 `root`, 数据库 `dolphinscheduler`) 和 **ZooKeeper** 服务将会默认启动
+> 注意:请不要将 apache/dolphinscheduler-standalone-server 镜像作为生产镜像,应该仅仅作为快速体验 DolphinScheduler 的功能的途径。
+> 除了因为他将全部服务运行在一个进程中外,还因为其使用内存数据库 H2 储存其元数据,当服务停止时内存数据库中的数据将会被清空。另外
+> apache/dolphinscheduler-standalone-server 仅包含 DolphinScheduler 核心服务,部分任务组件(如 Spark 和 Flink 等),
+> 告警组件(如 Telegram 和 Dingtalk 等)需要外部的组件或对应的配置后
-#### 3、登录系统
+### 使用 docker-compose 启动服务
-访问前端页面:http://localhost:12345/dolphinscheduler,如果有需要请修改成对应的 IP 地址
+使用 docker-compose 启动服务相比 standalone-server 的优点是 DolphinScheduler 的各个是独立的容器和进程,相互影响降到最小,且能够在
+服务重启的时候保留元数据(如需要挂载到本地路径需要做指定)。他更健壮,能保证用户体验更加完整的 DolphinScheduler 服务。这种方式需要先安装
+[docker-compose](https://docs.docker.com/compose/install/),链接适用于 Mac,Linux,Windows。
-默认的用户是`admin`,默认的密码是`dolphinscheduler123`
+安装完成 docker-compose 后我们需要修改部分配置以便能更好体验 DolphinScheduler 服务,我们需要配置不少于 4GB 的内存:
-
-
-请参考用户手册章节的[快速上手](../start/quick-start.md)查看如何使用DolphinScheduler
-
-### 二、通过指定已存在的 PostgreSQL 和 ZooKeeper 服务
-
-这种方式需要先安装 [docker](https://docs.docker.com/engine/install/), docker 的安装网上已经有非常多的资料,请自行安装即可
-
-#### 1、基础软件安装 (请自行安装)
-
-- [PostgreSQL](https://www.postgresql.org/download/) (8.2.15+)
-- [ZooKeeper](https://zookeeper.apache.org/releases.html) (3.4.6+)
-- [Docker](https://docs.docker.com/engine/install/) (1.13.1+)
-
-#### 2、请登录 PostgreSQL 数据库,创建名为 `dolphinscheduler` 数据库
-
-#### 3、初始化数据库,导入 `sql/dolphinscheduler_postgre.sql` 进行创建表及基础数据导入
+- Mac:点击 `Docker Desktop -> Preferences -> Resources -> Memory` 调整内存大小
+- Windows Docker Desktop:
+ - Hyper-V 模式:点击 `Docker Desktop -> Settings -> Resources -> Memory` 调整内存大小
+ - WSL 2 模式 模式:参考 [WSL 2 utility VM](https://docs.microsoft.com/zh-cn/windows/wsl/wsl-config#configure-global-options-with-wslconfig) 调整内存大小
-#### 4、下载 DolphinScheduler 镜像
+配置完成后我们需要获取 `docker-compose.yaml` 文件,通过[下载页面](/zh-cn/download/download.html)下载对应版本源码包可能是最快的方法,
+源码包对应的值为 "Total Source Code"。当下载完源码后就可以运行命令进行部署了。
-我们已将面向用户的 DolphinScheduler 镜像上传至 docker 仓库,用户无需在本地构建镜像,直接执行以下命令从 docker 仓库 pull 镜像:
-
-```
-docker pull dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:3.0.0-alpha
-```
-
-#### 5、运行一个 DolphinScheduler 实例
-
-```
-$ docker run -d --name dolphinscheduler \
--e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
--e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
--e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
--p 12345:12345 \
-apache/dolphinscheduler:3.0.0-alpha all
+```shell
+$ DOLPHINSCHEDULER_VERSION=3.0.0-alpha
+$ tar -zxf apache-dolphinscheduler-"${DOLPHINSCHEDULER_VERSION}"-src.tar.gz
+# Mac Linux 用户
+$ cd apache-dolphinscheduler-"${DOLPHINSCHEDULER_VERSION}"-src/deploy/docker
+# Windows 用户
+$ cd apache-dolphinscheduler-"${DOLPHINSCHEDULER_VERSION}"-src\deploy\docker
+$ docker-compose up -d
```
-注:数据库用户 test 和密码 test 需要替换为实际的 PostgreSQL 用户和密码,192.168.x.x 需要替换为 PostgreSQL 和 ZooKeeper 的主机 IP
+> 提醒:通过 docker-compose 启动服务时,除了会启动 DolphinScheduler 对应的服务外,还会启动必要依赖服务,如数据库 PostgreSQL(用户
+> `root`, 密码 `root`, 数据库 `dolphinscheduler`) 和 服务发现 ZooKeeper。
-#### 6、登录系统
+### 沿用已有的 PostgreSQL 和 ZooKeeper 服务
-同上
+使用 docker-compose 启动服务会新启动数据库,以及 ZooKeeper 服务。如果你已经有在运行中的数据库,或者
+ZooKeeper 且不想启动新的服务,可以使用这个方式分别启动 DolphinScheduler 容器。
-### 三、运行 DolphinScheduler 中的独立服务
-
-在容器启动时,会自动启动以下服务:
-
-```
- MasterServer ----- master 服务
- WorkerServer ----- worker 服务
- ApiApplicationServer ----- api 服务
- AlertServer ----- alert 服务
-```
-
-如果你只是想运行 dolphinscheduler 中的部分服务,你可以够通执行以下命令来运行 dolphinscheduler 中的部分服务
-
-* 启动一个 **master server**, 如下:
-
-```
+```shell
+$ DOLPHINSCHEDULER_VERSION=3.0.0-alpha
+# 初始化数据库,其确保数据库 <DATABASE> 已经存在
+$ docker run -d --name dolphinscheduler-tools \
+ -e DATABASE="postgresql" \
+ # 如果使用 MySQL 则使用 "com.mysql.cj.jdbc.driver"
+ -e SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/<DATABASE>" \
+ -e SPRING_DATASOURCE_USERNAME="<USER>" \
+ -e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
+ apache/dolphinscheduler-tools:"${DOLPHINSCHEDULER_VERSION}" bin/create-schema.sh
+# 启动 DolphinScheduler 对应的服务
$ docker run -d --name dolphinscheduler-master \
--e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
--e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
--e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
-apache/dolphinscheduler:3.0.0-alpha master-server
-```
-
-* 启动一个 **worker server**, 如下:
-
-```
+ -e DATABASE="postgresql" \
+ -e SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_USERNAME="<USER>" \
+ -e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
+ -e REGISTRY_ZOOKEEPER_CONNECT_STRING="localhost:2181" \
+ -d apache/dolphinscheduler-master:"${DOLPHINSCHEDULER_VERSION}"
$ docker run -d --name dolphinscheduler-worker \
--e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
--e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
--e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
-apache/dolphinscheduler:3.0.0-alpha worker-server
-```
-
-* 启动一个 **api server**, 如下:
-
-```
+ -e DATABASE="postgresql" \
+ -e SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_USERNAME="<USER>" \
+ -e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
+ -e REGISTRY_ZOOKEEPER_CONNECT_STRING="localhost:2181" \
+ -d apache/dolphinscheduler-worker:"${DOLPHINSCHEDULER_VERSION}"
$ docker run -d --name dolphinscheduler-api \
--e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
--e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
--e ZOOKEEPER_QUORUM="192.168.x.x:2181" \
--p 12345:12345 \
-apache/dolphinscheduler:3.0.0-alpha api-server
-```
+ -e DATABASE="postgresql" \
+ -e SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_USERNAME="<USER>" \
+ -e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
+ -e REGISTRY_ZOOKEEPER_CONNECT_STRING="localhost:2181" \
+ -d apache/dolphinscheduler-api:"${DOLPHINSCHEDULER_VERSION}"
+$ docker run -d --name dolphinscheduler-alert-server \
+ -e DATABASE="postgresql" \
+ -e SPRING_DATASOURCE_DRIVER_CLASS_NAME="org.postgresql.Driver" \
+ -e SPRING_DATASOURCE_URL="jdbc:postgresql://localhost:5432/dolphinscheduler" \
+ -e SPRING_DATASOURCE_USERNAME="<USER>" \
+ -e SPRING_DATASOURCE_PASSWORD="<PASSWORD>" \
+ -e REGISTRY_ZOOKEEPER_CONNECT_STRING="localhost:2181" \
+ -d apache/dolphinscheduler-alert-server:"${DOLPHINSCHEDULER_VERSION}"
+```
+
+> 注意:如果你本地还没有对应的数据库和 ZooKeeper 服务,但是想要尝试这个启动方式,可以先安装并启动
+> [PostgreSQL](https://www.postgresql.org/download/)(8.2.15+) 以及 [ZooKeeper](https://zookeeper.apache.org/releases.html)(3.4.6+)
+
+## 登录系统
+
+不管你是用那种方式启动的服务,只要服务启动后,你都可以通过 [http://localhost:12345/dolphinscheduler/ui](http://localhost:12345/dolphinscheduler/ui)
+访问 DolphinScheduler。访问上述链接后会跳转到登陆页面,DolphinScheduler 默认的用户和密码分别为 `admin` 和 `dolphinscheduler123`。
+想要了解更多操作请参考用户手册[快速上手](../start/quick-start.md)。
-* 启动一个 **alert server**, 如下:
-
-```
-$ docker run -d --name dolphinscheduler-alert \
--e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
--e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
-apache/dolphinscheduler:3.0.0-alpha alert-server
-```
+
-**注意**: 当你运行 dolphinscheduler 中的部分服务时,你必须指定这些环境变量 `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_DATABASE`, `DATABASE_USERNAME`, `DATABASE_PASSWORD`, `ZOOKEEPER_QUORUM`。
+> 注意:如果你使用沿用已有的 PostgreSQL 和 ZooKeeper 服务方式启动服务,且服务分布在多台机器中,
+> 请将上述的地址改成你 API 容器启动的 hostname 或者 IP。
## 环境变量
-Docker 容器通过环境变量进行配置,[附录-环境变量](#appendix-environment-variables) 列出了 DolphinScheduler 的可配置环境变量及其默认值 <!-- markdown-link-check-disable-line -->
-
-特别地,在 Docker Compose 和 Docker Swarm 中,可以通过环境变量配置文件 `config.env.sh` 进行配置
-
-## 支持矩阵
-
-| Type | 支持 | 备注 |
-| ------------------------------------------------------------ | ------- | --------------------- |
-| Shell | 是 | |
-| Python2 | 是 | |
-| Python3 | 间接支持 | 详见 FAQ |
-| Hadoop2 | 间接支持 | 详见 FAQ |
-| Hadoop3 | 尚未确定 | 尚未测试 |
-| Spark-Local(client) | 间接支持 | 详见 FAQ |
-| Spark-YARN(cluster) | 间接支持 | 详见 FAQ |
-| Spark-Standalone(cluster) | 尚不 | |
-| Spark-Kubernetes(cluster) | 尚不 | |
-| Flink-Local(local>=1.11) | 尚不 | Generic CLI 模式尚未支持 |
-| Flink-YARN(yarn-cluster) | 间接支持 | 详见 FAQ |
-| Flink-YARN(yarn-session/yarn-per-job/yarn-application>=1.11) | 尚不 | Generic CLI 模式尚未支持 |
-| Flink-Standalone(default) | 尚不 | |
-| Flink-Standalone(remote>=1.11) | 尚不 | Generic CLI 模式尚未支持 |
-| Flink-Kubernetes(default) | 尚不 | |
-| Flink-Kubernetes(remote>=1.11) | 尚不 | Generic CLI 模式尚未支持 |
-| Flink-NativeKubernetes(kubernetes-session/application>=1.11) | 尚不 | Generic CLI 模式尚未支持 |
-| MapReduce | 间接支持 | 详见 FAQ |
-| Kerberos | 间接支持 | 详见 FAQ |
-| HTTP | 是 | |
-| DataX | 间接支持 | 详见 FAQ |
-| Sqoop | 间接支持 | 详见 FAQ |
-| SQL-MySQL | 间接支持 | 详见 FAQ |
-| SQL-PostgreSQL | 是 | |
-| SQL-Hive | 间接支持 | 详见 FAQ |
-| SQL-Spark | 间接支持 | 详见 FAQ |
-| SQL-ClickHouse | 间接支持 | 详见 FAQ |
-| SQL-Oracle | 间接支持 | 详见 FAQ |
-| SQL-SQLServer | 间接支持 | 详见 FAQ |
-| SQL-DB2 | 间接支持 | 详见 FAQ |
-
-## FAQ
-
-### 如何通过 docker-compose 管理 DolphinScheduler?
-
-启动、重启、停止或列出所有容器:
-
-```
-docker-compose start
-docker-compose restart
-docker-compose stop
-docker-compose ps
-```
-
-停止所有容器并移除所有容器、网络:
-
-```
-docker-compose down
-```
-
-停止所有容器并移除所有容器、网络和存储卷:
-
-```
-docker-compose down -v
-```
-
-### 如何查看一个容器的日志?
-
-列出所有运行的容器:
-
-```
-docker ps
-docker ps --format "{{.Names}}" # 只打印名字
-```
-
-查看名为 docker-swarm_dolphinscheduler-api_1 的容器的日志:
-
-```
-docker logs docker-swarm_dolphinscheduler-api_1
-docker logs -f docker-swarm_dolphinscheduler-api_1 # 跟随日志输出
-docker logs --tail 10 docker-swarm_dolphinscheduler-api_1 # 显示倒数10行日志
-```
-
-### 如何通过 docker-compose 扩缩容 master 和 worker?
-
-扩缩容 master 至 2 个实例:
-
-```
-docker-compose up -d --scale dolphinscheduler-master=2 dolphinscheduler-master
-```
-
-扩缩容 worker 至 3 个实例:
-
-```
-docker-compose up -d --scale dolphinscheduler-worker=3 dolphinscheduler-worker
-```
-
-### 如何在 Docker Swarm 上部署 DolphinScheduler?
-
-假设 Docker Swarm 集群已经部署(如果还没有创建 Docker Swarm 集群,请参考 [create-swarm](https://docs.docker.com/engine/swarm/swarm-tutorial/create-swarm/))
-
-启动名为 dolphinscheduler 的 stack:
-
-```
-docker stack deploy -c docker-stack.yml dolphinscheduler
-```
-
-列出名为 dolphinscheduler 的 stack 的所有服务:
-
-```
-docker stack services dolphinscheduler
-```
-
-停止并移除名为 dolphinscheduler 的 stack:
-
-```
-docker stack rm dolphinscheduler
-```
-
-移除名为 dolphinscheduler 的 stack 的所有存储卷:
-
-```
-docker volume rm -f $(docker volume ls --format "{{.Name}}" | grep -e "^dolphinscheduler")
-```
-
-### 如何在 Docker Swarm 上扩缩容 master 和 worker?
-
-扩缩容名为 dolphinscheduler 的 stack 的 master 至 2 个实例:
-
-```
-docker service scale dolphinscheduler_dolphinscheduler-master=2
-```
-
-扩缩容名为 dolphinscheduler 的 stack 的 worker 至 3 个实例:
-
-```
-docker service scale dolphinscheduler_dolphinscheduler-worker=3
-```
-
-### 如何构建一个 Docker 镜像?
-
-#### 从源码构建 (需要 Maven 3.3+ & JDK 1.8+)
-
-类 Unix 系统,在 Terminal 中执行:
-
-```bash
-$ bash ./docker/build/hooks/build
-```
-
-Windows 系统,在 cmd 或 PowerShell 中执行:
-
-```bat
-C:\dolphinscheduler-src>.\docker\build\hooks\build.bat
-```
-
-如果你不理解 `./docker/build/hooks/build` `./docker/build/hooks/build.bat` 这些脚本,请阅读里面的内容
-
-#### 从二进制包构建 (不需要 Maven 3.3+ & JDK 1.8+)
-
-请下载二进制包 apache-dolphinscheduler-3.0.0-alpha-bin.tar.gz,下载地址: [下载](/zh-cn/download/download.html). 然后将 apache-dolphinscheduler-3.0.0-alpha-bin.tar.gz 放到 `apache-dolphinscheduler-3.0.0-alpha-src/docker/build` 目录里,在 Terminal 或 PowerShell 中执行:
-
-```
-$ cd apache-dolphinscheduler-3.0.0-alpha-src/docker/build
-$ docker build --build-arg VERSION=3.0.0-alpha -t apache/dolphinscheduler:3.0.0-alpha .
-```
-
-> PowerShell 应该使用 `cd apache-dolphinscheduler-3.0.0-alpha-src/docker/build`
-
-#### 构建多平台架构镜像
-
-目前支持构建 `linux/amd64` 和 `linux/arm64` 平台架构的镜像,要求:
-
-1. 支持 [docker buildx](https://docs.docker.com/engine/reference/commandline/buildx/)
-2. 具有 https://hub.docker.com/r/apache/dolphinscheduler 的 push 权限(**务必谨慎**: 构建命令默认会自动将多平台架构镜像推送到 apache/dolphinscheduler 的 docker hub)
-
-执行:
-
-```bash
-$ docker login # 登录, 用于推送 apache/dolphinscheduler
-$ bash ./docker/build/hooks/build x
-```
-
-### 如何为 Docker 添加一个环境变量?
-
-如果你想在编译的时候或者运行的时候附加一些其它的操作及新增一些环境变量,你可以在`/root/start-init-conf.sh`文件中进行修改,同时如果涉及到配置文件的修改,请在`/opt/dolphinscheduler/conf/*.tpl`中修改相应的配置文件
-
-例如,在`/root/start-init-conf.sh`添加一个环境变量`SECURITY_AUTHENTICATION_TYPE`:
-
-```
-export SECURITY_AUTHENTICATION_TYPE=PASSWORD
-```
-
-当添加以上环境变量后,你应该在相应的模板文件`application-api.properties.tpl`中添加这个环境变量配置:
-
-```
-security.authentication.type=${SECURITY_AUTHENTICATION_TYPE}
-```
-
-`/root/start-init-conf.sh` 将根据模板文件动态的生成配置文件:
-
-```sh
-echo "generate dolphinscheduler config"
-ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
-eval "cat << EOF
-$(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
-EOF
-" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
-done
-```
-
-### 如何用 MySQL 替代 PostgreSQL 作为 DolphinScheduler 的数据库?
-
-> 由于商业许可证的原因,我们不能直接使用 MySQL 的驱动包.
->
-> 如果你要使用 MySQL, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建.
-
-1. 下载 MySQL 驱动包 [mysql-connector-java-8.0.16.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar)
-
-2. 创建一个新的 `Dockerfile`,用于添加 MySQL 的驱动包:
-
-```
-FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:3.0.0-alpha
-COPY mysql-connector-java-8.0.16.jar /opt/dolphinscheduler/lib
-```
-
-3. 构建一个包含 MySQL 驱动包的新镜像:
-
-```
-docker build -t apache/dolphinscheduler:mysql-driver .
-```
-
-4. 修改 `docker-compose.yml` 文件中的所有 image 字段为 `apache/dolphinscheduler:mysql-driver`
-
-> 如果你想在 Docker Swarm 上部署 dolphinscheduler,你需要修改 `docker-stack.yml`
-
-5. 注释 `docker-compose.yml` 文件中的 `dolphinscheduler-postgresql` 块
-
-6. 在 `docker-compose.yml` 文件中添加 `dolphinscheduler-mysql` 服务(**可选**,你可以直接使用一个外部的 MySQL 数据库)
-
-7. 修改 `config.env.sh` 文件中的 DATABASE 环境变量
-
-```
-DATABASE_TYPE=mysql
-DATABASE_DRIVER=com.mysql.jdbc.Driver
-DATABASE_HOST=dolphinscheduler-mysql
-DATABASE_PORT=3306
-DATABASE_USERNAME=root
-DATABASE_PASSWORD=root
-DATABASE_DATABASE=dolphinscheduler
-DATABASE_PARAMS=useUnicode=true&characterEncoding=UTF-8
-```
-
-> 如果你已经添加了 `dolphinscheduler-mysql` 服务,设置 `DATABASE_HOST` 为 `dolphinscheduler-mysql` 即可
-
-8. 运行 dolphinscheduler (详见**如何使用docker镜像**)
-
-### 如何在数据源中心支持 MySQL 数据源?
-
-> 由于商业许可证的原因,我们不能直接使用 MySQL 的驱动包.
->
-> 如果你要添加 MySQL 数据源, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建.
-
-1. 下载 MySQL 驱动包 [mysql-connector-java-8.0.16.jar](https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar)
-
-2. 创建一个新的 `Dockerfile`,用于添加 MySQL 驱动包:
-
-```
-FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:3.0.0-alpha
-COPY mysql-connector-java-8.0.16.jar /opt/dolphinscheduler/lib
-```
-
-3. 构建一个包含 MySQL 驱动包的新镜像:
-
-```
-docker build -t apache/dolphinscheduler:mysql-driver .
-```
-
-4. 将 `docker-compose.yml` 文件中的所有 `image` 字段修改为 `apache/dolphinscheduler:mysql-driver`
-
-> 如果你想在 Docker Swarm 上部署 dolphinscheduler,你需要修改 `docker-stack.yml`
-
-5. 运行 dolphinscheduler (详见**如何使用docker镜像**)
-
-6. 在数据源中心添加一个 MySQL 数据源
-
-### 如何在数据源中心支持 Oracle 数据源?
-
-> 由于商业许可证的原因,我们不能直接使用 Oracle 的驱动包.
->
-> 如果你要添加 Oracle 数据源, 你可以基于官方镜像 `apache/dolphinscheduler` 进行构建.
-
-1. 下载 Oracle 驱动包 [ojdbc8.jar](https://repo1.maven.org/maven2/com/oracle/database/jdbc/ojdbc8/) (such as `ojdbc8-19.9.0.0.jar`)
-
-2. 创建一个新的 `Dockerfile`,用于添加 Oracle 驱动包:
-
-```
-FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:3.0.0-alpha
-COPY ojdbc8-19.9.0.0.jar /opt/dolphinscheduler/lib
-```
-
-3. 构建一个包含 Oracle 驱动包的新镜像:
-
-```
-docker build -t apache/dolphinscheduler:oracle-driver .
-```
-
-4. 将 `docker-compose.yml` 文件中的所有 `image` 字段修改为 `apache/dolphinscheduler:oracle-driver`
-
-> 如果你想在 Docker Swarm 上部署 dolphinscheduler,你需要修改 `docker-stack.yml`
-
-5. 运行 dolphinscheduler (详见**如何使用docker镜像**)
-
-6. 在数据源中心添加一个 Oracle 数据源
-
-### 如何支持 Python 2 pip 以及自定义 requirements.txt?
-
-1. 创建一个新的 `Dockerfile`,用于安装 pip:
-
-```
-FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:3.0.0-alpha
-COPY requirements.txt /tmp
-RUN apt-get update && \
- apt-get install -y --no-install-recommends python-pip && \
- pip install --no-cache-dir -r /tmp/requirements.txt && \
- rm -rf /var/lib/apt/lists/*
-```
-
-这个命令会安装默认的 **pip 18.1**. 如果你想升级 pip, 只需添加一行
-
-```
- pip install --no-cache-dir -U pip && \
-```
-
-2. 构建一个包含 pip 的新镜像:
-
-```
-docker build -t apache/dolphinscheduler:pip .
-```
-
-3. 将 `docker-compose.yml` 文件中的所有 `image` 字段修改为 `apache/dolphinscheduler:pip`
-
-> 如果你想在 Docker Swarm 上部署 dolphinscheduler,你需要修改 `docker-stack.yml`
-
-4. 运行 dolphinscheduler (详见**如何使用docker镜像**)
-
-5. 在一个新 Python 任务下验证 pip
-
-### 如何支持 Python 3?
-
-1. 创建一个新的 `Dockerfile`,用于安装 Python 3:
-
-```
-FROM dolphinscheduler.docker.scarf.sh/apache/dolphinscheduler:3.0.0-alpha
-RUN apt-get update && \
- apt-get install -y --no-install-recommends python3 && \
- rm -rf /var/lib/apt/lists/*
-```
-
-这个命令会安装默认的 **Python 3.7.3**. 如果你也想安装 **pip3**, 将 `python3` 替换为 `python3-pip` 即可
-
-```
- apt-get install -y --no-install-recommends python3-pip && \
-```
-
-2. 构建一个包含 Python 3 的新镜像:
-
-```
-docker build -t apache/dolphinscheduler:python3 .
-```
-
-3. 将 `docker-compose.yml` 文件中的所有 `image` 字段修改为 `apache/dolphinscheduler:python3`
-
-> 如果你想在 Docker Swarm 上部署 dolphinscheduler,你需要修改 `docker-stack.yml`
-
-4. 修改 `config.env.sh` 文件中的 `PYTHON_HOME` 为 `/usr/bin/python3`
-
-5. 运行 dolphinscheduler (详见**如何使用docker镜像**)
-
-6. 在一个新 Python 任务下验证 Python 3
-
-### 如何支持 Hadoop, Spark, Flink, Hive 或 DataX?
-
-以 Spark 2.4.7 为例:
-
-1. 下载 Spark 2.4.7 发布的二进制包 `spark-2.4.7-bin-hadoop2.7.tgz`
-
-2. 运行 dolphinscheduler (详见**如何使用docker镜像**)
-
-3. 复制 Spark 2.4.7 二进制包到 Docker 容器中
-
-```bash
-docker cp spark-2.4.7-bin-hadoop2.7.tgz docker-swarm_dolphinscheduler-worker_1:/opt/soft
-```
-
-因为存储卷 `dolphinscheduler-shared-local` 被挂载到 `/opt/soft`, 因此 `/opt/soft` 中的所有文件都不会丢失
-
-4. 登录到容器并确保 `SPARK_HOME2` 存在
-
-```bash
-docker exec -it docker-swarm_dolphinscheduler-worker_1 bash
-cd /opt/soft
-tar zxf spark-2.4.7-bin-hadoop2.7.tgz
-rm -f spark-2.4.7-bin-hadoop2.7.tgz
-ln -s spark-2.4.7-bin-hadoop2.7 spark2 # 或者 mv
-$SPARK_HOME2/bin/spark-submit --version
-```
-
-如果一切执行正常,最后一条命令将会打印 Spark 版本信息
-
-5. 在一个 Shell 任务下验证 Spark
-
-```
-$SPARK_HOME2/bin/spark-submit --class org.apache.spark.examples.SparkPi $SPARK_HOME2/examples/jars/spark-examples_2.11-2.4.7.jar
-```
-
-检查任务日志是否包含输出 `Pi is roughly 3.146015`
-
-6. 在一个 Spark 任务下验证 Spark
-
-文件 `spark-examples_2.11-2.4.7.jar` 需要先被上传到资源中心,然后创建一个 Spark 任务并设置:
-
-- Spark版本: `SPARK2`
-- 主函数的 Class: `org.apache.spark.examples.SparkPi`
-- 主程序包: `spark-examples_2.11-2.4.7.jar`
-- 部署方式: `local`
-
-同样地, 检查任务日志是否包含输出 `Pi is roughly 3.146015`
-
-7. 验证 Spark on YARN
-
-Spark on YARN (部署方式为 `cluster` 或 `client`) 需要 Hadoop 支持. 类似于 Spark 支持, 支持 Hadoop 的操作几乎和前面的步骤相同
-
-确保 `$HADOOP_HOME` 和 `$HADOOP_CONF_DIR` 存在
-
-### 如何支持 Spark 3?
-
-事实上,使用 `spark-submit` 提交应用的方式是相同的, 无论是 Spark 1, 2 或 3. 换句话说,`SPARK_HOME2` 的语义是第二个 `SPARK_HOME`, 而非 `SPARK2` 的 `HOME`, 因此只需设置 `SPARK_HOME2=/path/to/spark3` 即可
-
-以 Spark 3.1.1 为例:
-
-1. 下载 Spark 3.1.1 发布的二进制包 `spark-3.1.1-bin-hadoop2.7.tgz`
-
-2. 运行 dolphinscheduler (详见**如何使用docker镜像**)
-
-3. 复制 Spark 3.1.1 二进制包到 Docker 容器中
-
-```bash
-docker cp spark-3.1.1-bin-hadoop2.7.tgz docker-swarm_dolphinscheduler-worker_1:/opt/soft
-```
-
-4. 登录到容器并确保 `SPARK_HOME2` 存在
-
-```bash
-docker exec -it docker-swarm_dolphinscheduler-worker_1 bash
-cd /opt/soft
-tar zxf spark-3.1.1-bin-hadoop2.7.tgz
-rm -f spark-3.1.1-bin-hadoop2.7.tgz
-ln -s spark-3.1.1-bin-hadoop2.7 spark2 # 或者 mv
-$SPARK_HOME2/bin/spark-submit --version
-```
-
-如果一切执行正常,最后一条命令将会打印 Spark 版本信息
-
-5. 在一个 Shell 任务下验证 Spark
-
-```
-$SPARK_HOME2/bin/spark-submit --class org.apache.spark.examples.SparkPi $SPARK_HOME2/examples/jars/spark-examples_2.12-3.1.1.jar
-```
-
-检查任务日志是否包含输出 `Pi is roughly 3.146015`
-
-### 如何在 Master、Worker 和 Api 服务之间支持共享存储?
-
-> **注意**: 如果是在单机上通过 docker-compose 部署,则步骤 1 和 2 可以直接跳过,并且执行命令如 `docker cp hadoop-3.2.2.tar.gz docker-swarm_dolphinscheduler-worker_1:/opt/soft` 将 Hadoop 放到容器中的共享目录 /opt/soft 下
-
-例如, Master、Worker 和 Api 服务可能同时使用 Hadoop
-
-1. 修改 `docker-compose.yml` 文件中的 `dolphinscheduler-shared-local` 存储卷,以支持 nfs
-
-> 如果你想在 Docker Swarm 上部署 dolphinscheduler,你需要修改 `docker-stack.yml`
-
-```yaml
-volumes:
- dolphinscheduler-shared-local:
- driver_opts:
- type: "nfs"
- o: "addr=10.40.0.199,nolock,soft,rw"
- device: ":/path/to/shared/dir"
-```
-
-2. 将 Hadoop 放到 nfs
-
-3. 确保 `$HADOOP_HOME` 和 `$HADOOP_CONF_DIR` 正确
-
-### 如何支持本地文件存储而非 HDFS 和 S3?
-
-> **注意**: 如果是在单机上通过 docker-compose 部署,则步骤 2 可以直接跳过
-
-1. 修改 `config.env.sh` 文件中下面的环境变量:
-
-```
-RESOURCE_STORAGE_TYPE=HDFS
-FS_DEFAULT_FS=file:///
-```
-
-2. 修改 `docker-compose.yml` 文件中的 `dolphinscheduler-resource-local` 存储卷,以支持 nfs
-
-> 如果你想在 Docker Swarm 上部署 dolphinscheduler,你需要修改 `docker-stack.yml`
-
-```yaml
-volumes:
- dolphinscheduler-resource-local:
- driver_opts:
- type: "nfs"
- o: "addr=10.40.0.199,nolock,soft,rw"
- device: ":/path/to/resource/dir"
-```
-
-### 如何支持 S3 资源存储,例如 MinIO?
-
-以 MinIO 为例: 修改 `config.env.sh` 文件中下面的环境变量
-
-```
-RESOURCE_STORAGE_TYPE=S3
-RESOURCE_UPLOAD_PATH=/dolphinscheduler
-FS_DEFAULT_FS=s3a://BUCKET_NAME
-FS_S3A_ENDPOINT=http://MINIO_IP:9000
-FS_S3A_ACCESS_KEY=MINIO_ACCESS_KEY
-FS_S3A_SECRET_KEY=MINIO_SECRET_KEY
-```
-
-`BUCKET_NAME`, `MINIO_IP`, `MINIO_ACCESS_KEY` 和 `MINIO_SECRET_KEY` 需要被修改为实际值
-
-> **注意**: `MINIO_IP` 只能使用 IP 而非域名, 因为 DolphinScheduler 尚不支持 S3 路径风格访问 (S3 path style access)
-
-### 如何配置 SkyWalking?
-
-修改 `config.env.sh` 文件中的 SKYWALKING 环境变量
-
-```
-SKYWALKING_ENABLE=true
-SW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11800
-SW_GRPC_LOG_SERVER_HOST=127.0.0.1
-SW_GRPC_LOG_SERVER_PORT=11800
-```
-
-## 附录-环境变量
-
-### 数据库
-
-**`DATABASE_TYPE`**
-
-配置`database`的`TYPE`, 默认值 `postgresql`。
-
-**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
-
-**`DATABASE_DRIVER`**
-
-配置`database`的`DRIVER`, 默认值 `org.postgresql.Driver`。
-
-**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
-
-**`DATABASE_HOST`**
-
-配置`database`的`HOST`, 默认值 `127.0.0.1`。
-
-**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
-
-**`DATABASE_PORT`**
-
-配置`database`的`PORT`, 默认值 `5432`。
-
-**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
-
-**`DATABASE_USERNAME`**
-
-配置`database`的`USERNAME`, 默认值 `root`。
-
-**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
-
-**`DATABASE_PASSWORD`**
-
-配置`database`的`PASSWORD`, 默认值 `root`。
-
-**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
-
-**`DATABASE_DATABASE`**
-
-配置`database`的`DATABASE`, 默认值 `dolphinscheduler`。
-
-**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
-
-**`DATABASE_PARAMS`**
-
-配置`database`的`PARAMS`, 默认值 `characterEncoding=utf8`。
-
-**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`、`alert-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
-
-### ZooKeeper
-
-**`ZOOKEEPER_QUORUM`**
-
-配置`dolphinscheduler`的`Zookeeper`地址, 默认值 `127.0.0.1:2181`。
-
-**注意**: 当运行`dolphinscheduler`中`master-server`、`worker-server`、`api-server`这些服务时,必须指定这个环境变量,以便于你更好的搭建分布式服务。
-
-**`ZOOKEEPER_ROOT`**
-
-配置`dolphinscheduler`在`zookeeper`中数据存储的根目录,默认值 `/dolphinscheduler`。
-
-### 通用
-
-**`DOLPHINSCHEDULER_OPTS`**
-
-配置`dolphinscheduler`的`jvm options`,适用于`master-server`、`worker-server`、`api-server`、`alert-server`,默认值 `""`、
-
-**`DATA_BASEDIR_PATH`**
-
-用户数据目录, 用户自己配置, 请确保这个目录存在并且用户读写权限, 默认值 `/tmp/dolphinscheduler`。
-
-**`RESOURCE_STORAGE_TYPE`**
-
-配置`dolphinscheduler`的资源存储类型,可选项为 `HDFS`、`S3`、`NONE`,默认值 `HDFS`。
-
-**`RESOURCE_UPLOAD_PATH`**
-
-配置`HDFS/S3`上的资源存储路径,默认值 `/dolphinscheduler`。
-
-**`FS_DEFAULT_FS`**
-
-配置资源存储的文件系统协议,如 `file:///`, `hdfs://mycluster:8020` or `s3a://dolphinscheduler`,默认值 `file:///`。
-
-**`FS_S3A_ENDPOINT`**
-
-当`RESOURCE_STORAGE_TYPE=S3`时,需要配置`S3`的访问路径,默认值 `s3.xxx.amazonaws.com`。
-
-**`FS_S3A_ACCESS_KEY`**
-
-当`RESOURCE_STORAGE_TYPE=S3`时,需要配置`S3`的`s3 access key`,默认值 `xxxxxxx`。
-
-**`FS_S3A_SECRET_KEY`**
-
-当`RESOURCE_STORAGE_TYPE=S3`时,需要配置`S3`的`s3 secret key`,默认值 `xxxxxxx`。
-
-**`HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE`**
-
-配置`dolphinscheduler`是否启用kerberos,默认值 `false`。
-
-**`JAVA_SECURITY_KRB5_CONF_PATH`**
-
-配置`dolphinscheduler`的java.security.krb5.conf路径,默认值 `/opt/krb5.conf`。
-
-**`LOGIN_USER_KEYTAB_USERNAME`**
-
-配置`dolphinscheduler`登录用户的keytab用户名,默认值 `hdfs@HADOOP.COM`。
-
-**`LOGIN_USER_KEYTAB_PATH`**
-
-配置`dolphinscheduler`登录用户的keytab路径,默认值 `/opt/hdfs.keytab`。
-
-**`KERBEROS_EXPIRE_TIME`**
-
-配置`dolphinscheduler`的kerberos过期时间,单位为小时,默认值 `2`。
-
-**`HDFS_ROOT_USER`**
-
-当`RESOURCE_STORAGE_TYPE=HDFS`时,配置`dolphinscheduler`的hdfs的root用户名,默认值 `hdfs`。
-
-**`RESOURCE_MANAGER_HTTPADDRESS_PORT`**
-
-配置`dolphinscheduler`的resource manager httpaddress 端口,默认值 `8088`。
-
-**`YARN_RESOURCEMANAGER_HA_RM_IDS`**
-
-配置`dolphinscheduler`的yarn resourcemanager ha rm ids,默认值 `空`。
-
-**`YARN_APPLICATION_STATUS_ADDRESS`**
-
-配置`dolphinscheduler`的yarn application status地址,默认值 `http://ds1:%s/ws/v1/cluster/apps/%s`。
-
-**`SKYWALKING_ENABLE`**
-
-配置`skywalking`是否启用. 默认值 `false`。
-
-**`SW_AGENT_COLLECTOR_BACKEND_SERVICES`**
-
-配置`skywalking`的collector后端地址. 默认值 `127.0.0.1:11800`。
-
-**`SW_GRPC_LOG_SERVER_HOST`**
-
-配置`skywalking`的grpc服务主机或IP. 默认值 `127.0.0.1`。
-
-**`SW_GRPC_LOG_SERVER_PORT`**
-
-配置`skywalking`的grpc服务端口. 默认值 `11800`。
-
-**`HADOOP_HOME`**
-
-配置`dolphinscheduler`的`HADOOP_HOME`,默认值 `/opt/soft/hadoop`。
-
-**`HADOOP_CONF_DIR`**
-
-配置`dolphinscheduler`的`HADOOP_CONF_DIR`,默认值 `/opt/soft/hadoop/etc/hadoop`。
-
-**`SPARK_HOME1`**
-
-配置`dolphinscheduler`的`SPARK_HOME1`,默认值 `/opt/soft/spark1`。
-
-**`SPARK_HOME2`**
-
-配置`dolphinscheduler`的`SPARK_HOME2`,默认值 `/opt/soft/spark2`。
-
-**`PYTHON_HOME`**
-
-配置`dolphinscheduler`的`PYTHON_HOME`,默认值 `/usr/bin/python`。
-
-**`JAVA_HOME`**
-
-配置`dolphinscheduler`的`JAVA_HOME`,默认值 `/usr/local/openjdk-8`。
-
-**`HIVE_HOME`**
-
-配置`dolphinscheduler`的`HIVE_HOME`,默认值 `/opt/soft/hive`。
-
-**`FLINK_HOME`**
-
-配置`dolphinscheduler`的`FLINK_HOME`,默认值 `/opt/soft/flink`。
-
-**`DATAX_HOME`**
-
-配置`dolphinscheduler`的`DATAX_HOME`,默认值 `/opt/soft/datax`。
-
-### Master Server
-
-**`MASTER_SERVER_OPTS`**
-
-配置`master-server`的`jvm options`,默认值 `-Xms1g -Xmx1g -Xmn512m`。
-
-**`MASTER_EXEC_THREADS`**
-
-配置`master-server`中的执行线程数量,默认值 `100`。
-
-**`MASTER_EXEC_TASK_NUM`**
-
-配置`master-server`中的执行任务数量,默认值 `20`。
-
-**`MASTER_DISPATCH_TASK_NUM`**
-
-配置`master-server`中的派发任务数量,默认值 `3`。
-
-**`MASTER_HOST_SELECTOR`**
-
-配置`master-server`中派发任务时worker host的选择器,可选值为`Random`, `RoundRobin`和`LowerWeight`,默认值 `LowerWeight`。
-
-**`MASTER_HEARTBEAT_INTERVAL`**
-
-配置`master-server`中的心跳交互时间,默认值 `10`。
-
-**`MASTER_TASK_COMMIT_RETRYTIMES`**
-
-配置`master-server`中的任务提交重试次数,默认值 `5`。
-
-**`MASTER_TASK_COMMIT_INTERVAL`**
-
-配置`master-server`中的任务提交交互时间,默认值 `1`。
-
-**`MASTER_MAX_CPULOAD_AVG`**
-
-配置`master-server`中的CPU中的`load average`值,默认值 `-1`。
-
-**`MASTER_RESERVED_MEMORY`**
-
-配置`master-server`的保留内存,单位为G,默认值 `0.3`。
-
-### Worker Server
-
-**`WORKER_SERVER_OPTS`**
-
-配置`worker-server`的`jvm options`,默认值 `-Xms1g -Xmx1g -Xmn512m`。
-
-**`WORKER_EXEC_THREADS`**
-
-配置`worker-server`中的执行线程数量,默认值 `100`。
-
-**`WORKER_HEARTBEAT_INTERVAL`**
-
-配置`worker-server`中的心跳交互时间,默认值 `10`。
-
-**`WORKER_MAX_CPULOAD_AVG`**
-
-配置`worker-server`中的CPU中的最大`load average`值,默认值 `-1`。
-
-**`WORKER_RESERVED_MEMORY`**
-
-配置`worker-server`的保留内存,单位为G,默认值 `0.3`。
-
-**`WORKER_GROUPS`**
-
-配置`worker-server`的分组,默认值 `default`。
-
-### Alert Server
-
-**`ALERT_SERVER_OPTS`**
-
-配置`alert-server`的`jvm options`,默认值 `-Xms512m -Xmx512m -Xmn256m`。
-
-**`XLS_FILE_PATH`**
-
-配置`alert-server`的`XLS`文件的存储路径,默认值 `/tmp/xls`。
-
-**`MAIL_SERVER_HOST`**
-
-配置`alert-server`的邮件服务地址,默认值 `空`。
-
-**`MAIL_SERVER_PORT`**
-
-配置`alert-server`的邮件服务端口,默认值 `空`。
-
-**`MAIL_SENDER`**
-
-配置`alert-server`的邮件发送人,默认值 `空`。
-
-**`MAIL_USER=`**
-
-配置`alert-server`的邮件服务用户名,默认值 `空`。
-
-**`MAIL_PASSWD`**
-
-配置`alert-server`的邮件服务用户密码,默认值 `空`。
-
-**`MAIL_SMTP_STARTTLS_ENABLE`**
-
-配置`alert-server`的邮件服务是否启用TLS,默认值 `true`。
-
-**`MAIL_SMTP_SSL_ENABLE`**
-
-配置`alert-server`的邮件服务是否启用SSL,默认值 `false`。
-
-**`MAIL_SMTP_SSL_TRUST`**
-
-配置`alert-server`的邮件服务SSL的信任地址,默认值 `空`。
-
-**`ENTERPRISE_WECHAT_ENABLE`**
-
-配置`alert-server`的邮件服务是否启用企业微信,默认值 `false`。
-
-**`ENTERPRISE_WECHAT_CORP_ID`**
-
-配置`alert-server`的邮件服务企业微信`ID`,默认值 `空`。
-
-**`ENTERPRISE_WECHAT_SECRET`**
-
-配置`alert-server`的邮件服务企业微信`SECRET`,默认值 `空`。
-
-**`ENTERPRISE_WECHAT_AGENT_ID`**
-
-配置`alert-server`的邮件服务企业微信`AGENT_ID`,默认值 `空`。
-
-**`ENTERPRISE_WECHAT_USERS`**
-
-配置`alert-server`的邮件服务企业微信`USERS`,默认值 `空`。
-
-### Api Server
-
-**`API_SERVER_OPTS`**
-
-配置`api-server`的`jvm options`,默认值 `-Xms512m -Xmx512m -Xmn256m`。
+可以通过环境变量来修改 Docker 运行的配置,我们在沿用已有的 PostgreSQL 和 ZooKeeper 服务中就通过环境变量修改了 Docker 的数据库配置和
+注册中心配置,关于全部的配置环境可以查看[全部的配置文件](https://github.com/apache/dolphinscheduler/blob/3.0.0-alpha/script/env/dolphinscheduler_env.sh) 了解 <!-- markdown-link-check-disable-line -->
diff --git a/docs/docs/zh/guide/task/datax.md b/docs/docs/zh/guide/task/datax.md
index 5c0501a2f7..ffa0a39f14 100644
--- a/docs/docs/zh/guide/task/datax.md
+++ b/docs/docs/zh/guide/task/datax.md
@@ -40,7 +40,7 @@ DataX 任务类型,用于执行 DataX 程序。对于 DataX 节点,worker
### 在 DolphinScheduler 中配置 DataX 环境
-若生产环境中要是使用到 DataX 任务类型,则需要先配置好所需的环境。配置文件如下:`/dolphinscheduler/conf/env/dolphinscheduler_env.sh`。
+若生产环境中要是使用到 DataX 任务类型,则需要先配置好所需的环境。配置文件如下:`worker/bin/dolphinscheduler_env.sh`。

diff --git a/docs/docs/zh/guide/task/flink.md b/docs/docs/zh/guide/task/flink.md
index 4e415164e9..4c52af806a 100644
--- a/docs/docs/zh/guide/task/flink.md
+++ b/docs/docs/zh/guide/task/flink.md
@@ -46,7 +46,7 @@ Flink 任务类型,用于执行 Flink 程序。对于 Flink 节点,worker
#### 在 DolphinScheduler 中配置 flink 环境
-若生产环境中要是使用到 flink 任务类型,则需要先配置好所需的环境。配置文件如下:`/dolphinscheduler/conf/env/dolphinscheduler_env.sh`。
+若生产环境中要是使用到 flink 任务类型,则需要先配置好所需的环境。配置文件如下:`worker/bin/dolphinscheduler_env.sh`。

diff --git a/docs/docs/zh/guide/task/map-reduce.md b/docs/docs/zh/guide/task/map-reduce.md
index e0f647442e..259ac3fec7 100644
--- a/docs/docs/zh/guide/task/map-reduce.md
+++ b/docs/docs/zh/guide/task/map-reduce.md
@@ -54,7 +54,7 @@ MapReduce(MR) 任务类型,用于执行 MapReduce 程序。对于 MapReduce
#### 在 DolphinScheduler 中配置 MapReduce 环境
-若生产环境中要是使用到 MapReduce 任务类型,则需要先配置好所需的环境。配置文件如下:`/dolphinscheduler/conf/env/dolphinscheduler_env.sh`。
+若生产环境中要是使用到 MapReduce 任务类型,则需要先配置好所需的环境。配置文件如下:`worker/bin/dolphinscheduler_env.sh`。

diff --git a/docs/docs/zh/guide/task/shell.md b/docs/docs/zh/guide/task/shell.md
index dd896c5bc8..a656c098e7 100644
--- a/docs/docs/zh/guide/task/shell.md
+++ b/docs/docs/zh/guide/task/shell.md
@@ -45,4 +45,4 @@ Shell 任务类型,用于创建 Shell 类型的任务并执行一系列的 She
## 注意事项
-无
\ No newline at end of file
+Shell 任务类型通过解析任务日志是否包含 ```application_xxx_xxx``` 的内容来判断是否 Yarn 任务,如果是则会将相应的 ```application_id``` 的状态作为当前 Shell 节点的运行状态判断,此时如果操作停止工作流则会 Kill 相应的 ```application_id```
\ No newline at end of file
diff --git a/docs/docs/zh/guide/task/spark.md b/docs/docs/zh/guide/task/spark.md
index 442eb8b3ee..015c0fdd11 100644
--- a/docs/docs/zh/guide/task/spark.md
+++ b/docs/docs/zh/guide/task/spark.md
@@ -46,7 +46,7 @@ Spark 任务类型,用于执行 Spark 程序。对于 Spark 节点,worker
#### 在 DolphinScheduler 中配置 Spark 环境
-若生产环境中要是使用到 Spark 任务类型,则需要先配置好所需的环境。配置文件如下:`/dolphinscheduler/conf/env/dolphinscheduler_env.sh`。
+若生产环境中要是使用到 Spark 任务类型,则需要先配置好所需的环境。配置文件如下:`worker/bin/dolphinscheduler_env.sh`。

diff --git a/docs/docs/zh/guide/upgrade.md b/docs/docs/zh/guide/upgrade.md
index 3d1b3016aa..ee8b7e436c 100644
--- a/docs/docs/zh/guide/upgrade.md
+++ b/docs/docs/zh/guide/upgrade.md
@@ -13,24 +13,21 @@
- 以下升级操作都需要在新版本的目录进行
## 4. 数据库升级
-- 修改conf/datasource.properties中的下列属性
-
-- 如果选择 MySQL,请注释掉 PostgreSQL 相关配置(反之同理), 还需要手动添加 [[ mysql-connector-java 驱动 jar ](https://downloads.MySQL.com/archives/c-j/)] 包到 lib 目录下,这里下载的是mysql-connector-java-8.0.16.jar,然后正确配置数据库连接相关信息
-
- ```properties
- # postgre
- #spring.datasource.driver-class-name=org.postgresql.Driver
- #spring.datasource.url=jdbc:postgresql://localhost:5432/dolphinscheduler
- # mysql
- spring.datasource.driver-class-name=com.mysql.jdbc.Driver
- spring.datasource.url=jdbc:mysql://xxx:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true 需要修改ip,本机localhost即可
- spring.datasource.username=xxx 需要修改为上面的{user}值
- spring.datasource.password=xxx 需要修改为上面的{password}值
+
+- 如果选择 MySQL,请修改`tools/bin/dolphinscheduler_env.sh`中的如下配置({user}和{password}改成你数据库的用户名和密码), 还需要手动添加 [ mysql-connector-java 驱动 jar ](https://downloads.MySQL.com/archives/c-j/) 包到 lib 目录(`./tools/lib`)下,这里下载的是mysql-connector-java-8.0.16.jar
+
+ ```shell
+ export DATABASE=${DATABASE:-mysql}
+ export SPRING_PROFILES_ACTIVE=${DATABASE}
+ export SPRING_DATASOURCE_DRIVER_CLASS_NAME=com.mysql.cj.jdbc.Driver
+ export SPRING_DATASOURCE_URL=jdbc:mysql://127.0.0.1:3306/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8
+ export SPRING_DATASOURCE_USERNAME={user}
+ export SPRING_DATASOURCE_PASSWORD={password}
```
- 执行数据库升级脚本
-`sh ./script/upgrade-dolphinscheduler.sh`
+`sh ./tools/bin/upgrade-schema.sh`
## 5. 服务升级
diff --git a/docs/img/alert/alert_scenarios_en.png b/docs/img/alert/alert_scenarios_en.png
new file mode 100644
index 0000000000..adc1b1ad6f
Binary files /dev/null and b/docs/img/alert/alert_scenarios_en.png differ
diff --git a/docs/img/alert/alert_scenarios_zh.png b/docs/img/alert/alert_scenarios_zh.png
new file mode 100644
index 0000000000..bd2d6236d9
Binary files /dev/null and b/docs/img/alert/alert_scenarios_zh.png differ
diff --git a/docs/img/alert/alert_step_2.png b/docs/img/alert/alert_step_2.png
index dabd5e9a03..75fbe3ee21 100644
Binary files a/docs/img/alert/alert_step_2.png and b/docs/img/alert/alert_step_2.png differ
diff --git a/docs/img/alert/alert_step_3.png b/docs/img/alert/alert_step_3.png
index 125e4945aa..0501d84ab4 100644
Binary files a/docs/img/alert/alert_step_3.png and b/docs/img/alert/alert_step_3.png differ
diff --git a/docs/img/alert/alert_step_4.png b/docs/img/alert/alert_step_4.png
index ed62cc67d4..0ed8893d98 100644
Binary files a/docs/img/alert/alert_step_4.png and b/docs/img/alert/alert_step_4.png differ
diff --git a/docs/img/alert/enterprise-wechat-app-msg-md.png b/docs/img/alert/enterprise-wechat-app-msg-md.png
new file mode 100644
index 0000000000..6503e0eb8b
Binary files /dev/null and b/docs/img/alert/enterprise-wechat-app-msg-md.png differ
diff --git a/docs/img/alert/enterprise-wechat-app-msg.png b/docs/img/alert/enterprise-wechat-app-msg.png
new file mode 100644
index 0000000000..d439171345
Binary files /dev/null and b/docs/img/alert/enterprise-wechat-app-msg.png differ
diff --git a/docs/img/alert/enterprise-wechat-create-group.png b/docs/img/alert/enterprise-wechat-create-group.png
new file mode 100644
index 0000000000..3e7d304f38
Binary files /dev/null and b/docs/img/alert/enterprise-wechat-create-group.png differ
diff --git a/docs/img/alert/enterprise-wechat-group-msg-md.png b/docs/img/alert/enterprise-wechat-group-msg-md.png
new file mode 100644
index 0000000000..3fb5da33c9
Binary files /dev/null and b/docs/img/alert/enterprise-wechat-group-msg-md.png differ
diff --git a/docs/img/alert/enterprise-wechat-group-msg.png b/docs/img/alert/enterprise-wechat-group-msg.png
new file mode 100644
index 0000000000..39830bd312
Binary files /dev/null and b/docs/img/alert/enterprise-wechat-group-msg.png differ
diff --git a/docs/img/alert/enterprise-wechat-query-userid.png b/docs/img/alert/enterprise-wechat-query-userid.png
new file mode 100644
index 0000000000..a7fb3b50bb
Binary files /dev/null and b/docs/img/alert/enterprise-wechat-query-userid.png differ
diff --git a/docs/img/alert/http-get-example.png b/docs/img/alert/http-get-example.png
new file mode 100644
index 0000000000..a6c29b48c9
Binary files /dev/null and b/docs/img/alert/http-get-example.png differ
diff --git a/docs/img/alert/http-post-example.png b/docs/img/alert/http-post-example.png
new file mode 100644
index 0000000000..5cf0d3e64d
Binary files /dev/null and b/docs/img/alert/http-post-example.png differ
diff --git a/docs/img/alert/script-plugin.png b/docs/img/alert/script-plugin.png
new file mode 100644
index 0000000000..65f8396f9c
Binary files /dev/null and b/docs/img/alert/script-plugin.png differ
diff --git a/docs/img/alert/wechat-app-form-example.png b/docs/img/alert/wechat-app-form-example.png
new file mode 100644
index 0000000000..d425da784b
Binary files /dev/null and b/docs/img/alert/wechat-app-form-example.png differ
diff --git a/docs/img/alert/wechat-group-form-example.png b/docs/img/alert/wechat-group-form-example.png
new file mode 100644
index 0000000000..c289ebc95c
Binary files /dev/null and b/docs/img/alert/wechat-group-form-example.png differ
diff --git a/docs/img/tasks/demo/custom_sql_check.png b/docs/img/tasks/demo/custom_sql_check.png
new file mode 100644
index 0000000000..b1e1b9b6f8
Binary files /dev/null and b/docs/img/tasks/demo/custom_sql_check.png differ
diff --git a/docs/img/tasks/demo/enumeration_check.png b/docs/img/tasks/demo/enumeration_check.png
new file mode 100644
index 0000000000..e2e16554cf
Binary files /dev/null and b/docs/img/tasks/demo/enumeration_check.png differ
diff --git a/docs/img/tasks/demo/field_length_check.png b/docs/img/tasks/demo/field_length_check.png
new file mode 100644
index 0000000000..17c1822ba1
Binary files /dev/null and b/docs/img/tasks/demo/field_length_check.png differ
diff --git a/docs/img/tasks/demo/multi_table_accuracy_check.png b/docs/img/tasks/demo/multi_table_accuracy_check.png
new file mode 100644
index 0000000000..c78fbc0e0c
Binary files /dev/null and b/docs/img/tasks/demo/multi_table_accuracy_check.png differ
diff --git a/docs/img/tasks/demo/multi_table_comparison_check.png b/docs/img/tasks/demo/multi_table_comparison_check.png
new file mode 100644
index 0000000000..f027158337
Binary files /dev/null and b/docs/img/tasks/demo/multi_table_comparison_check.png differ
diff --git a/docs/img/tasks/demo/null_check.png b/docs/img/tasks/demo/null_check.png
new file mode 100644
index 0000000000..46beaa5b42
Binary files /dev/null and b/docs/img/tasks/demo/null_check.png differ
diff --git a/docs/img/tasks/demo/regexp_check.png b/docs/img/tasks/demo/regexp_check.png
new file mode 100644
index 0000000000..de5681451f
Binary files /dev/null and b/docs/img/tasks/demo/regexp_check.png differ
diff --git a/docs/img/tasks/demo/result.png b/docs/img/tasks/demo/result.png
new file mode 100644
index 0000000000..07f93b4eb9
Binary files /dev/null and b/docs/img/tasks/demo/result.png differ
diff --git a/docs/img/tasks/demo/rule_detail.png b/docs/img/tasks/demo/rule_detail.png
new file mode 100644
index 0000000000..bf5a2d066d
Binary files /dev/null and b/docs/img/tasks/demo/rule_detail.png differ
diff --git a/docs/img/tasks/demo/rule_list.png b/docs/img/tasks/demo/rule_list.png
new file mode 100644
index 0000000000..f61224f5a1
Binary files /dev/null and b/docs/img/tasks/demo/rule_list.png differ
diff --git a/docs/img/tasks/demo/shell.jpg b/docs/img/tasks/demo/shell.jpg
index 3a01184690..1c5e2f5e18 100644
Binary files a/docs/img/tasks/demo/shell.jpg and b/docs/img/tasks/demo/shell.jpg differ
diff --git a/docs/img/tasks/demo/shell_custom_param.jpg b/docs/img/tasks/demo/shell_custom_param.jpg
index c52b0d14f1..6dd45a6386 100644
Binary files a/docs/img/tasks/demo/shell_custom_param.jpg and b/docs/img/tasks/demo/shell_custom_param.jpg differ
diff --git a/docs/img/tasks/demo/table_count_check.png b/docs/img/tasks/demo/table_count_check.png
new file mode 100644
index 0000000000..4bcded6033
Binary files /dev/null and b/docs/img/tasks/demo/table_count_check.png differ
diff --git a/docs/img/tasks/demo/timeliness_check.png b/docs/img/tasks/demo/timeliness_check.png
new file mode 100644
index 0000000000..86505f6b53
Binary files /dev/null and b/docs/img/tasks/demo/timeliness_check.png differ
diff --git a/docs/img/tasks/demo/uniqueness_check.png b/docs/img/tasks/demo/uniqueness_check.png
new file mode 100644
index 0000000000..708d374458
Binary files /dev/null and b/docs/img/tasks/demo/uniqueness_check.png differ