You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by gi...@apache.org on 2022/12/15 02:37:58 UTC
[dolphinscheduler-website] branch asf-site updated: Automated deployment: ad7490895fbb1b067bbe5bc155de2b9352f5c5ef
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler-website.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 73a72530a7 Automated deployment: ad7490895fbb1b067bbe5bc155de2b9352f5c5ef
73a72530a7 is described below
commit 73a72530a70809a925eafabc96d1e84841243e4e
Author: github-actions[bot] <gi...@users.noreply.github.com>
AuthorDate: Thu Dec 15 02:37:50 2022 +0000
Automated deployment: ad7490895fbb1b067bbe5bc155de2b9352f5c5ef
---
asset-manifest.json | 6 +-
data/doc/en-us.json | 12831 -------------------
index.html | 2 +-
pages/doc/en-us/1.3.6/standalone-deployment.json | 2 +-
pages/doc/en-us/1.3.8/standalone-deployment.json | 2 +-
pages/doc/en-us/2.0.0/architecture/design.json | 2 +-
.../doc/en-us/2.0.0/guide/expansion-reduction.json | 2 +-
.../en-us/2.0.0/guide/installation/cluster.json | 2 +-
.../en-us/2.0.0/guide/installation/kubernetes.json | 2 +-
.../2.0.0/guide/installation/pseudo-cluster.json | 2 +-
.../en-us/2.0.0/guide/installation/standalone.json | 2 +-
pages/doc/en-us/2.0.0/guide/parameter/context.json | 2 +-
pages/doc/en-us/2.0.0/guide/parameter/local.json | 2 +-
.../doc/en-us/2.0.0/guide/parameter/priority.json | 2 +-
.../2.0.0/guide/project/workflow-definition.json | 2 +-
.../2.0.0/guide/project/workflow-instance.json | 2 +-
pages/doc/en-us/2.0.0/guide/resource.json | 2 +-
pages/doc/en-us/2.0.0/guide/task/conditions.json | 2 +-
pages/doc/en-us/2.0.0/guide/task/switch.json | 2 +-
pages/doc/en-us/2.0.0/guide/upgrade.json | 2 +-
pages/doc/en-us/2.0.1/architecture/design.json | 2 +-
pages/doc/en-us/2.0.1/changelog.json | 2 +-
.../doc/en-us/2.0.1/guide/expansion-reduction.json | 2 +-
.../en-us/2.0.1/guide/installation/cluster.json | 2 +-
.../en-us/2.0.1/guide/installation/kubernetes.json | 2 +-
.../2.0.1/guide/installation/pseudo-cluster.json | 2 +-
.../en-us/2.0.1/guide/installation/standalone.json | 2 +-
pages/doc/en-us/2.0.1/guide/parameter/context.json | 2 +-
pages/doc/en-us/2.0.1/guide/parameter/local.json | 2 +-
.../doc/en-us/2.0.1/guide/parameter/priority.json | 2 +-
.../2.0.1/guide/project/workflow-definition.json | 2 +-
.../2.0.1/guide/project/workflow-instance.json | 2 +-
pages/doc/en-us/2.0.1/guide/resource.json | 2 +-
pages/doc/en-us/2.0.1/guide/task/conditions.json | 2 +-
pages/doc/en-us/2.0.1/guide/task/switch.json | 2 +-
pages/doc/en-us/2.0.1/guide/upgrade.json | 2 +-
pages/doc/en-us/2.0.2/architecture/design.json | 2 +-
pages/doc/en-us/2.0.2/changelog.json | 2 +-
.../doc/en-us/2.0.2/guide/expansion-reduction.json | 2 +-
.../en-us/2.0.2/guide/installation/cluster.json | 2 +-
.../en-us/2.0.2/guide/installation/kubernetes.json | 2 +-
.../2.0.2/guide/installation/pseudo-cluster.json | 2 +-
.../en-us/2.0.2/guide/installation/standalone.json | 2 +-
pages/doc/en-us/2.0.2/guide/parameter/context.json | 2 +-
pages/doc/en-us/2.0.2/guide/parameter/local.json | 2 +-
.../doc/en-us/2.0.2/guide/parameter/priority.json | 2 +-
.../2.0.2/guide/project/workflow-definition.json | 2 +-
.../2.0.2/guide/project/workflow-instance.json | 2 +-
pages/doc/en-us/2.0.2/guide/resource.json | 2 +-
pages/doc/en-us/2.0.2/guide/task/conditions.json | 2 +-
pages/doc/en-us/2.0.2/guide/task/switch.json | 2 +-
pages/doc/en-us/2.0.2/guide/upgrade.json | 2 +-
pages/doc/en-us/2.0.3/architecture/design.json | 2 +-
pages/doc/en-us/2.0.3/changelog.json | 2 +-
.../doc/en-us/2.0.3/guide/expansion-reduction.json | 2 +-
.../en-us/2.0.3/guide/installation/cluster.json | 2 +-
.../en-us/2.0.3/guide/installation/kubernetes.json | 2 +-
.../2.0.3/guide/installation/pseudo-cluster.json | 2 +-
.../en-us/2.0.3/guide/installation/standalone.json | 2 +-
pages/doc/en-us/2.0.3/guide/parameter/context.json | 2 +-
pages/doc/en-us/2.0.3/guide/parameter/local.json | 2 +-
.../doc/en-us/2.0.3/guide/parameter/priority.json | 2 +-
.../2.0.3/guide/project/workflow-definition.json | 2 +-
pages/doc/en-us/2.0.3/guide/resource.json | 2 +-
pages/doc/en-us/2.0.3/guide/task/conditions.json | 2 +-
pages/doc/en-us/2.0.3/guide/task/switch.json | 2 +-
pages/doc/en-us/2.0.3/guide/upgrade.json | 2 +-
pages/doc/en-us/2.0.5/architecture/design.json | 2 +-
.../doc/en-us/2.0.5/guide/expansion-reduction.json | 2 +-
.../en-us/2.0.5/guide/installation/cluster.json | 2 +-
.../en-us/2.0.5/guide/installation/kubernetes.json | 2 +-
.../2.0.5/guide/installation/pseudo-cluster.json | 2 +-
.../en-us/2.0.5/guide/installation/standalone.json | 2 +-
pages/doc/en-us/2.0.5/guide/parameter/context.json | 2 +-
pages/doc/en-us/2.0.5/guide/parameter/local.json | 2 +-
.../doc/en-us/2.0.5/guide/parameter/priority.json | 2 +-
.../2.0.5/guide/project/workflow-definition.json | 2 +-
.../2.0.5/guide/project/workflow-instance.json | 2 +-
pages/doc/en-us/2.0.5/guide/resource.json | 2 +-
pages/doc/en-us/2.0.5/guide/task/conditions.json | 2 +-
pages/doc/en-us/2.0.5/guide/task/switch.json | 2 +-
pages/doc/en-us/2.0.5/guide/upgrade.json | 2 +-
pages/doc/en-us/2.0.6/architecture/design.json | 2 +-
.../doc/en-us/2.0.6/guide/expansion-reduction.json | 2 +-
.../en-us/2.0.6/guide/installation/cluster.json | 2 +-
.../en-us/2.0.6/guide/installation/kubernetes.json | 2 +-
.../2.0.6/guide/installation/pseudo-cluster.json | 2 +-
.../en-us/2.0.6/guide/installation/standalone.json | 2 +-
pages/doc/en-us/2.0.6/guide/parameter/context.json | 2 +-
pages/doc/en-us/2.0.6/guide/parameter/local.json | 2 +-
.../doc/en-us/2.0.6/guide/parameter/priority.json | 2 +-
.../2.0.6/guide/project/workflow-definition.json | 2 +-
.../2.0.6/guide/project/workflow-instance.json | 2 +-
pages/doc/en-us/2.0.6/guide/resource.json | 2 +-
pages/doc/en-us/2.0.6/guide/task/conditions.json | 2 +-
pages/doc/en-us/2.0.6/guide/task/switch.json | 2 +-
pages/doc/en-us/2.0.6/guide/upgrade.json | 2 +-
pages/doc/en-us/2.0.7/architecture/design.json | 2 +-
.../doc/en-us/2.0.7/guide/expansion-reduction.json | 2 +-
.../en-us/2.0.7/guide/installation/cluster.json | 2 +-
.../en-us/2.0.7/guide/installation/kubernetes.json | 2 +-
.../2.0.7/guide/installation/pseudo-cluster.json | 2 +-
.../en-us/2.0.7/guide/installation/standalone.json | 2 +-
pages/doc/en-us/2.0.7/guide/parameter/context.json | 2 +-
pages/doc/en-us/2.0.7/guide/parameter/local.json | 2 +-
.../doc/en-us/2.0.7/guide/parameter/priority.json | 2 +-
.../2.0.7/guide/project/workflow-definition.json | 2 +-
.../2.0.7/guide/project/workflow-instance.json | 2 +-
pages/doc/en-us/2.0.7/guide/resource.json | 2 +-
pages/doc/en-us/2.0.7/guide/task/conditions.json | 2 +-
pages/doc/en-us/2.0.7/guide/task/switch.json | 2 +-
pages/doc/en-us/2.0.7/guide/upgrade.json | 2 +-
pages/doc/en-us/3.0.0/architecture/design.json | 2 +-
pages/doc/en-us/3.0.0/guide/datasource/mysql.json | 2 +-
.../doc/en-us/3.0.0/guide/expansion-reduction.json | 2 +-
.../en-us/3.0.0/guide/installation/cluster.json | 2 +-
.../en-us/3.0.0/guide/installation/kubernetes.json | 2 +-
.../3.0.0/guide/installation/pseudo-cluster.json | 2 +-
.../en-us/3.0.0/guide/installation/standalone.json | 2 +-
pages/doc/en-us/3.0.0/guide/parameter/context.json | 2 +-
pages/doc/en-us/3.0.0/guide/parameter/local.json | 2 +-
.../doc/en-us/3.0.0/guide/parameter/priority.json | 2 +-
.../en-us/3.0.0/guide/project/task-definition.json | 2 +-
.../3.0.0/guide/project/workflow-definition.json | 2 +-
.../3.0.0/guide/project/workflow-instance.json | 2 +-
.../en-us/3.0.0/guide/resource/configuration.json | 2 +-
pages/doc/en-us/3.0.0/guide/task/conditions.json | 2 +-
pages/doc/en-us/3.0.0/guide/task/datax.json | 2 +-
pages/doc/en-us/3.0.0/guide/task/flink.json | 2 +-
pages/doc/en-us/3.0.0/guide/task/map-reduce.json | 2 +-
pages/doc/en-us/3.0.0/guide/task/python.json | 2 +-
pages/doc/en-us/3.0.0/guide/task/shell.json | 2 +-
pages/doc/en-us/3.0.0/guide/task/spark.json | 2 +-
pages/doc/en-us/3.0.0/guide/task/sql.json | 2 +-
pages/doc/en-us/3.0.0/guide/task/sub-process.json | 2 +-
pages/doc/en-us/3.0.0/guide/task/switch.json | 2 +-
pages/doc/en-us/3.0.0/guide/upgrade.json | 2 +-
pages/doc/en-us/3.0.1/architecture/design.json | 2 +-
pages/doc/en-us/3.0.1/guide/datasource/mysql.json | 2 +-
.../doc/en-us/3.0.1/guide/expansion-reduction.json | 2 +-
.../en-us/3.0.1/guide/installation/cluster.json | 2 +-
.../en-us/3.0.1/guide/installation/kubernetes.json | 2 +-
.../3.0.1/guide/installation/pseudo-cluster.json | 2 +-
.../en-us/3.0.1/guide/installation/standalone.json | 2 +-
pages/doc/en-us/3.0.1/guide/parameter/context.json | 2 +-
pages/doc/en-us/3.0.1/guide/parameter/local.json | 2 +-
.../doc/en-us/3.0.1/guide/parameter/priority.json | 2 +-
.../en-us/3.0.1/guide/project/task-definition.json | 2 +-
.../3.0.1/guide/project/workflow-definition.json | 2 +-
.../3.0.1/guide/project/workflow-instance.json | 2 +-
.../en-us/3.0.1/guide/resource/configuration.json | 2 +-
pages/doc/en-us/3.0.1/guide/task/conditions.json | 2 +-
pages/doc/en-us/3.0.1/guide/task/datax.json | 2 +-
pages/doc/en-us/3.0.1/guide/task/flink.json | 2 +-
pages/doc/en-us/3.0.1/guide/task/map-reduce.json | 2 +-
pages/doc/en-us/3.0.1/guide/task/python.json | 2 +-
pages/doc/en-us/3.0.1/guide/task/shell.json | 2 +-
pages/doc/en-us/3.0.1/guide/task/spark.json | 2 +-
pages/doc/en-us/3.0.1/guide/task/sql.json | 2 +-
pages/doc/en-us/3.0.1/guide/task/sub-process.json | 2 +-
pages/doc/en-us/3.0.1/guide/task/switch.json | 2 +-
pages/doc/en-us/3.0.1/guide/upgrade.json | 2 +-
pages/doc/en-us/3.0.2/architecture/design.json | 2 +-
pages/doc/en-us/3.0.2/guide/datasource/mysql.json | 2 +-
.../doc/en-us/3.0.2/guide/expansion-reduction.json | 2 +-
.../en-us/3.0.2/guide/installation/cluster.json | 2 +-
.../en-us/3.0.2/guide/installation/kubernetes.json | 2 +-
.../3.0.2/guide/installation/pseudo-cluster.json | 2 +-
.../en-us/3.0.2/guide/installation/standalone.json | 2 +-
pages/doc/en-us/3.0.2/guide/parameter/context.json | 2 +-
pages/doc/en-us/3.0.2/guide/parameter/local.json | 2 +-
.../doc/en-us/3.0.2/guide/parameter/priority.json | 2 +-
.../en-us/3.0.2/guide/project/task-definition.json | 2 +-
.../3.0.2/guide/project/workflow-definition.json | 2 +-
.../3.0.2/guide/project/workflow-instance.json | 2 +-
.../en-us/3.0.2/guide/resource/configuration.json | 2 +-
pages/doc/en-us/3.0.2/guide/task/conditions.json | 2 +-
pages/doc/en-us/3.0.2/guide/task/datax.json | 2 +-
pages/doc/en-us/3.0.2/guide/task/flink.json | 2 +-
pages/doc/en-us/3.0.2/guide/task/map-reduce.json | 2 +-
pages/doc/en-us/3.0.2/guide/task/python.json | 2 +-
pages/doc/en-us/3.0.2/guide/task/shell.json | 2 +-
pages/doc/en-us/3.0.2/guide/task/spark.json | 2 +-
pages/doc/en-us/3.0.2/guide/task/sql.json | 2 +-
pages/doc/en-us/3.0.2/guide/task/sub-process.json | 2 +-
pages/doc/en-us/3.0.2/guide/task/switch.json | 2 +-
pages/doc/en-us/3.0.2/guide/upgrade.json | 2 +-
pages/doc/en-us/3.0.3/architecture/design.json | 2 +-
pages/doc/en-us/3.0.3/guide/datasource/mysql.json | 2 +-
.../doc/en-us/3.0.3/guide/expansion-reduction.json | 2 +-
.../en-us/3.0.3/guide/installation/cluster.json | 2 +-
.../en-us/3.0.3/guide/installation/kubernetes.json | 2 +-
.../3.0.3/guide/installation/pseudo-cluster.json | 2 +-
.../en-us/3.0.3/guide/installation/standalone.json | 2 +-
pages/doc/en-us/3.0.3/guide/parameter/context.json | 2 +-
pages/doc/en-us/3.0.3/guide/parameter/local.json | 2 +-
.../doc/en-us/3.0.3/guide/parameter/priority.json | 2 +-
.../en-us/3.0.3/guide/project/task-definition.json | 2 +-
.../3.0.3/guide/project/workflow-definition.json | 2 +-
.../3.0.3/guide/project/workflow-instance.json | 2 +-
.../en-us/3.0.3/guide/resource/configuration.json | 2 +-
pages/doc/en-us/3.0.3/guide/task/conditions.json | 2 +-
pages/doc/en-us/3.0.3/guide/task/datax.json | 2 +-
pages/doc/en-us/3.0.3/guide/task/flink.json | 2 +-
pages/doc/en-us/3.0.3/guide/task/map-reduce.json | 2 +-
pages/doc/en-us/3.0.3/guide/task/python.json | 2 +-
pages/doc/en-us/3.0.3/guide/task/shell.json | 2 +-
pages/doc/en-us/3.0.3/guide/task/spark.json | 2 +-
pages/doc/en-us/3.0.3/guide/task/sql.json | 2 +-
pages/doc/en-us/3.0.3/guide/task/sub-process.json | 2 +-
pages/doc/en-us/3.0.3/guide/task/switch.json | 2 +-
pages/doc/en-us/3.0.3/guide/upgrade.json | 2 +-
pages/doc/en-us/3.1.0/architecture/design.json | 2 +-
pages/doc/en-us/3.1.0/guide/datasource/athena.json | 2 +-
pages/doc/en-us/3.1.0/guide/datasource/mysql.json | 2 +-
.../doc/en-us/3.1.0/guide/expansion-reduction.json | 2 +-
.../en-us/3.1.0/guide/installation/cluster.json | 2 +-
.../en-us/3.1.0/guide/installation/kubernetes.json | 2 +-
.../3.1.0/guide/installation/pseudo-cluster.json | 2 +-
.../en-us/3.1.0/guide/installation/standalone.json | 2 +-
pages/doc/en-us/3.1.0/guide/parameter/context.json | 2 +-
pages/doc/en-us/3.1.0/guide/parameter/local.json | 2 +-
.../doc/en-us/3.1.0/guide/parameter/priority.json | 2 +-
.../en-us/3.1.0/guide/project/task-definition.json | 2 +-
.../3.1.0/guide/project/workflow-definition.json | 2 +-
.../3.1.0/guide/project/workflow-instance.json | 2 +-
.../en-us/3.1.0/guide/resource/configuration.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/appendix.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/chunjun.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/conditions.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/datax.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/dependent.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/dinky.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/dvc.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/emr.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/flink.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/hive-cli.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/http.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/jupyter.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/kubernetes.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/map-reduce.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/mlflow.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/openmldb.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/pigeon.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/python.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/pytorch.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/sagemaker.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/seatunnel.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/shell.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/spark.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/sql.json | 2 +-
.../en-us/3.1.0/guide/task/stored-procedure.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/sub-process.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/switch.json | 2 +-
pages/doc/en-us/3.1.0/guide/task/zeppelin.json | 2 +-
pages/doc/en-us/3.1.0/guide/upgrade/upgrade.json | 2 +-
pages/doc/en-us/3.1.1/architecture/design.json | 2 +-
pages/doc/en-us/3.1.1/guide/datasource/athena.json | 2 +-
pages/doc/en-us/3.1.1/guide/datasource/mysql.json | 2 +-
.../doc/en-us/3.1.1/guide/expansion-reduction.json | 2 +-
.../en-us/3.1.1/guide/installation/cluster.json | 2 +-
.../en-us/3.1.1/guide/installation/kubernetes.json | 2 +-
.../3.1.1/guide/installation/pseudo-cluster.json | 2 +-
.../en-us/3.1.1/guide/installation/standalone.json | 2 +-
pages/doc/en-us/3.1.1/guide/parameter/context.json | 2 +-
pages/doc/en-us/3.1.1/guide/parameter/local.json | 2 +-
.../doc/en-us/3.1.1/guide/parameter/priority.json | 2 +-
.../en-us/3.1.1/guide/project/task-definition.json | 2 +-
.../3.1.1/guide/project/workflow-definition.json | 2 +-
.../3.1.1/guide/project/workflow-instance.json | 2 +-
.../en-us/3.1.1/guide/resource/configuration.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/appendix.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/chunjun.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/conditions.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/datax.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/dependent.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/dinky.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/dvc.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/emr.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/flink.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/hive-cli.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/http.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/jupyter.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/kubernetes.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/map-reduce.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/mlflow.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/openmldb.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/pigeon.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/python.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/pytorch.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/sagemaker.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/seatunnel.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/shell.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/spark.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/sql.json | 2 +-
.../en-us/3.1.1/guide/task/stored-procedure.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/sub-process.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/switch.json | 2 +-
pages/doc/en-us/3.1.1/guide/task/zeppelin.json | 2 +-
pages/doc/en-us/3.1.1/guide/upgrade/upgrade.json | 2 +-
pages/doc/en-us/3.1.2/architecture/design.json | 2 +-
pages/doc/en-us/3.1.2/guide/datasource/athena.json | 2 +-
pages/doc/en-us/3.1.2/guide/datasource/mysql.json | 2 +-
.../doc/en-us/3.1.2/guide/expansion-reduction.json | 2 +-
.../en-us/3.1.2/guide/installation/cluster.json | 2 +-
.../en-us/3.1.2/guide/installation/kubernetes.json | 2 +-
.../3.1.2/guide/installation/pseudo-cluster.json | 2 +-
.../en-us/3.1.2/guide/installation/standalone.json | 2 +-
pages/doc/en-us/3.1.2/guide/parameter/context.json | 2 +-
pages/doc/en-us/3.1.2/guide/parameter/local.json | 2 +-
.../doc/en-us/3.1.2/guide/parameter/priority.json | 2 +-
.../en-us/3.1.2/guide/project/task-definition.json | 2 +-
.../3.1.2/guide/project/workflow-definition.json | 2 +-
.../3.1.2/guide/project/workflow-instance.json | 2 +-
.../en-us/3.1.2/guide/resource/configuration.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/appendix.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/chunjun.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/conditions.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/datax.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/dependent.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/dinky.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/dvc.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/emr.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/flink.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/hive-cli.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/http.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/jupyter.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/kubernetes.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/map-reduce.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/mlflow.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/openmldb.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/pigeon.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/python.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/pytorch.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/sagemaker.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/seatunnel.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/shell.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/spark.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/sql.json | 2 +-
.../en-us/3.1.2/guide/task/stored-procedure.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/sub-process.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/switch.json | 2 +-
pages/doc/en-us/3.1.2/guide/task/zeppelin.json | 2 +-
pages/doc/en-us/3.1.2/guide/upgrade/upgrade.json | 2 +-
pages/doc/zh-cn/1.2.0/upgrade.json | 2 +-
pages/doc/zh-cn/1.2.1/upgrade.json | 2 +-
pages/doc/zh-cn/1.3.1/upgrade.json | 2 +-
pages/doc/zh-cn/1.3.2/upgrade.json | 2 +-
pages/doc/zh-cn/1.3.3/upgrade.json | 2 +-
pages/doc/zh-cn/1.3.4/upgrade.json | 2 +-
pages/doc/zh-cn/1.3.5/upgrade.json | 2 +-
pages/doc/zh-cn/1.3.6/upgrade.json | 2 +-
pages/doc/zh-cn/1.3.8/upgrade.json | 2 +-
pages/doc/zh-cn/1.3.9/upgrade.json | 2 +-
pages/doc/zh-cn/2.0.0/architecture/design.json | 2 +-
.../doc/zh-cn/2.0.0/guide/expansion-reduction.json | 2 +-
.../zh-cn/2.0.0/guide/installation/cluster.json | 2 +-
.../zh-cn/2.0.0/guide/installation/kubernetes.json | 2 +-
.../2.0.0/guide/installation/pseudo-cluster.json | 2 +-
.../zh-cn/2.0.0/guide/installation/standalone.json | 2 +-
pages/doc/zh-cn/2.0.0/guide/parameter/context.json | 2 +-
pages/doc/zh-cn/2.0.0/guide/parameter/local.json | 2 +-
.../doc/zh-cn/2.0.0/guide/parameter/priority.json | 2 +-
.../2.0.0/guide/project/workflow-definition.json | 2 +-
.../2.0.0/guide/project/workflow-instance.json | 2 +-
pages/doc/zh-cn/2.0.0/guide/resource.json | 2 +-
pages/doc/zh-cn/2.0.0/guide/task/conditions.json | 2 +-
pages/doc/zh-cn/2.0.0/guide/task/switch.json | 2 +-
pages/doc/zh-cn/2.0.0/guide/upgrade.json | 2 +-
pages/doc/zh-cn/2.0.1/architecture/design.json | 2 +-
.../doc/zh-cn/2.0.1/guide/expansion-reduction.json | 2 +-
.../zh-cn/2.0.1/guide/installation/cluster.json | 2 +-
.../zh-cn/2.0.1/guide/installation/kubernetes.json | 2 +-
.../2.0.1/guide/installation/pseudo-cluster.json | 2 +-
.../zh-cn/2.0.1/guide/installation/standalone.json | 2 +-
pages/doc/zh-cn/2.0.1/guide/parameter/context.json | 2 +-
pages/doc/zh-cn/2.0.1/guide/parameter/local.json | 2 +-
.../doc/zh-cn/2.0.1/guide/parameter/priority.json | 2 +-
.../2.0.1/guide/project/workflow-definition.json | 2 +-
.../2.0.1/guide/project/workflow-instance.json | 2 +-
pages/doc/zh-cn/2.0.1/guide/resource.json | 2 +-
pages/doc/zh-cn/2.0.1/guide/task/conditions.json | 2 +-
pages/doc/zh-cn/2.0.1/guide/task/switch.json | 2 +-
pages/doc/zh-cn/2.0.1/guide/upgrade.json | 2 +-
pages/doc/zh-cn/2.0.2/architecture/design.json | 2 +-
.../doc/zh-cn/2.0.2/guide/expansion-reduction.json | 2 +-
.../zh-cn/2.0.2/guide/installation/cluster.json | 2 +-
.../zh-cn/2.0.2/guide/installation/kubernetes.json | 2 +-
.../2.0.2/guide/installation/pseudo-cluster.json | 2 +-
.../zh-cn/2.0.2/guide/installation/standalone.json | 2 +-
pages/doc/zh-cn/2.0.2/guide/parameter/context.json | 2 +-
pages/doc/zh-cn/2.0.2/guide/parameter/local.json | 2 +-
.../doc/zh-cn/2.0.2/guide/parameter/priority.json | 2 +-
.../2.0.2/guide/project/workflow-definition.json | 2 +-
.../2.0.2/guide/project/workflow-instance.json | 2 +-
pages/doc/zh-cn/2.0.2/guide/resource.json | 2 +-
pages/doc/zh-cn/2.0.2/guide/task/conditions.json | 2 +-
pages/doc/zh-cn/2.0.2/guide/task/switch.json | 2 +-
pages/doc/zh-cn/2.0.2/guide/upgrade.json | 2 +-
pages/doc/zh-cn/2.0.3/architecture/design.json | 2 +-
.../doc/zh-cn/2.0.3/guide/expansion-reduction.json | 2 +-
.../zh-cn/2.0.3/guide/installation/cluster.json | 2 +-
.../zh-cn/2.0.3/guide/installation/kubernetes.json | 2 +-
.../2.0.3/guide/installation/pseudo-cluster.json | 2 +-
.../zh-cn/2.0.3/guide/installation/standalone.json | 2 +-
pages/doc/zh-cn/2.0.3/guide/parameter/context.json | 2 +-
pages/doc/zh-cn/2.0.3/guide/parameter/local.json | 2 +-
.../doc/zh-cn/2.0.3/guide/parameter/priority.json | 2 +-
.../2.0.3/guide/project/workflow-definition.json | 2 +-
.../2.0.3/guide/project/workflow-instance.json | 2 +-
pages/doc/zh-cn/2.0.3/guide/resource.json | 2 +-
pages/doc/zh-cn/2.0.3/guide/task/conditions.json | 2 +-
pages/doc/zh-cn/2.0.3/guide/task/switch.json | 2 +-
pages/doc/zh-cn/2.0.3/guide/upgrade.json | 2 +-
pages/doc/zh-cn/2.0.5/architecture/design.json | 2 +-
.../doc/zh-cn/2.0.5/guide/expansion-reduction.json | 2 +-
.../zh-cn/2.0.5/guide/installation/cluster.json | 2 +-
.../zh-cn/2.0.5/guide/installation/kubernetes.json | 2 +-
.../2.0.5/guide/installation/pseudo-cluster.json | 2 +-
.../zh-cn/2.0.5/guide/installation/standalone.json | 2 +-
pages/doc/zh-cn/2.0.5/guide/parameter/context.json | 2 +-
pages/doc/zh-cn/2.0.5/guide/parameter/local.json | 2 +-
.../doc/zh-cn/2.0.5/guide/parameter/priority.json | 2 +-
.../2.0.5/guide/project/workflow-definition.json | 2 +-
.../2.0.5/guide/project/workflow-instance.json | 2 +-
pages/doc/zh-cn/2.0.5/guide/resource.json | 2 +-
pages/doc/zh-cn/2.0.5/guide/task/conditions.json | 2 +-
pages/doc/zh-cn/2.0.5/guide/task/switch.json | 2 +-
pages/doc/zh-cn/2.0.5/guide/upgrade.json | 2 +-
pages/doc/zh-cn/2.0.6/architecture/design.json | 2 +-
.../doc/zh-cn/2.0.6/guide/expansion-reduction.json | 2 +-
.../zh-cn/2.0.6/guide/installation/cluster.json | 2 +-
.../zh-cn/2.0.6/guide/installation/kubernetes.json | 2 +-
.../2.0.6/guide/installation/pseudo-cluster.json | 2 +-
.../zh-cn/2.0.6/guide/installation/standalone.json | 2 +-
pages/doc/zh-cn/2.0.6/guide/parameter/context.json | 2 +-
pages/doc/zh-cn/2.0.6/guide/parameter/local.json | 2 +-
.../doc/zh-cn/2.0.6/guide/parameter/priority.json | 2 +-
.../2.0.6/guide/project/workflow-definition.json | 2 +-
.../2.0.6/guide/project/workflow-instance.json | 2 +-
pages/doc/zh-cn/2.0.6/guide/resource.json | 2 +-
pages/doc/zh-cn/2.0.6/guide/task/conditions.json | 2 +-
pages/doc/zh-cn/2.0.6/guide/task/switch.json | 2 +-
pages/doc/zh-cn/2.0.6/guide/upgrade.json | 2 +-
pages/doc/zh-cn/2.0.7/architecture/design.json | 2 +-
.../doc/zh-cn/2.0.7/guide/expansion-reduction.json | 2 +-
.../zh-cn/2.0.7/guide/installation/cluster.json | 2 +-
.../zh-cn/2.0.7/guide/installation/kubernetes.json | 2 +-
.../2.0.7/guide/installation/pseudo-cluster.json | 2 +-
.../zh-cn/2.0.7/guide/installation/standalone.json | 2 +-
pages/doc/zh-cn/2.0.7/guide/parameter/context.json | 2 +-
pages/doc/zh-cn/2.0.7/guide/parameter/local.json | 2 +-
.../doc/zh-cn/2.0.7/guide/parameter/priority.json | 2 +-
.../2.0.7/guide/project/workflow-definition.json | 2 +-
.../2.0.7/guide/project/workflow-instance.json | 2 +-
pages/doc/zh-cn/2.0.7/guide/resource.json | 2 +-
pages/doc/zh-cn/2.0.7/guide/task/conditions.json | 2 +-
pages/doc/zh-cn/2.0.7/guide/task/switch.json | 2 +-
pages/doc/zh-cn/2.0.7/guide/upgrade.json | 2 +-
pages/doc/zh-cn/3.0.0/guide/datasource/mysql.json | 2 +-
.../doc/zh-cn/3.0.0/guide/expansion-reduction.json | 2 +-
.../zh-cn/3.0.0/guide/installation/cluster.json | 2 +-
.../zh-cn/3.0.0/guide/installation/kubernetes.json | 2 +-
.../3.0.0/guide/installation/pseudo-cluster.json | 2 +-
.../zh-cn/3.0.0/guide/installation/standalone.json | 2 +-
pages/doc/zh-cn/3.0.0/guide/parameter/context.json | 2 +-
pages/doc/zh-cn/3.0.0/guide/parameter/local.json | 2 +-
.../doc/zh-cn/3.0.0/guide/parameter/priority.json | 2 +-
.../zh-cn/3.0.0/guide/project/task-definition.json | 2 +-
.../3.0.0/guide/project/workflow-definition.json | 2 +-
.../3.0.0/guide/project/workflow-instance.json | 2 +-
.../zh-cn/3.0.0/guide/resource/configuration.json | 2 +-
pages/doc/zh-cn/3.0.0/guide/start/docker.json | 2 +-
pages/doc/zh-cn/3.0.0/guide/task/conditions.json | 2 +-
pages/doc/zh-cn/3.0.0/guide/task/flink.json | 2 +-
pages/doc/zh-cn/3.0.0/guide/task/map-reduce.json | 2 +-
pages/doc/zh-cn/3.0.0/guide/task/shell.json | 2 +-
pages/doc/zh-cn/3.0.0/guide/task/spark.json | 2 +-
pages/doc/zh-cn/3.0.0/guide/task/sql.json | 2 +-
pages/doc/zh-cn/3.0.0/guide/task/sub-process.json | 2 +-
pages/doc/zh-cn/3.0.0/guide/task/switch.json | 2 +-
pages/doc/zh-cn/3.0.0/guide/upgrade.json | 2 +-
pages/doc/zh-cn/3.0.1/guide/datasource/mysql.json | 2 +-
.../doc/zh-cn/3.0.1/guide/expansion-reduction.json | 2 +-
.../zh-cn/3.0.1/guide/installation/cluster.json | 2 +-
.../zh-cn/3.0.1/guide/installation/kubernetes.json | 2 +-
.../3.0.1/guide/installation/pseudo-cluster.json | 2 +-
.../zh-cn/3.0.1/guide/installation/standalone.json | 2 +-
pages/doc/zh-cn/3.0.1/guide/parameter/context.json | 2 +-
pages/doc/zh-cn/3.0.1/guide/parameter/local.json | 2 +-
.../doc/zh-cn/3.0.1/guide/parameter/priority.json | 2 +-
.../zh-cn/3.0.1/guide/project/task-definition.json | 2 +-
.../3.0.1/guide/project/workflow-definition.json | 2 +-
.../3.0.1/guide/project/workflow-instance.json | 2 +-
.../zh-cn/3.0.1/guide/resource/configuration.json | 2 +-
pages/doc/zh-cn/3.0.1/guide/start/docker.json | 2 +-
pages/doc/zh-cn/3.0.1/guide/task/conditions.json | 2 +-
pages/doc/zh-cn/3.0.1/guide/task/flink.json | 2 +-
pages/doc/zh-cn/3.0.1/guide/task/map-reduce.json | 2 +-
pages/doc/zh-cn/3.0.1/guide/task/shell.json | 2 +-
pages/doc/zh-cn/3.0.1/guide/task/spark.json | 2 +-
pages/doc/zh-cn/3.0.1/guide/task/sql.json | 2 +-
pages/doc/zh-cn/3.0.1/guide/task/sub-process.json | 2 +-
pages/doc/zh-cn/3.0.1/guide/task/switch.json | 2 +-
pages/doc/zh-cn/3.0.1/guide/upgrade.json | 2 +-
pages/doc/zh-cn/3.0.2/guide/datasource/mysql.json | 2 +-
.../doc/zh-cn/3.0.2/guide/expansion-reduction.json | 2 +-
.../zh-cn/3.0.2/guide/installation/cluster.json | 2 +-
.../zh-cn/3.0.2/guide/installation/kubernetes.json | 2 +-
.../3.0.2/guide/installation/pseudo-cluster.json | 2 +-
.../zh-cn/3.0.2/guide/installation/standalone.json | 2 +-
pages/doc/zh-cn/3.0.2/guide/parameter/context.json | 2 +-
pages/doc/zh-cn/3.0.2/guide/parameter/local.json | 2 +-
.../doc/zh-cn/3.0.2/guide/parameter/priority.json | 2 +-
.../zh-cn/3.0.2/guide/project/task-definition.json | 2 +-
.../3.0.2/guide/project/workflow-definition.json | 2 +-
.../3.0.2/guide/project/workflow-instance.json | 2 +-
.../zh-cn/3.0.2/guide/resource/configuration.json | 2 +-
pages/doc/zh-cn/3.0.2/guide/start/docker.json | 2 +-
pages/doc/zh-cn/3.0.2/guide/task/conditions.json | 2 +-
pages/doc/zh-cn/3.0.2/guide/task/flink.json | 2 +-
pages/doc/zh-cn/3.0.2/guide/task/map-reduce.json | 2 +-
pages/doc/zh-cn/3.0.2/guide/task/shell.json | 2 +-
pages/doc/zh-cn/3.0.2/guide/task/spark.json | 2 +-
pages/doc/zh-cn/3.0.2/guide/task/sql.json | 2 +-
pages/doc/zh-cn/3.0.2/guide/task/sub-process.json | 2 +-
pages/doc/zh-cn/3.0.2/guide/task/switch.json | 2 +-
pages/doc/zh-cn/3.0.2/guide/upgrade.json | 2 +-
pages/doc/zh-cn/3.0.3/guide/datasource/mysql.json | 2 +-
.../doc/zh-cn/3.0.3/guide/expansion-reduction.json | 2 +-
.../zh-cn/3.0.3/guide/installation/cluster.json | 2 +-
.../zh-cn/3.0.3/guide/installation/kubernetes.json | 2 +-
.../3.0.3/guide/installation/pseudo-cluster.json | 2 +-
.../zh-cn/3.0.3/guide/installation/standalone.json | 2 +-
pages/doc/zh-cn/3.0.3/guide/parameter/context.json | 2 +-
pages/doc/zh-cn/3.0.3/guide/parameter/local.json | 2 +-
.../doc/zh-cn/3.0.3/guide/parameter/priority.json | 2 +-
.../zh-cn/3.0.3/guide/project/task-definition.json | 2 +-
.../3.0.3/guide/project/workflow-definition.json | 2 +-
.../3.0.3/guide/project/workflow-instance.json | 2 +-
.../zh-cn/3.0.3/guide/resource/configuration.json | 2 +-
pages/doc/zh-cn/3.0.3/guide/start/docker.json | 2 +-
pages/doc/zh-cn/3.0.3/guide/task/conditions.json | 2 +-
pages/doc/zh-cn/3.0.3/guide/task/flink.json | 2 +-
pages/doc/zh-cn/3.0.3/guide/task/map-reduce.json | 2 +-
pages/doc/zh-cn/3.0.3/guide/task/shell.json | 2 +-
pages/doc/zh-cn/3.0.3/guide/task/spark.json | 2 +-
pages/doc/zh-cn/3.0.3/guide/task/sql.json | 2 +-
pages/doc/zh-cn/3.0.3/guide/task/sub-process.json | 2 +-
pages/doc/zh-cn/3.0.3/guide/task/switch.json | 2 +-
pages/doc/zh-cn/3.0.3/guide/upgrade.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/datasource/athena.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/datasource/mysql.json | 2 +-
.../doc/zh-cn/3.1.0/guide/expansion-reduction.json | 2 +-
.../zh-cn/3.1.0/guide/installation/cluster.json | 2 +-
.../zh-cn/3.1.0/guide/installation/kubernetes.json | 2 +-
.../3.1.0/guide/installation/pseudo-cluster.json | 2 +-
.../zh-cn/3.1.0/guide/installation/standalone.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/parameter/context.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/parameter/local.json | 2 +-
.../doc/zh-cn/3.1.0/guide/parameter/priority.json | 2 +-
.../zh-cn/3.1.0/guide/project/task-definition.json | 2 +-
.../3.1.0/guide/project/workflow-definition.json | 2 +-
.../3.1.0/guide/project/workflow-instance.json | 2 +-
.../zh-cn/3.1.0/guide/resource/configuration.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/start/docker.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/appendix.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/chunjun.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/conditions.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/datax.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/dependent.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/dinky.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/dvc.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/emr.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/flink.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/hive-cli.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/http.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/jupyter.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/kubernetes.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/map-reduce.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/mlflow.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/openmldb.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/pigeon.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/python.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/pytorch.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/sagemaker.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/seatunnel.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/shell.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/spark.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/sql.json | 2 +-
.../zh-cn/3.1.0/guide/task/stored-procedure.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/sub-process.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/switch.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/task/zeppelin.json | 2 +-
pages/doc/zh-cn/3.1.0/guide/upgrade/upgrade.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/datasource/athena.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/datasource/mysql.json | 2 +-
.../doc/zh-cn/3.1.1/guide/expansion-reduction.json | 2 +-
.../zh-cn/3.1.1/guide/installation/cluster.json | 2 +-
.../zh-cn/3.1.1/guide/installation/kubernetes.json | 2 +-
.../3.1.1/guide/installation/pseudo-cluster.json | 2 +-
.../zh-cn/3.1.1/guide/installation/standalone.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/parameter/context.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/parameter/local.json | 2 +-
.../doc/zh-cn/3.1.1/guide/parameter/priority.json | 2 +-
.../zh-cn/3.1.1/guide/project/task-definition.json | 2 +-
.../3.1.1/guide/project/workflow-definition.json | 2 +-
.../3.1.1/guide/project/workflow-instance.json | 2 +-
.../zh-cn/3.1.1/guide/resource/configuration.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/start/docker.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/appendix.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/chunjun.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/conditions.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/datax.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/dependent.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/dinky.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/dvc.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/emr.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/flink.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/hive-cli.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/http.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/jupyter.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/kubernetes.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/map-reduce.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/mlflow.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/openmldb.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/pigeon.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/python.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/pytorch.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/sagemaker.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/seatunnel.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/shell.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/spark.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/sql.json | 2 +-
.../zh-cn/3.1.1/guide/task/stored-procedure.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/sub-process.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/switch.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/task/zeppelin.json | 2 +-
pages/doc/zh-cn/3.1.1/guide/upgrade/upgrade.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/datasource/athena.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/datasource/mysql.json | 2 +-
.../doc/zh-cn/3.1.2/guide/expansion-reduction.json | 2 +-
.../zh-cn/3.1.2/guide/installation/cluster.json | 2 +-
.../zh-cn/3.1.2/guide/installation/kubernetes.json | 2 +-
.../3.1.2/guide/installation/pseudo-cluster.json | 2 +-
.../zh-cn/3.1.2/guide/installation/standalone.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/parameter/context.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/parameter/local.json | 2 +-
.../doc/zh-cn/3.1.2/guide/parameter/priority.json | 2 +-
.../zh-cn/3.1.2/guide/project/task-definition.json | 2 +-
.../3.1.2/guide/project/workflow-definition.json | 2 +-
.../3.1.2/guide/project/workflow-instance.json | 2 +-
.../zh-cn/3.1.2/guide/resource/configuration.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/start/docker.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/appendix.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/chunjun.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/conditions.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/datax.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/dependent.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/dinky.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/dvc.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/emr.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/flink.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/hive-cli.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/http.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/jupyter.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/kubernetes.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/map-reduce.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/mlflow.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/openmldb.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/pigeon.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/python.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/pytorch.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/sagemaker.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/seatunnel.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/shell.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/spark.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/sql.json | 2 +-
.../zh-cn/3.1.2/guide/task/stored-procedure.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/sub-process.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/switch.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/task/zeppelin.json | 2 +-
pages/doc/zh-cn/3.1.2/guide/upgrade/upgrade.json | 2 +-
python/2.0.5/.buildinfo | 2 +-
python/2.0.5/.doctrees/api.doctree | Bin 1222281 -> 1222281 bytes
python/2.0.5/.doctrees/concept.doctree | Bin 25193 -> 25193 bytes
python/2.0.5/.doctrees/environment.pickle | Bin 609570 -> 609570 bytes
python/2.0.5/.doctrees/index.doctree | Bin 8680 -> 8680 bytes
python/2.0.5/.doctrees/start.doctree | Bin 22392 -> 22392 bytes
python/2.0.5/.doctrees/tasks/condition.doctree | Bin 70158 -> 70158 bytes
python/2.0.5/.doctrees/tasks/datax.doctree | Bin 60143 -> 60143 bytes
python/2.0.5/.doctrees/tasks/dependent.doctree | Bin 112530 -> 112530 bytes
python/2.0.5/.doctrees/tasks/flink.doctree | Bin 51328 -> 51328 bytes
python/2.0.5/.doctrees/tasks/http.doctree | Bin 45738 -> 45738 bytes
python/2.0.5/.doctrees/tasks/index.doctree | Bin 4942 -> 4942 bytes
python/2.0.5/.doctrees/tasks/map_reduce.doctree | Bin 27975 -> 27975 bytes
python/2.0.5/.doctrees/tasks/procedure.doctree | Bin 21686 -> 21686 bytes
python/2.0.5/.doctrees/tasks/python.doctree | Bin 19790 -> 19790 bytes
python/2.0.5/.doctrees/tasks/shell.doctree | Bin 23080 -> 23080 bytes
python/2.0.5/.doctrees/tasks/spark.doctree | Bin 52816 -> 52816 bytes
python/2.0.5/.doctrees/tasks/sql.doctree | Bin 33452 -> 33452 bytes
python/2.0.5/.doctrees/tasks/sub_process.doctree | Bin 23710 -> 23710 bytes
python/2.0.5/.doctrees/tasks/switch.doctree | Bin 63412 -> 63412 bytes
python/2.0.5/.doctrees/tutorial.doctree | Bin 36479 -> 36479 bytes
python/2.0.6/.buildinfo | 2 +-
python/2.0.6/.doctrees/api.doctree | Bin 1222281 -> 1222281 bytes
python/2.0.6/.doctrees/concept.doctree | Bin 25193 -> 25193 bytes
python/2.0.6/.doctrees/environment.pickle | Bin 609570 -> 609570 bytes
python/2.0.6/.doctrees/index.doctree | Bin 8680 -> 8680 bytes
python/2.0.6/.doctrees/start.doctree | Bin 22392 -> 22392 bytes
python/2.0.6/.doctrees/tasks/condition.doctree | Bin 70158 -> 70158 bytes
python/2.0.6/.doctrees/tasks/datax.doctree | Bin 60143 -> 60143 bytes
python/2.0.6/.doctrees/tasks/dependent.doctree | Bin 112530 -> 112530 bytes
python/2.0.6/.doctrees/tasks/flink.doctree | Bin 51328 -> 51328 bytes
python/2.0.6/.doctrees/tasks/http.doctree | Bin 45738 -> 45738 bytes
python/2.0.6/.doctrees/tasks/index.doctree | Bin 4942 -> 4942 bytes
python/2.0.6/.doctrees/tasks/map_reduce.doctree | Bin 27975 -> 27975 bytes
python/2.0.6/.doctrees/tasks/procedure.doctree | Bin 21686 -> 21686 bytes
python/2.0.6/.doctrees/tasks/python.doctree | Bin 19790 -> 19790 bytes
python/2.0.6/.doctrees/tasks/shell.doctree | Bin 23080 -> 23080 bytes
python/2.0.6/.doctrees/tasks/spark.doctree | Bin 52816 -> 52816 bytes
python/2.0.6/.doctrees/tasks/sql.doctree | Bin 33452 -> 33452 bytes
python/2.0.6/.doctrees/tasks/sub_process.doctree | Bin 23710 -> 23710 bytes
python/2.0.6/.doctrees/tasks/switch.doctree | Bin 63412 -> 63412 bytes
python/2.0.6/.doctrees/tutorial.doctree | Bin 36479 -> 36479 bytes
python/2.0.7/.buildinfo | 2 +-
python/2.0.7/.doctrees/api.doctree | Bin 1222281 -> 1222281 bytes
python/2.0.7/.doctrees/concept.doctree | Bin 25193 -> 25193 bytes
python/2.0.7/.doctrees/environment.pickle | Bin 609570 -> 609570 bytes
python/2.0.7/.doctrees/index.doctree | Bin 8680 -> 8680 bytes
python/2.0.7/.doctrees/start.doctree | Bin 22392 -> 22392 bytes
python/2.0.7/.doctrees/tasks/condition.doctree | Bin 70158 -> 70158 bytes
python/2.0.7/.doctrees/tasks/datax.doctree | Bin 60143 -> 60143 bytes
python/2.0.7/.doctrees/tasks/dependent.doctree | Bin 112530 -> 112530 bytes
python/2.0.7/.doctrees/tasks/flink.doctree | Bin 51328 -> 51328 bytes
python/2.0.7/.doctrees/tasks/http.doctree | Bin 45738 -> 45738 bytes
python/2.0.7/.doctrees/tasks/index.doctree | Bin 4942 -> 4942 bytes
python/2.0.7/.doctrees/tasks/map_reduce.doctree | Bin 27975 -> 27975 bytes
python/2.0.7/.doctrees/tasks/procedure.doctree | Bin 21686 -> 21686 bytes
python/2.0.7/.doctrees/tasks/python.doctree | Bin 19790 -> 19790 bytes
python/2.0.7/.doctrees/tasks/shell.doctree | Bin 23080 -> 23080 bytes
python/2.0.7/.doctrees/tasks/spark.doctree | Bin 52816 -> 52816 bytes
python/2.0.7/.doctrees/tasks/sql.doctree | Bin 33452 -> 33452 bytes
python/2.0.7/.doctrees/tasks/sub_process.doctree | Bin 23710 -> 23710 bytes
python/2.0.7/.doctrees/tasks/switch.doctree | Bin 63412 -> 63412 bytes
python/2.0.7/.doctrees/tutorial.doctree | Bin 36479 -> 36479 bytes
python/3.0.0/.buildinfo | 2 +-
python/3.0.0/.doctrees/api.doctree | Bin 1333878 -> 1333878 bytes
python/3.0.0/.doctrees/cli.doctree | Bin 16238 -> 16238 bytes
python/3.0.0/.doctrees/concept.doctree | Bin 25193 -> 25193 bytes
python/3.0.0/.doctrees/config.doctree | Bin 52206 -> 52206 bytes
python/3.0.0/.doctrees/environment.pickle | Bin 666254 -> 666252 bytes
python/3.0.0/.doctrees/howto/index.doctree | Bin 6156 -> 6156 bytes
python/3.0.0/.doctrees/howto/remote-submit.doctree | Bin 9629 -> 9629 bytes
python/3.0.0/.doctrees/index.doctree | Bin 8733 -> 8733 bytes
python/3.0.0/.doctrees/start.doctree | Bin 34062 -> 34062 bytes
python/3.0.0/.doctrees/tasks/condition.doctree | Bin 70158 -> 70158 bytes
python/3.0.0/.doctrees/tasks/datax.doctree | Bin 60143 -> 60143 bytes
python/3.0.0/.doctrees/tasks/dependent.doctree | Bin 112490 -> 112490 bytes
python/3.0.0/.doctrees/tasks/flink.doctree | Bin 51328 -> 51328 bytes
python/3.0.0/.doctrees/tasks/func_wrap.doctree | Bin 15112 -> 15112 bytes
python/3.0.0/.doctrees/tasks/http.doctree | Bin 45738 -> 45738 bytes
python/3.0.0/.doctrees/tasks/index.doctree | Bin 4965 -> 4965 bytes
python/3.0.0/.doctrees/tasks/map_reduce.doctree | Bin 27975 -> 27975 bytes
python/3.0.0/.doctrees/tasks/procedure.doctree | Bin 21686 -> 21686 bytes
python/3.0.0/.doctrees/tasks/python.doctree | Bin 26862 -> 26862 bytes
python/3.0.0/.doctrees/tasks/shell.doctree | Bin 23080 -> 23080 bytes
python/3.0.0/.doctrees/tasks/spark.doctree | Bin 52816 -> 52816 bytes
python/3.0.0/.doctrees/tasks/sql.doctree | Bin 36324 -> 36324 bytes
python/3.0.0/.doctrees/tasks/sub_process.doctree | Bin 23710 -> 23710 bytes
python/3.0.0/.doctrees/tasks/switch.doctree | Bin 63983 -> 63983 bytes
python/3.0.0/.doctrees/tutorial.doctree | Bin 57115 -> 57115 bytes
python/3.0.1/.buildinfo | 2 +-
python/3.0.1/.doctrees/api.doctree | Bin 1374320 -> 1374320 bytes
python/3.0.1/.doctrees/cli.doctree | Bin 16238 -> 16238 bytes
python/3.0.1/.doctrees/concept.doctree | Bin 25193 -> 25193 bytes
python/3.0.1/.doctrees/config.doctree | Bin 52205 -> 52205 bytes
python/3.0.1/.doctrees/environment.pickle | Bin 680982 -> 680984 bytes
python/3.0.1/.doctrees/howto/index.doctree | Bin 6156 -> 6156 bytes
python/3.0.1/.doctrees/howto/remote-submit.doctree | Bin 9629 -> 9629 bytes
python/3.0.1/.doctrees/index.doctree | Bin 8733 -> 8733 bytes
python/3.0.1/.doctrees/start.doctree | Bin 34062 -> 34062 bytes
python/3.0.1/.doctrees/tasks/condition.doctree | Bin 70170 -> 70170 bytes
python/3.0.1/.doctrees/tasks/datax.doctree | Bin 60143 -> 60143 bytes
python/3.0.1/.doctrees/tasks/dependent.doctree | Bin 112500 -> 112500 bytes
python/3.0.1/.doctrees/tasks/flink.doctree | Bin 51328 -> 51328 bytes
python/3.0.1/.doctrees/tasks/func_wrap.doctree | Bin 15112 -> 15112 bytes
python/3.0.1/.doctrees/tasks/http.doctree | Bin 45738 -> 45738 bytes
python/3.0.1/.doctrees/tasks/index.doctree | Bin 4965 -> 4965 bytes
python/3.0.1/.doctrees/tasks/map_reduce.doctree | Bin 27975 -> 27975 bytes
python/3.0.1/.doctrees/tasks/procedure.doctree | Bin 21686 -> 21686 bytes
python/3.0.1/.doctrees/tasks/python.doctree | Bin 26862 -> 26862 bytes
python/3.0.1/.doctrees/tasks/shell.doctree | Bin 23080 -> 23080 bytes
python/3.0.1/.doctrees/tasks/spark.doctree | Bin 52816 -> 52816 bytes
python/3.0.1/.doctrees/tasks/sql.doctree | Bin 36324 -> 36324 bytes
python/3.0.1/.doctrees/tasks/sub_process.doctree | Bin 23710 -> 23710 bytes
python/3.0.1/.doctrees/tasks/switch.doctree | Bin 63995 -> 63995 bytes
python/3.0.1/.doctrees/tutorial.doctree | Bin 57115 -> 57115 bytes
python/3.1.0/.buildinfo | 2 +-
python/3.1.0/.doctrees/api.doctree | Bin 2339705 -> 2339705 bytes
python/3.1.0/.doctrees/cli.doctree | Bin 18672 -> 18672 bytes
python/3.1.0/.doctrees/concept.doctree | Bin 25193 -> 25193 bytes
python/3.1.0/.doctrees/config.doctree | Bin 52205 -> 52205 bytes
python/3.1.0/.doctrees/environment.pickle | Bin 1069415 -> 1069415 bytes
python/3.1.0/.doctrees/howto/index.doctree | Bin 6156 -> 6156 bytes
python/3.1.0/.doctrees/howto/remote-submit.doctree | Bin 9629 -> 9629 bytes
python/3.1.0/.doctrees/index.doctree | Bin 8733 -> 8733 bytes
python/3.1.0/.doctrees/start.doctree | Bin 34062 -> 34062 bytes
python/3.1.0/.doctrees/tasks/condition.doctree | Bin 72354 -> 72354 bytes
python/3.1.0/.doctrees/tasks/datax.doctree | Bin 64297 -> 64297 bytes
python/3.1.0/.doctrees/tasks/dependent.doctree | Bin 118147 -> 118147 bytes
python/3.1.0/.doctrees/tasks/dvc.doctree | Bin 80870 -> 80870 bytes
python/3.1.0/.doctrees/tasks/flink.doctree | Bin 52400 -> 52400 bytes
python/3.1.0/.doctrees/tasks/func_wrap.doctree | Bin 15112 -> 15112 bytes
python/3.1.0/.doctrees/tasks/http.doctree | Bin 47385 -> 47385 bytes
python/3.1.0/.doctrees/tasks/index.doctree | Bin 5068 -> 5068 bytes
python/3.1.0/.doctrees/tasks/map_reduce.doctree | Bin 29071 -> 29071 bytes
python/3.1.0/.doctrees/tasks/mlflow.doctree | Bin 158045 -> 158045 bytes
python/3.1.0/.doctrees/tasks/openmldb.doctree | Bin 24290 -> 24290 bytes
python/3.1.0/.doctrees/tasks/procedure.doctree | Bin 22626 -> 22626 bytes
python/3.1.0/.doctrees/tasks/python.doctree | Bin 27825 -> 27825 bytes
python/3.1.0/.doctrees/tasks/pytorch.doctree | Bin 50519 -> 50519 bytes
python/3.1.0/.doctrees/tasks/sagemaker.doctree | Bin 25665 -> 25665 bytes
python/3.1.0/.doctrees/tasks/shell.doctree | Bin 24504 -> 24504 bytes
python/3.1.0/.doctrees/tasks/spark.doctree | Bin 53898 -> 53898 bytes
python/3.1.0/.doctrees/tasks/sql.doctree | Bin 39183 -> 39183 bytes
python/3.1.0/.doctrees/tasks/sub_process.doctree | Bin 26023 -> 26023 bytes
python/3.1.0/.doctrees/tasks/switch.doctree | Bin 67353 -> 67353 bytes
python/3.1.0/.doctrees/tutorial.doctree | Bin 74908 -> 74908 bytes
python/main/.buildinfo | 2 +-
python/main/.doctrees/api.doctree | Bin 2843886 -> 2845088 bytes
python/main/.doctrees/cli.doctree | Bin 18652 -> 18652 bytes
python/main/.doctrees/concept.doctree | Bin 45148 -> 45148 bytes
python/main/.doctrees/config.doctree | Bin 54889 -> 54889 bytes
python/main/.doctrees/environment.pickle | Bin 1331764 -> 1332136 bytes
python/main/.doctrees/howto/index.doctree | Bin 6187 -> 6187 bytes
.../main/.doctrees/howto/multi-resources.doctree | Bin 16119 -> 16119 bytes
python/main/.doctrees/howto/remote-submit.doctree | Bin 9629 -> 9629 bytes
python/main/.doctrees/index.doctree | Bin 30077 -> 30077 bytes
.../.doctrees/resources_plugin/develop.doctree | Bin 11497 -> 11497 bytes
.../main/.doctrees/resources_plugin/github.doctree | Bin 35045 -> 35045 bytes
.../main/.doctrees/resources_plugin/gitlab.doctree | Bin 36508 -> 36508 bytes
.../main/.doctrees/resources_plugin/index.doctree | Bin 4958 -> 4958 bytes
.../main/.doctrees/resources_plugin/local.doctree | Bin 16936 -> 16936 bytes
python/main/.doctrees/resources_plugin/oss.doctree | Bin 12128 -> 12128 bytes
.../resources_plugin/resource-plugin.doctree | Bin 30661 -> 30661 bytes
python/main/.doctrees/resources_plugin/s3.doctree | Bin 10744 -> 10744 bytes
python/main/.doctrees/start.doctree | Bin 35927 -> 35927 bytes
python/main/.doctrees/tasks/condition.doctree | Bin 73803 -> 73803 bytes
python/main/.doctrees/tasks/datax.doctree | Bin 74622 -> 74622 bytes
python/main/.doctrees/tasks/dependent.doctree | Bin 120915 -> 120915 bytes
python/main/.doctrees/tasks/dvc.doctree | Bin 86626 -> 86626 bytes
python/main/.doctrees/tasks/flink.doctree | Bin 53805 -> 53805 bytes
python/main/.doctrees/tasks/func_wrap.doctree | Bin 15062 -> 15062 bytes
python/main/.doctrees/tasks/http.doctree | Bin 48778 -> 48778 bytes
python/main/.doctrees/tasks/index.doctree | Bin 5093 -> 5093 bytes
python/main/.doctrees/tasks/kubernetes.doctree | Bin 28391 -> 28391 bytes
python/main/.doctrees/tasks/map_reduce.doctree | Bin 30483 -> 30483 bytes
python/main/.doctrees/tasks/mlflow.doctree | Bin 165605 -> 165605 bytes
python/main/.doctrees/tasks/openmldb.doctree | Bin 25728 -> 25728 bytes
python/main/.doctrees/tasks/procedure.doctree | Bin 24074 -> 24074 bytes
python/main/.doctrees/tasks/python.doctree | Bin 33781 -> 33781 bytes
python/main/.doctrees/tasks/pytorch.doctree | Bin 51946 -> 51946 bytes
python/main/.doctrees/tasks/sagemaker.doctree | Bin 27116 -> 27116 bytes
python/main/.doctrees/tasks/shell.doctree | Bin 29647 -> 29647 bytes
python/main/.doctrees/tasks/spark.doctree | Bin 47747 -> 47747 bytes
python/main/.doctrees/tasks/sql.doctree | Bin 44089 -> 44089 bytes
python/main/.doctrees/tasks/sub_workflow.doctree | Bin 27623 -> 27623 bytes
python/main/.doctrees/tasks/switch.doctree | Bin 68739 -> 68739 bytes
python/main/.doctrees/tutorial.doctree | Bin 73813 -> 73813 bytes
.../_modules/pydolphinscheduler/core/task.html | 11 +-
.../_modules/pydolphinscheduler/core/workflow.html | 2 +
python/main/api.html | 4 +-
static/css/main.322acf4b.css | 2 -
static/css/main.322acf4b.css.map | 1 -
static/css/main.5a15068e.css | 2 +
static/css/main.5a15068e.css.map | 1 +
877 files changed, 708 insertions(+), 13528 deletions(-)
diff --git a/asset-manifest.json b/asset-manifest.json
index d8ced66efe..2d27e12e25 100644
--- a/asset-manifest.json
+++ b/asset-manifest.json
@@ -1,6 +1,6 @@
{
"files": {
- "main.css": "./static/css/main.322acf4b.css",
+ "main.css": "./static/css/main.5a15068e.css",
"main.js": "./static/js/main.1d078a4b.js",
"static/media/use-case.png": "./static/media/use-case.02f86278dc7cff189b02.png",
"static/media/blog.png": "./static/media/blog.a3cd9850dee16b70c593.png",
@@ -13,11 +13,11 @@
"static/media/logo_white.webp": "./static/media/logo_white.bedc7884531414ef48a2.webp",
"static/media/logo_black.webp": "./static/media/logo_black.c9bf9b9c244f60328e1d.webp",
"index.html": "./index.html",
- "main.322acf4b.css.map": "./static/css/main.322acf4b.css.map",
+ "main.5a15068e.css.map": "./static/css/main.5a15068e.css.map",
"main.1d078a4b.js.map": "./static/js/main.1d078a4b.js.map"
},
"entrypoints": [
- "static/css/main.322acf4b.css",
+ "static/css/main.5a15068e.css",
"static/js/main.1d078a4b.js"
]
}
\ No newline at end of file
diff --git a/data/doc/en-us.json b/data/doc/en-us.json
deleted file mode 100644
index 275838fae3..0000000000
--- a/data/doc/en-us.json
+++ /dev/null
@@ -1,12831 +0,0 @@
-[
- {
- "content": "Hareware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux operating system version requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "1.2.0",
- "time": "2020-1-2",
- "link": "/hardware-environment",
- "title": "Hareware Environment",
- "location": [
- "Deployment Document",
- "Hardware Environment"
- ]
- },
- {
- "content": "Standalone Deployment\nDolphinScheduler Standalone deployment is divided into two parts: backend deployment and frontend deployment.\n1. Backend Deployment\n1.1: Before you begin (please install requirement basic software by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.6 or 5.7): Choose One\nJDK (1.8+): Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+): Required\nHadoop (2.6+) or MinIO: Optional. If you need to [...]
- "version": "1.2.0",
- "time": "2020-1-2",
- "link": "/standalone-deployment",
- "title": "Standalone Deployment",
- "location": [
- "Deployment Document",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\nDolphinScheduler Cluster deployment is divided into two parts: backend deployment and frontend deployment.\n1、Backend Deployment\n1.1: Before you begin (please install requirement basic software by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS [...]
- "version": "1.2.0",
- "time": "2020-1-2",
- "link": "/cluster-deployment",
- "title": "Cluster Deployment",
- "location": [
- "Deployment Document",
- "Cluster Deployment"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：192.168.xx.xx:8888 Username and password：admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate an worker group\n\n \n \n \n\n\nCreate an token\n\n \n \n\n\nLog in with regular users\n\n\n\nClick on the user name in the upper right corner to "exit" and re-use the [...]
- "version": "1.2.0",
- "time": "2020-1-2",
- "link": "/quick-start",
- "title": "Quick Start",
- "location": [
- "User Manual",
- "Quick Start"
- ]
- },
- {
- "content": "System Use Manual\nOperational Guidelines\nHome page\nThe homepage contains task status statistics, process status statistics, and workflow definition statistics for all user projects.\n\n \n \nCreate a project\n\nClick "Project - > Create Project", enter project name, description, and click "Submit" to create a new project.\nClick on the project name to enter the project home page.\n\n\n \n \n\nThe project home page contains task status [...]
- "version": "1.2.0",
- "time": "2020-1-2",
- "link": "/system-manual",
- "title": "System Use Manual",
- "location": [
- "User Manual",
- "User Manual"
- ]
- },
- {
- "content": "Dolphin Scheduler 1.2 MetaData\n\nDolphin Scheduler 1.2 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nque [...]
- "version": "1.2.0",
- "time": "2020-1-2",
- "link": "/metadata-1.2",
- "title": "Dolphin Scheduler 1.2 MetaData",
- "location": [
- "User Manual",
- "Metadata"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back up the previous version of the files and database\n2. Stop all services of dolphinscheduler\nsh ./script/stop-all.sh\n3. Download the new version of the installation package\n\ndownload, download the latest version of the front and back installation packages (backend referred to as dolphinscheduler-backend, front end referred to as dolphinscheduler-front)\nThe following upgrade operations need to be performed in the new vers [...]
- "version": "1.2.0",
- "time": "2020-1-2",
- "link": "/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Upgrade",
- "Upgrade"
- ]
- },
- {
- "content": "Change Log\nNew Features\n\nSupport postgre sql\nChange all Chinese names to English\nAdd flink and http task support\nCross project dependencies\nModify mybatis to mybatisplus, support multy databases.\nAdd export and import definition feaure\nGithub actions ci compile check\nAdd method and parameters comments\nAdd java doc for common module\n\nEnhancements\n\nAdd license and notice files\nMove batchDelete Process Define/Instance Outside for transactional\nRemove space b [...]
- "version": "1.2.0",
- "time": "2020-1-2",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "Hareware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux operating system version requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "1.2.1",
- "time": "2020-2-24",
- "link": "/hardware-environment",
- "title": "Hareware Environment",
- "location": [
- "Deployment Document",
- "Hardware Environment"
- ]
- },
- {
- "content": "Backend Deployment Document\nThere are two deployment modes for the backend:\n\nautomatic deployment\nsource code compile and then deployment\n\nPreparations\nDownload the latest version of the installation package, download address： download,\ndownload apache-dolphinscheduler-incubating-x.x.x-dolphinscheduler-backend-bin.tar.gz\nPreparations 1: Installation of basic software (self-installation of required items)\n\nPostgreSQL (8.2.15+) or MySQL (5.5+) : You can choose e [...]
- "version": "1.2.1",
- "time": "2020-2-24",
- "link": "/backend-deployment",
- "title": "Backend Deployment Document",
- "location": [
- "Deployment Document",
- "Backend deploy"
- ]
- },
- {
- "content": "frontend-deployment\nThe front-end has three deployment modes: automated deployment, manual deployment and compiled source deployment.\nPreparations\nDownload the installation package\nPlease download the latest version of the installation package, download address： download\nAfter downloading apache-dolphinscheduler-incubating-x.x.x-dolphinscheduler-front-bin.tar.gz，\ndecompresstar -zxvf apache-dolphinscheduler-incubating-x.x.x-dolphinscheduler-front-bin.tar.gz ./and ent [...]
- "version": "1.2.1",
- "time": "2020-2-24",
- "link": "/frontend-deployment",
- "title": "frontend-deployment",
- "location": [
- "Deployment Document",
- "Frontend deploy"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：192.168.xx.xx:8888 Username and password：admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate an worker group\n\n \n \n \n\n\nCreate an token\n\n \n \n\n\nLog in with regular users\n\n\n\nClick on the user name in the upper right corner to "exit" and re-use the [...]
- "version": "1.2.1",
- "time": "2020-2-24",
- "link": "/quick-start",
- "title": "Quick Start",
- "location": [
- "User Manual",
- "Quick Start"
- ]
- },
- {
- "content": "System Use Manual\nOperational Guidelines\nHome page\nThe homepage contains task status statistics, process status statistics, and workflow definition statistics for all user projects.\n\n \n \nCreate a project\n\nClick "Project - > Create Project", enter project name, description, and click "Submit" to create a new project.\nClick on the project name to enter the project home page.\n\n\n \n \n\nThe project home page contains task status [...]
- "version": "1.2.1",
- "time": "2020-2-24",
- "link": "/system-manual",
- "title": "System Use Manual",
- "location": [
- "User Manual",
- "User Manual"
- ]
- },
- {
- "content": "Dolphin Scheduler 1.2 MetaData\n\nDolphin Scheduler 1.2 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nque [...]
- "version": "1.2.1",
- "time": "2020-2-24",
- "link": "/metadata-1.2",
- "title": "Dolphin Scheduler 1.2 MetaData",
- "location": [
- "User Manual",
- "Metadata"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back up the previous version of the files and database\n2. Stop all services of dolphinscheduler\nsh ./script/stop-all.sh\n3. Download the new version of the installation package\n\ndownload, download the latest version of the front and back installation packages (backend referred to as dolphinscheduler-backend, front end referred to as dolphinscheduler-front)\nThe following upgrade operations need to be performed in the new vers [...]
- "version": "1.2.1",
- "time": "2020-2-24",
- "link": "/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Upgrade",
- "Upgrade"
- ]
- },
- {
- "content": "Change Log\nNew Features\n\n[#1497] DAG automatic layout when use api call process definition create\n[ #747] Worker server runtime log desensitization\n[#1635] merge configurations in order to reduce configuration files\n[#1460] Add CI\n[#1547] node text edit box supports full-screen magnification\n[#1646] Add github action to analyze code with sonarcloud\n[#1367] Add java checkstyle\n\nEnhancements\n\n[#184] Use the default workgroup in the page ,Instead of the value id [...]
- "version": "1.2.1",
- "time": "2020-2-24",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "Hareware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux operating system version requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "1.3.1",
- "time": "2020-7-13",
- "link": "/hardware-environment",
- "title": "Hareware Environment",
- "location": [
- "Deployment Document",
- "Hardware Environment"
- ]
- },
- {
- "content": "Standalone Deployment\n1、Before you begin (please install requirement basic software by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/CentOS/Ubuntu/Debian\nHadoop (2.6+) or MinIO : Optional. If you need to upl [...]
- "version": "1.3.1",
- "time": "2020-7-13",
- "link": "/standalone-deployment",
- "title": "Standalone Deployment",
- "location": [
- "Deployment Document",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\n1、Before you begin (please install requirement basic software by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/CentOS/Ubuntu/Debian\nHadoop (2.6+) or MinIO : Optional. If you need to upload [...]
- "version": "1.3.1",
- "time": "2020-7-13",
- "link": "/cluster-deployment",
- "title": "Cluster Deployment",
- "location": [
- "Deployment Document",
- "Cluster deploy"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password：admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate an worker group\n\n \n \n \n\n\nCreate an token\n\n \n \n\n\nLog in with regular users\n\n\n\nClick on the user name in the upper right corner to "exit& [...]
- "version": "1.3.1",
- "time": "2020-7-13",
- "link": "/quick-start",
- "title": "Quick Start",
- "location": [
- "User Manual",
- "Quick Start"
- ]
- },
- {
- "content": "System User Manual\nGet started quickly\n\nPlease refer toGet started quickly\n\nOperation guide\n1. Home\nThe home page contains task status statistics, process status statistics, and workflow definition statistics of all items of the user.\n\n\n\n2. Project management\n2.1 Create project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click " [...]
- "version": "1.3.1",
- "time": "2020-7-13",
- "link": "/system-manual",
- "title": "System User Manual",
- "location": [
- "User Manual",
- "User Manual"
- ]
- },
- {
- "content": "Dolphin Scheduler 1.3 Metadata document\n\nTable overview\n\n\n\nTable Name\nTable information\n\n\n\n\nt_ds_access_token\nAccess the token of the ds backend\n\n\nt_ds_alert\nWarning message\n\n\nt_ds_alertgroup\nAlarm group\n\n\nt_ds_command\nExcuting an order\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nWrong command\n\n\nt_ds_process_definition\nProcess definition\n\n\nt_ds_process_instance\nProcess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nque [...]
- "version": "1.3.1",
- "time": "2020-7-13",
- "link": "/metadata-1.3",
- "title": "Dolphin Scheduler 1.3 Metadata document",
- "location": [
- "Architecture Design",
- "Metadata"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the scheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the form of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until there are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "1.3.1",
- "time": "2020-7-13",
- "link": "/architecture-design",
- "title": "System Architecture Design",
- "location": [
- "Architecture Design",
- "Architecture Design"
- ]
- },
- {
- "content": "\nForeword\nThis document is a description of the dolphinscheduler configuration file, and the version is for dolphinscheduler-1.3.x.\nDirectory Structure\nAll configuration files of dolphinscheduler are currently in the [conf] directory.\nFor a more intuitive understanding of the location of the [conf] directory and the configuration files it contains, please see the simplified description of the dolphinscheduler installation directory below.\nThis article mainly talks a [...]
- "version": "1.3.1",
- "time": "2020-7-13",
- "link": "/configuration-file",
- "title": "Foreword",
- "location": [
- "Architecture Design",
- "Configuration File"
- ]
- },
- {
- "content": "Overall task storage structure\nAll tasks created in dolphinscheduler are saved in the t_ds_process_definition table.\nThe database table structure is shown in the following table:\n\n\n\nSerial number\nField\nTypes\nDescription\n\n\n\n\n1\nid\nint(11)\nPrimary key\n\n\n2\nname\nvarchar(255)\nProcess definition name\n\n\n3\nversion\nint(11)\nProcess definition version\n\n\n4\nrelease_state\ntinyint(4)\nRelease status of the process definition：0 not online, 1 online\n\n\n5 [...]
- "version": "1.3.1",
- "time": "2020-7-13",
- "link": "/task-structure",
- "title": "Overall task storage structure",
- "location": [
- "Architecture Design",
- "Task Structure"
- ]
- },
- {
- "content": "DolphinScheduler Upgrade document\n1. Back up files and databases of the previous version\n2. Stop all services of dolphinscheduler\nsh ./script/stop-all.sh\n3. Download the new version of the installation package\n\ndownload, Download the latest version of the binary installation package\nThe following upgrade operations need to be performed in the new version directory\n\n4. Database upgrade\n\n\nModify the following properties in conf/datasource.properties\n\n\nIf you [...]
- "version": "1.3.1",
- "time": "2020-7-13",
- "link": "/upgrade",
- "title": "DolphinScheduler Upgrade document",
- "location": [
- "Upgrade",
- "Upgrade"
- ]
- },
- {
- "content": "Change Log\nNew Features\n[#1658] Refactor WorkerServer: Using Netty as tcp framework between master and worker, remove zk as task queue, add three choices(Random, RoundRobin, and linear weighted based on cpu and memory) when master choose worker, worker don‘t operate database\n[#2584] Support k8s deployment\n[#404] Resource center supports directory management\n[#1469] Support ambari plugin installation\n[#205] Support condition detection type task\n[#203] Data synchroni [...]
- "version": "1.3.1",
- "time": "2020-7-13",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "Hareware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux operating system version requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "1.3.2",
- "time": "2020-9-1",
- "link": "/hardware-environment",
- "title": "Hareware Environment",
- "location": [
- "Deployment Document",
- "Hardware Environment"
- ]
- },
- {
- "content": "Standalone Deployment\n1、Install basic softwares (please install required softwares by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One,If use MySQL,It is strongly recommended that MySQL version is 5.7 or higher\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/ [...]
- "version": "1.3.2",
- "time": "2020-9-1",
- "link": "/standalone-deployment",
- "title": "Standalone Deployment",
- "location": [
- "Deployment Document",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\n1、Before you begin (please install requirement basic software by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One,If use MySQL,It is strongly recommended that MySQL version is 5.7 or higher\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\nHadoop (2.6+) or MinIO : Optional. If you need to upload a resource function, you can choose a local file directory a [...]
- "version": "1.3.2",
- "time": "2020-9-1",
- "link": "/cluster-deployment",
- "title": "Cluster Deployment",
- "location": [
- "Deployment Document",
- "Cluster deploy"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password：admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate an worker group\n\n \n \n \n\n\nCreate an token\n\n \n \n\n\nLog in with regular users\n\n\n\nClick on the user name in the upper right corner to "exit& [...]
- "version": "1.3.2",
- "time": "2020-9-1",
- "link": "/quick-start",
- "title": "Quick Start",
- "location": [
- "User Manual",
- "Quick Start"
- ]
- },
- {
- "content": "System User Manual\nGet started quickly\n\nPlease refer to Quick Start\n\nOperation guide\n1. Home\nThe home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.\n\n\n\n2. Project management\n2.1 Create project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Su [...]
- "version": "1.3.2",
- "time": "2020-9-1",
- "link": "/system-manual",
- "title": "System User Manual",
- "location": [
- "User Manual",
- "User Manual"
- ]
- },
- {
- "content": "Dolphin Scheduler 1.3 MetaData\n\nDolphin Scheduler 1.2 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nque [...]
- "version": "1.3.2",
- "time": "2020-9-1",
- "link": "/metadata-1.3",
- "title": "Dolphin Scheduler 1.3 MetaData",
- "location": [
- "Architecture Design",
- "Metadata"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the scheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the form of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until there are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "1.3.2",
- "time": "2020-9-1",
- "link": "/architecture-design",
- "title": "System Architecture Design",
- "location": [
- "Architecture Design",
- "Architecture Design"
- ]
- },
- {
- "content": "\nForeword\nThis document is a description of the dolphinscheduler configuration file, and the version is for dolphinscheduler-1.3.x.\nDirectory Structure\nAll configuration files of dolphinscheduler are currently in the [conf] directory.\nFor a more intuitive understanding of the location of the [conf] directory and the configuration files it contains, please see the simplified description of the dolphinscheduler installation directory below.\nThis article mainly talks a [...]
- "version": "1.3.2",
- "time": "2020-9-1",
- "link": "/configuration-file",
- "title": "Foreword",
- "location": [
- "Architecture Design",
- "Configuration File"
- ]
- },
- {
- "content": "Overall task storage structure\nAll tasks created in dolphinscheduler are saved in the t_ds_process_definition table.\nThe database table structure is shown in the following table:\n\n\n\nSerial number\nField\nTypes\nDescription\n\n\n\n\n1\nid\nint(11)\nPrimary key\n\n\n2\nname\nvarchar(255)\nProcess definition name\n\n\n3\nversion\nint(11)\nProcess definition version\n\n\n4\nrelease_state\ntinyint(4)\nRelease status of the process definition：0 not online, 1 online\n\n\n5 [...]
- "version": "1.3.2",
- "time": "2020-9-1",
- "link": "/task-structure",
- "title": "Overall task storage structure",
- "location": [
- "Architecture Design",
- "Task Structure"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back up previous version's files and database.\n2. Stop all services of DolphinScheduler.\nsh ./script/stop-all.sh\n3. Download the new version's installation package.\n\ndownload the latest version of the installation packages.\nThe following upgrade operations need to be performed in the new version's directory.\n\n4. Database upgrade\n\n\nModify the following properties in conf/datasource.properties.\n\n\nIf you use MySQL as d [...]
- "version": "1.3.2",
- "time": "2020-9-1",
- "link": "/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Upgrade",
- "Upgrade"
- ]
- },
- {
- "content": "DolphinScheduler Expansion and Reduction\n1. Expansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine where the expansion master or worker node is located has already installed the scheduled service, skip to [1.4 Modify configuration] Edit the configuration fil [...]
- "version": "1.3.2",
- "time": "2020-9-1",
- "link": "/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Expansion/Reduction",
- "Capacity expansion and reduction"
- ]
- },
- {
- "content": "Change Log\nNew Features\n[#2815] One worker server can set multiple worker groups\n[#3370] JVM parameter optimization\n[#3223] Click DAG connection to add label function\nBug Fixes\n[#3536] If user didn't have tenant,create resource will NPE\n[#3480] zookeeper multi directories, tasks cannot be assigned\n[#3431] After the resource is re-uploaded, the deleted resource directory displayed in the workflow definition is incorrect\n[#3463] rename the udf resource file associa [...]
- "version": "1.3.2",
- "time": "2020-9-1",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "Hareware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux operating system version requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "1.3.3",
- "time": "2020-11-9",
- "link": "/hardware-environment",
- "title": "Hareware Environment",
- "location": [
- "Deployment Document",
- "Hardware Environment"
- ]
- },
- {
- "content": "Standalone Deployment\n1、Install basic softwares (please install required softwares by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One,If use MySQL,It is strongly recommended that MySQL version is 5.7 or higher\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/ [...]
- "version": "1.3.3",
- "time": "2020-11-9",
- "link": "/standalone-deployment",
- "title": "Standalone Deployment",
- "location": [
- "Deployment Document",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\n1、Before you begin (please install requirement basic software by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One,If use MySQL,It is strongly recommended that MySQL version is 5.7 or higher\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/Ce [...]
- "version": "1.3.3",
- "time": "2020-11-9",
- "link": "/cluster-deployment",
- "title": "Cluster Deployment",
- "location": [
- "Deployment Document",
- "Cluster deploy"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password：admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate an worker group\n\n \n \n \n\n\nCreate an token\n\n \n \n\n\nLog in with regular users\n\n\n\nClick on the user name in the upper right corner to "exit& [...]
- "version": "1.3.3",
- "time": "2020-11-9",
- "link": "/quick-start",
- "title": "Quick Start",
- "location": [
- "User Manual",
- "Quick Start"
- ]
- },
- {
- "content": "System User Manual\nGet started quickly\n\nPlease refer to Quick Start\n\nOperation guide\n1. Home\nThe home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.\n\n\n\n2. Project management\n2.1 Create project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Su [...]
- "version": "1.3.3",
- "time": "2020-11-9",
- "link": "/system-manual",
- "title": "System User Manual",
- "location": [
- "User Manual",
- "User Manual"
- ]
- },
- {
- "content": "Dolphin Scheduler 1.3 MetaData\n\nDolphin Scheduler 1.2 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nque [...]
- "version": "1.3.3",
- "time": "2020-11-9",
- "link": "/metadata-1.3",
- "title": "Dolphin Scheduler 1.3 MetaData",
- "location": [
- "Architecture Design",
- "Metadata"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the scheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the form of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until there are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "1.3.3",
- "time": "2020-11-9",
- "link": "/architecture-design",
- "title": "System Architecture Design",
- "location": [
- "Architecture Design",
- "Architecture Design"
- ]
- },
- {
- "content": "\nForeword\nThis document is a description of the dolphinscheduler configuration file, and the version is for dolphinscheduler-1.3.x.\nDirectory Structure\nAll configuration files of dolphinscheduler are currently in the [conf] directory.\nFor a more intuitive understanding of the location of the [conf] directory and the configuration files it contains, please see the simplified description of the dolphinscheduler installation directory below.\nThis article mainly talks a [...]
- "version": "1.3.3",
- "time": "2020-11-9",
- "link": "/configuration-file",
- "title": "Foreword",
- "location": [
- "Architecture Design",
- "Configuration File"
- ]
- },
- {
- "content": "Overall task storage structure\nAll tasks created in dolphinscheduler are saved in the t_ds_process_definition table.\nThe database table structure is shown in the following table:\n\n\n\nSerial number\nField\nTypes\nDescription\n\n\n\n\n1\nid\nint(11)\nPrimary key\n\n\n2\nname\nvarchar(255)\nProcess definition name\n\n\n3\nversion\nint(11)\nProcess definition version\n\n\n4\nrelease_state\ntinyint(4)\nRelease status of the process definition：0 not online, 1 online\n\n\n5 [...]
- "version": "1.3.3",
- "time": "2020-11-9",
- "link": "/task-structure",
- "title": "Overall task storage structure",
- "location": [
- "Architecture Design",
- "Task Structure"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back up previous version's files and database.\n2. Stop all services of DolphinScheduler.\nsh ./script/stop-all.sh\n3. Download the new version's installation package.\n\ndownload the latest version of the installation packages.\nThe following upgrade operations need to be performed in the new version's directory.\n\n4. Database upgrade\n\n\nModify the following properties in conf/datasource.properties.\n\n\nIf you use MySQL as d [...]
- "version": "1.3.3",
- "time": "2020-11-9",
- "link": "/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Upgrade",
- "Upgrade"
- ]
- },
- {
- "content": "DolphinScheduler Expansion and Reduction\n1. Expansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine where the expansion master or worker node is located has already installed the scheduled service, skip to [1.4 Modify configuration] Edit the configuration fil [...]
- "version": "1.3.3",
- "time": "2020-11-9",
- "link": "/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Expansion/Reduction",
- "Capacity expansion and reduction"
- ]
- },
- {
- "content": "Change Log\nBug Fixes\n[#3615] [master] After the task is executed successfully, but the next task has not been submitted, stop the master，the workflow will fail\n[#4010] [master] When the condition node fails, the process status is failed after fault tolerance\n[#3929] [master/worker] After the master and worker are fault-tolerant, the process instance fails after the task instance is successfully executed\n[#3994] [Sql] Failed to import SQL file of sql/dolphinscheduler_ [...]
- "version": "1.3.3",
- "time": "2020-11-9",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "Hareware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux operating system version requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "1.3.4",
- "time": "2020-12-29",
- "link": "/hardware-environment",
- "title": "Hareware Environment",
- "location": [
- "Deployment Document",
- "Hardware Environment"
- ]
- },
- {
- "content": "Standalone Deployment\n1、Install basic softwares (please install required softwares by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One, JDBC Driver 5.1.47+ is required if MySQL is used\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/CentOS/Ubuntu/Debian\nHado [...]
- "version": "1.3.4",
- "time": "2020-12-29",
- "link": "/standalone-deployment",
- "title": "Standalone Deployment",
- "location": [
- "Deployment Document",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\n1、Before you begin (please install requirement basic software by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One, JDBC Driver 5.1.47+ is required if MySQL is used\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/CentOS/Ubuntu/Debian\nHadoop [...]
- "version": "1.3.4",
- "time": "2020-12-29",
- "link": "/cluster-deployment",
- "title": "Cluster Deployment",
- "location": [
- "Deployment Document",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Docker\nPrerequisites\n\nDocker 1.13.1+\nDocker Compose 1.11.0+\n\nHow to use this Docker image\nHere're 3 ways to quickly install DolphinScheduler\nThe First Way: Start a DolphinScheduler by docker-compose (recommended)\nIn this way, you need to install docker-compose as a prerequisite, please install it yourself according to the rich docker-compose installation guidance on the Internet\n1. Download the Source Code Package\nPlease download the source code p [...]
- "version": "1.3.4",
- "time": "2020-12-29",
- "link": "/docker-deployment",
- "title": "QuickStart in Docker",
- "location": [
- "Deployment Document",
- "Docker Deployment"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password：admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate an worker group\n\n \n \n \n\n\nCreate an token\n\n \n \n\n\nLog in with regular users\n\n\n\nClick on the user name in the upper right corner to "exit& [...]
- "version": "1.3.4",
- "time": "2020-12-29",
- "link": "/quick-start",
- "title": "Quick Start",
- "location": [
- "User Manual",
- "Quick Start"
- ]
- },
- {
- "content": "System User Manual\nGet started quickly\n\nPlease refer to Quick Start\n\nOperation guide\n1. Home\nThe home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.\n\n\n\n2. Project management\n2.1 Create project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Su [...]
- "version": "1.3.4",
- "time": "2020-12-29",
- "link": "/system-manual",
- "title": "System User Manual",
- "location": [
- "User Manual",
- "User Manual"
- ]
- },
- {
- "content": "Dolphin Scheduler 1.3 MetaData\n\nDolphin Scheduler 1.2 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nque [...]
- "version": "1.3.4",
- "time": "2020-12-29",
- "link": "/metadata-1.3",
- "title": "Dolphin Scheduler 1.3 MetaData",
- "location": [
- "Architecture Design",
- "Metadata"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the scheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the form of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until there are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "1.3.4",
- "time": "2020-12-29",
- "link": "/architecture-design",
- "title": "System Architecture Design",
- "location": [
- "Architecture Design",
- "Architecture Design"
- ]
- },
- {
- "content": "\nPreface\nThis document explains the DolphinScheduler application configurations according to DolphinScheduler-1.3.x versions.\nDirectory Structure\nCurrently, all the configuration files are under [conf ] directory. Please check the following simplified DolphinScheduler installation directories to have a direct view about the position [conf] directory in and configuration files inside. This document only describes DolphinScheduler configurations and other modules are no [...]
- "version": "1.3.4",
- "time": "2020-12-29",
- "link": "/configuration-file",
- "title": "Preface",
- "location": [
- "Architecture Design",
- "Configuration File"
- ]
- },
- {
- "content": "Overall Tasks Storage Structure\nAll tasks created in Dolphinscheduler are saved in the t_ds_process_definition table.\nThe following shows the 't_ds_process_definition' table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not online, 1 online\n\n\n5\nprojec [...]
- "version": "1.3.4",
- "time": "2020-12-29",
- "link": "/task-structure",
- "title": "Overall Tasks Storage Structure",
- "location": [
- "Architecture Design",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker load balancing algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\nWeighted random (random)\nSmoothing polling (roundrobin)\nLinear load (lowerweight)\nThe default configuration is the linear [...]
- "version": "1.3.4",
- "time": "2020-12-29",
- "link": "/load-balance",
- "title": "Load Balance",
- "location": [
- "Architecture Design",
- "Load Balance"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back up previous version's files and database.\n2. Stop all services of DolphinScheduler.\nsh ./script/stop-all.sh\n3. Download the new version's installation package.\n\ndownload the latest version of the installation packages.\nThe following upgrade operations need to be performed in the new version's directory.\n\n4. Database upgrade\n\n\nModify the following properties in conf/datasource.properties.\n\n\nIf you use MySQL as d [...]
- "version": "1.3.4",
- "time": "2020-12-29",
- "link": "/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Upgrade",
- "Upgrade"
- ]
- },
- {
- "content": "DolphinScheduler Expansion and Reduction\n1. Expansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine where the expansion master or worker node is located has already installed the scheduled service, skip to [1.4 Modify configuration] Edit the configuration fil [...]
- "version": "1.3.4",
- "time": "2020-12-29",
- "link": "/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Expansion/Reduction",
- "Capacity expansion and reduction"
- ]
- },
- {
- "content": "Change Log\nFeatures/Updates\n[#2925][server] init TaskLogger in TaskExecuteProcessor\n[ambari-update] change version from 1.3.3 to 1.3.4\n[sqoop-update] modify sqoop task param when upgrade ds to 1.3.4\n[#4143][quartz-update] upgrade quartz version to 2.3.0\nBug Fixes\n[#2917 #4034 #4104 #4135][sqoop] sqoop optimization and bug fix\n[#3177] task time parameter parsing error\n[#3457][flink] fix flink args build problem\n[#4034][server] fix sqoop import fail\n[#3900][serve [...]
- "version": "1.3.4",
- "time": "2020-12-29",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "Hareware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux operating system version requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/hardware-environment",
- "title": "Hareware Environment",
- "location": [
- "Deployment Document",
- "Hardware Environment"
- ]
- },
- {
- "content": "Standalone Deployment\n1、Install basic softwares (please install required softwares by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One, JDBC Driver 5.1.47+ is required if MySQL is used\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/CentOS/Ubuntu/Debian\nHado [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/standalone-deployment",
- "title": "Standalone Deployment",
- "location": [
- "Deployment Document",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\n1、Before you begin (please install requirement basic software by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One, JDBC Driver 5.1.47+ is required if MySQL is used\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/CentOS/Ubuntu/Debian\nHadoop [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/cluster-deployment",
- "title": "Cluster Deployment",
- "location": [
- "Deployment Document",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Docker\nPrerequisites\n\nDocker 1.13.1+\nDocker Compose 1.11.0+\n\nHow to use this Docker image\nHere're 3 ways to quickly install DolphinScheduler\nThe First Way: Start a DolphinScheduler by docker-compose (recommended)\nIn this way, you need to install docker-compose as a prerequisite, please install it yourself according to the rich docker-compose installation guidance on the Internet\n1. Download the Source Code Package\nPlease download the source code p [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/docker-deployment",
- "title": "QuickStart in Docker",
- "location": [
- "Deployment Document",
- "Docker Deployment"
- ]
- },
- {
- "content": "QuickStart in Kubernetes\nPrerequisites\n\nHelm 3.1.0+\nKubernetes 1.12+\nPV provisioner support in the underlying infrastructure\n\nInstalling the Chart\n\nNOTE: You need to change your Chart.yaml locally to make it work. Due to a change in the Bitnami repo,\nhttps://charts.bitnami.com/bitnami was truncated only containing entries for the latest 6 months (from January 2022 on).\nThis URL: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami contain [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/kubernetes-deployment",
- "title": "QuickStart in Kubernetes",
- "location": [
- "Deployment Document",
- "Kubernetes Deployment"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password：admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate an worker group\n\n \n \n \n\n\nCreate an token\n\n \n \n\n\nLog in with regular users\n\n\n\nClick on the user name in the upper right corner to "exit& [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/quick-start",
- "title": "Quick Start",
- "location": [
- "User Manual",
- "Quick Start"
- ]
- },
- {
- "content": "System User Manual\nGet started quickly\n\nPlease refer to Quick Start\n\nOperation guide\n1. Home\nThe home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.\n\n\n\n2. Project management\n2.1 Create project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Su [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/system-manual",
- "title": "System User Manual",
- "location": [
- "User Manual",
- "User Manual"
- ]
- },
- {
- "content": "Open API\nBackground\nGenerally, projects and processes are created through pages, but integration with third-party systems requires API calls to manage projects and workflows.\nOperating steps\nCreate token\n\nLog in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.\n\n\n \n \n\nSelect the "Expiration time" (Token validity), select "User" (to p [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/open-api",
- "title": "Open API",
- "location": [
- "User Manual",
- "Open API"
- ]
- },
- {
- "content": "Dolphin Scheduler 1.3 MetaData\n\nDolphin Scheduler 1.2 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nque [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/metadata-1.3",
- "title": "Dolphin Scheduler 1.3 MetaData",
- "location": [
- "Architecture Design",
- "Metadata"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the scheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the form of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until there are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/architecture-design",
- "title": "System Architecture Design",
- "location": [
- "Architecture Design",
- "Architecture Design"
- ]
- },
- {
- "content": "\nPreface\nThis document explains the DolphinScheduler application configurations according to DolphinScheduler-1.3.x versions.\nDirectory Structure\nCurrently, all the configuration files are under [conf ] directory. Please check the following simplified DolphinScheduler installation directories to have a direct view about the position [conf] directory in and configuration files inside. This document only describes DolphinScheduler configurations and other modules are no [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/configuration-file",
- "title": "Preface",
- "location": [
- "Architecture Design",
- "Configuration File"
- ]
- },
- {
- "content": "Overall Tasks Storage Structure\nAll tasks created in Dolphinscheduler are saved in the t_ds_process_definition table.\nThe following shows the 't_ds_process_definition' table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not online, 1 online\n\n\n5\nprojec [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/task-structure",
- "title": "Overall Tasks Storage Structure",
- "location": [
- "Architecture Design",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker load balancing algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\nWeighted random (random)\nSmoothing polling (roundrobin)\nLinear load (lowerweight)\nThe default configuration is the linear [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/load-balance",
- "title": "Load Balance",
- "location": [
- "Architecture Design",
- "Load Balance"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back up previous version's files and database.\n2. Stop all services of DolphinScheduler.\nsh ./script/stop-all.sh\n3. Download the new version's installation package.\n\ndownload the latest version of the installation packages.\nThe following upgrade operations need to be performed in the new version's directory.\n\n4. Database upgrade\n\n\nModify the following properties in conf/datasource.properties.\n\n\nIf you use MySQL as d [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Upgrade",
- "Upgrade"
- ]
- },
- {
- "content": "DolphinScheduler Expansion and Reduction\n1. Expansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine where the expansion master or worker node is located has already installed the scheduled service, skip to [1.4 Modify configuration] Edit the configuration fil [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Expansion/Reduction",
- "Capacity expansion and reduction"
- ]
- },
- {
- "content": "Change Log\nFeatures/Updates\n[K8s] Support resource storage when fs.defaultFS=file:///\n[Docker] Update readme\n[UI] Improve parameter name\n[UI] download url resolve and security page disappear delay problem under GENERAL_USER\n[API]enable response resources gzip compression\n[#703]Docker image optimization\n[#4439][api] Cors inteceptor problems\n[#4418][login controller] Need password when open in new tab\nBug Fixes\n[#3237][docker] the file requirements.yaml in helm m [...]
- "version": "1.3.5",
- "time": "2021-2-14",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "Hareware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux operating system version requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/hardware-environment",
- "title": "Hareware Environment",
- "location": [
- "Deployment Document",
- "Hardware Environment"
- ]
- },
- {
- "content": "Standalone Deployment\n1、Install basic softwares (please install required softwares by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One, JDBC Driver 5.1.47+ is required if MySQL is used\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/CentOS/Ubuntu/Debian\nHado [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/standalone-deployment",
- "title": "Standalone Deployment",
- "location": [
- "Deployment Document",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\n1、Before you begin (please install requirement basic software by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One, JDBC Driver 5.1.47+ is required if MySQL is used\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/CentOS/Ubuntu/Debian\nHadoop [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/cluster-deployment",
- "title": "Cluster Deployment",
- "location": [
- "Deployment Document",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Docker\nPrerequisites\n\nDocker 1.13.1+\nDocker Compose 1.11.0+\n\nHow to use this Docker image\nHere're 3 ways to quickly install DolphinScheduler\nThe First Way: Start a DolphinScheduler by docker-compose (recommended)\nIn this way, you need to install docker-compose as a prerequisite, please install it yourself according to the rich docker-compose installation guidance on the Internet\nFor Windows 7-10, you can install Docker Toolbox. For Windows 10 64-bi [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/docker-deployment",
- "title": "QuickStart in Docker",
- "location": [
- "Deployment Document",
- "Docker Deployment"
- ]
- },
- {
- "content": "QuickStart in Kubernetes\nPrerequisites\n\nHelm 3.1.0+\nKubernetes 1.12+\nPV provisioner support in the underlying infrastructure\n\nInstalling the Chart\n\nNOTE: You need to change your Chart.yaml locally to make it work. Due to a change in the Bitnami repo,\nhttps://charts.bitnami.com/bitnami was truncated only containing entries for the latest 6 months (from January 2022 on).\nThis URL: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami contain [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/kubernetes-deployment",
- "title": "QuickStart in Kubernetes",
- "location": [
- "Deployment Document",
- "Kubernetes Deployment"
- ]
- },
- {
- "content": "SkyWalking Agent Deployment\nThe dolphinscheduler-skywalking module provides Skywalking monitor agent for the Dolphinscheduler project.\nThis document describes how to enable Skywalking 8.4+ support with this module (recommended to use SkyWalking 8.5.0).\nInstallation\nThe following configuration is used to enable Skywalking agent.\nThrough environment variable configuration (for Docker Compose)\nModify SKYWALKING environment variables in docker/docker-swarm/config.env.sh [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/skywalking-agent-deployment",
- "title": "SkyWalking Agent Deployment",
- "location": [
- "Deployment Document",
- "SkyWalking-Agent Deployment"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password：admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate an worker group\n\n \n \n \n\n\nCreate an token\n\n \n \n\n\nLog in with regular users\n\n\n\nClick on the user name in the upper right corner to "exit& [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/quick-start",
- "title": "Quick Start",
- "location": [
- "User Manual",
- "Quick Start"
- ]
- },
- {
- "content": "System User Manual\nGet started quickly\n\nPlease refer to Quick Start\n\nOperation guide\n1. Home\nThe home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.\n\n\n\n2. Project management\n2.1 Create project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Su [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/system-manual",
- "title": "System User Manual",
- "location": [
- "User Manual",
- "User Manual"
- ]
- },
- {
- "content": "Open API\nBackground\nGenerally, projects and processes are created through pages, but integration with third-party systems requires API calls to manage projects and workflows.\nThe Operation Steps of DS API Calls\nCreate a token\n\nLog in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.\n\n\n \n \n\nSelect the "Expiration time" (Token validity), select [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/open-api",
- "title": "Open API",
- "location": [
- "User Manual",
- "Open API"
- ]
- },
- {
- "content": "Flink Calls Operating steps\nCreate a queue\n\nLog in to the scheduling system, click "Security", then click "Queue manage" on the left, and click "Create queue" to create a queue.\nFill in the name and value of queue, and click "Submit"\n\n\n \n \nCreate a tenant\n1.The tenant corresponds to a Linux user, which the user worker uses to submit jobs. If Linux OS environment does not have this user, the worker will create this user w [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/flink-call",
- "title": "Flink Calls Operating steps",
- "location": [
- "User Manual",
- "Flink"
- ]
- },
- {
- "content": "Dolphin Scheduler 1.3 MetaData\n\nDolphin Scheduler 1.2 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nque [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/metadata-1.3",
- "title": "Dolphin Scheduler 1.3 MetaData",
- "location": [
- "Architecture Design",
- "Metadata"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the scheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the form of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until there are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/architecture-design",
- "title": "System Architecture Design",
- "location": [
- "Architecture Design",
- "Architecture Design"
- ]
- },
- {
- "content": "\nPreface\nThis document explains the DolphinScheduler application configurations according to DolphinScheduler-1.3.x versions.\nDirectory Structure\nCurrently, all the configuration files are under [conf ] directory. Please check the following simplified DolphinScheduler installation directories to have a direct view about the position [conf] directory in and configuration files inside. This document only describes DolphinScheduler configurations and other modules are no [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/configuration-file",
- "title": "Preface",
- "location": [
- "Architecture Design",
- "Configuration File"
- ]
- },
- {
- "content": "Overall Tasks Storage Structure\nAll tasks created in Dolphinscheduler are saved in the t_ds_process_definition table.\nThe following shows the 't_ds_process_definition' table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not online, 1 online\n\n\n5\nprojec [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/task-structure",
- "title": "Overall Tasks Storage Structure",
- "location": [
- "Architecture Design",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker load balancing algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\nWeighted random (random)\nSmoothing polling (roundrobin)\nLinear load (lowerweight)\nThe default configuration is the linear [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/load-balance",
- "title": "Load Balance",
- "location": [
- "Architecture Design",
- "Load Balance"
- ]
- },
- {
- "content": "Instructions for using the DolphinScheduler's Ambari plug-in\nNote\n\nThis document is intended for users with a basic understanding of Ambari\nThis document is a description of adding the DolphinScheduler service to the installed Ambari service\nThis document is based on version 2.5.2 of Ambari\n\nInstallation preparation\n\n\nPrepare the RPM packages\n\nIt is generated by executing the command mvn -U clean install -Prpmbuild -Dmaven.test.skip=true -X in the project root [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/ambari-integration",
- "title": "Instructions for using the DolphinScheduler's Ambari plug-in",
- "location": [
- "Integration",
- "Ambari Integration"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back up previous version's files and database.\n2. Stop all services of DolphinScheduler.\nsh ./script/stop-all.sh\n3. Download the new version's installation package.\n\ndownload the latest version of the installation packages.\nThe following upgrade operations need to be performed in the new version's directory.\n\n4. Database upgrade\n\n\nModify the following properties in conf/datasource.properties.\n\n\nIf you use MySQL as d [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Upgrade",
- "Upgrade"
- ]
- },
- {
- "content": "DolphinScheduler Expansion and Reduction\n1. Expansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine where the expansion master or worker node is located has already installed the scheduled service, skip to [1.4 Modify configuration] Edit the configuration fil [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Expansion/Reduction",
- "Capacity expansion and reduction"
- ]
- },
- {
- "content": "Change Log\nFeatures/Updates\n[#5062][feature][Worker Group] Create/Edit/Delete Worker Group in Web UI\n[#5175][improvement][Server] Optimize and reduce worker group queries of database and zookeeper\n[#4852][feature] Support SkyWalking agent plugin\n[#4837][improvement][Docker/K8s] Docker & K8s Improvement Plan #4843 #4732 #4871 #4881 #4804 #4886 #2687 #4897 #3124 #5155 #5068 #5100 #2584 #4905 #5351\n[#5023][improvement][Docker/K8s] Improve docker and k8s #2584 #2687 [...]
- "version": "1.3.6",
- "time": "2021-4-28",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "Hareware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux operating system version requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/hardware-environment",
- "title": "Hareware Environment",
- "location": [
- "Deployment Document",
- "Hardware Environment"
- ]
- },
- {
- "content": "Standalone Deployment\n1、Install Basic Software (please install required software by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One, JDBC Driver 5.1.47+ is required if MySQL is used\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/CentOS/Ubuntu/Debian\nHadoop [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/standalone-deployment",
- "title": "Standalone Deployment",
- "location": [
- "Deployment Document",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\n1、Before you begin (please install requirement basic software by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One, JDBC Driver 5.1.47+ is required if MySQL is used\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/CentOS/Ubuntu/Debian\nHadoop [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/cluster-deployment",
- "title": "Cluster Deployment",
- "location": [
- "Deployment Document",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Docker\nPrerequisites\n\nDocker 1.13.1+\nDocker Compose 1.11.0+\n\nHow to use this Docker image\nHere're 3 ways to quickly install DolphinScheduler\nThe First Way: Start a DolphinScheduler by docker-compose (recommended)\nIn this way, you need to install docker-compose as a prerequisite, please install it yourself according to the rich docker-compose installation guidance on the Internet\nFor Windows 7-10, you can install Docker Toolbox. For Windows 10 64-bi [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/docker-deployment",
- "title": "QuickStart in Docker",
- "location": [
- "Deployment Document",
- "Docker Deployment"
- ]
- },
- {
- "content": "QuickStart in Kubernetes\nPrerequisites\n\nHelm 3.1.0+\nKubernetes 1.12+\nPV provisioner support in the underlying infrastructure\n\nInstalling the Chart\n\nNOTE: You need to change your Chart.yaml locally to make it work. Due to a change in the Bitnami repo,\nhttps://charts.bitnami.com/bitnami was truncated only containing entries for the latest 6 months (from January 2022 on).\nThis URL: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami contain [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/kubernetes-deployment",
- "title": "QuickStart in Kubernetes",
- "location": [
- "Deployment Document",
- "Kubernetes Deployment"
- ]
- },
- {
- "content": "SkyWalking Agent Deployment\nThe dolphinscheduler-skywalking module provides SkyWalking monitor agent for the Dolphinscheduler project.\nThis document describes how to enable SkyWalking 8.4+ support with this module (recommended to use SkyWalking 8.5.0).\nInstallation\nThe following configuration is used to enable SkyWalking agent.\nThrough environment variable configuration (for Docker Compose)\nModify SkyWalking environment variables in docker/docker-swarm/config.env.sh [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/skywalking-agent-deployment",
- "title": "SkyWalking Agent Deployment",
- "location": [
- "Deployment Document",
- "SkyWalking-Agent Deployment"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password: admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate a worker group\n\n \n \n \n\n\nCreate a token\n\n \n \n\n\nLogin with regular users\n\n\n\nClick on the user name in the upper right corner to "exit&qu [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/quick-start",
- "title": "Quick Start",
- "location": [
- "User Manual",
- "Quick Start"
- ]
- },
- {
- "content": "System User Manual\nGet started quickly\n\nPlease refer to Quick Start\n\nOperation guide\n1. Home\nThe home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.\n\n\n\n2. Project management\n2.1 Create project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Su [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/system-manual",
- "title": "System User Manual",
- "location": [
- "User Manual",
- "User Manual"
- ]
- },
- {
- "content": "Open API\nBackground\nGenerally, projects and processes are created through pages, but integration with third-party systems requires API calls to manage projects and workflows.\nThe Operation Steps of DS API Calls\nCreate a token\n\nLog in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.\n\n\n \n \n\nSelect the "Expiration time" (Token validity), select [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/open-api",
- "title": "Open API",
- "location": [
- "User Manual",
- "Open API"
- ]
- },
- {
- "content": "Flink Calls Operating steps\nCreate a queue\n\nLog in to the scheduling system, click "Security", then click "Queue manage" on the left, and click "Create queue" to create a queue.\nFill in the name and value of the queue, and click "Submit"\n\n\n \n \nCreate a tenant\n1. The tenant corresponds to a Linux user, which the user worker uses to submit jobs. If Linux OS environment does not have this user, the worker will create this u [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/flink-call",
- "title": "Flink Calls Operating steps",
- "location": [
- "User Manual",
- "Flink"
- ]
- },
- {
- "content": "Dolphinscheduler parameter introduction\n1. System parameters\n\n variabledeclaration methodmeaning\n \n system.biz.date\n ${system.biz.date}\n The day before the scheduled time of the daily scheduling instance, the format is yyyyMMdd, when the data is supplemented, the date is +1\n \n \n system.biz.curdate\n ${system.biz.curdate}\n The timing time of the daily scheduling instance, the format is yyyyMMdd, when the data [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/parameters-introduction",
- "title": "Dolphinscheduler parameter introduction",
- "location": [
- "User Manual",
- "Parameter Introduction"
- ]
- },
- {
- "content": "Dolphin Scheduler 1.3 MetaData\n\nDolphin Scheduler 1.2 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nque [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/metadata-1.3",
- "title": "Dolphin Scheduler 1.3 MetaData",
- "location": [
- "Architecture Design",
- "Metadata"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the scheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the form of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until there are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/architecture-design",
- "title": "System Architecture Design",
- "location": [
- "Architecture Design",
- "Architecture Design"
- ]
- },
- {
- "content": "\nPreface\nThis document explains the DolphinScheduler application configurations according to DolphinScheduler-1.3.x versions.\nDirectory Structure\nCurrently, all the configuration files are under [conf ] directory. Please check the following simplified DolphinScheduler installation directories to have a direct view about the position [conf] directory in and configuration files inside. This document only describes DolphinScheduler configurations and other modules are no [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/configuration-file",
- "title": "Preface",
- "location": [
- "Architecture Design",
- "Configuration File"
- ]
- },
- {
- "content": "Overall Tasks Storage Structure\nAll tasks created in DolphinScheduler are saved in the t_ds_process_definition table.\nThe following shows the 't_ds_process_definition' table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not online, 1 online\n\n\n5\nprojec [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/task-structure",
- "title": "Overall Tasks Storage Structure",
- "location": [
- "Architecture Design",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker load balancing algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\nWeighted random (random)\nSmoothing polling (roundrobin)\nLinear load (lowerweight)\nThe default configuration is the linear [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/load-balance",
- "title": "Load Balance",
- "location": [
- "Architecture Design",
- "Load Balance"
- ]
- },
- {
- "content": "Instructions for using the DolphinScheduler's Ambari plug-in\nNote\n\nThis document is intended for users with a basic understanding of Ambari\nThis document is a description of adding the DolphinScheduler service to the installed Ambari service\nThis document is based on version 2.5.2 of Ambari\n\nInstallation preparation\n\n\nPrepare the RPM packages\n\nIt is generated by executing the command mvn -U clean install -Prpmbuild -Dmaven.test.skip=true -X in the project root [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/ambari-integration",
- "title": "Instructions for using the DolphinScheduler's Ambari plug-in",
- "location": [
- "Integration",
- "Ambari Integration"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back Up Previous Version's Files and Database.\n2. Stop All Services of DolphinScheduler.\nsh ./script/stop-all.sh\n3. Download the New Version's Installation Package.\n\ndownload the latest version of the installation packages.\nThe following upgrade operations need to be performed in the new version's directory.\n\n4. Database Upgrade\n\n\nModify the following properties in conf/datasource.properties.\n\n\nIf you use MySQL as t [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Upgrade",
- "Upgrade"
- ]
- },
- {
- "content": "\nDolphinScheduler Expansion and Reduction\n1. Expansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine where the expansion master or worker node is located has already installed the scheduled service, skip to [1.4 Modify configuration] Edit the configuration f [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Expansion/Reduction",
- "Capacity expansion and reduction"
- ]
- },
- {
- "content": "Change Log\nFeatures/Updates\n[#5405] [Improvement]Docker & K8s Improvement Plan Round 2\n[#5858][improvement][Docker] Docker image should support multi-arch like arm64 in docker-compose\n[#5706][improvement][common] Upgrade the version of fastjson from 1.2.61 to 1.2.75\n[#5577][improvement][UI] Add Project Name in Project Page\n[#5567][improvement][UI] Add project id in web ui url for sharing\n[#5475][improvement][Api] Upload resource to remote failed, the local tmp [...]
- "version": "1.3.8",
- "time": "2021-9-7",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "Hareware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux operating system version requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/hardware-environment",
- "title": "Hareware Environment",
- "location": [
- "Deployment Document",
- "Hardware Environment"
- ]
- },
- {
- "content": "Standalone Experience Quick Deployment (Standalone)\nNOTICE:\nRecommended for less than 20 workflows, Using embedded technology, including H2 Database,Zookeeper Testing Server.\n1. Preparation\n\nJDK (1.8+), please configure JAVA_HOME and PATH variables under /etc/profile\n\n2. One key start\n\nSwitch to a user with sudo privileges and run the script\n\nsh . /bin/dolphinscheduler-daemon.sh start standalone-server\n\n3. Login to the system\n\nAccess front-end page address, [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/standalone-server",
- "title": "Standalone Experience Quick Deployment (Standalone)",
- "location": [
- "Deployment Document",
- "Quick deployment of stand-alone experience version"
- ]
- },
- {
- "content": "Standalone Deployment\n1、Install Basic Software (please install required software by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One, JDBC Driver 5.1.47+ is required if MySQL is used\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/CentOS/Ubuntu/Debian\nHadoop [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/standalone-deployment",
- "title": "Standalone Deployment",
- "location": [
- "Deployment Document",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\n1、Before you begin (please install requirement basic software by yourself)\n\nPostgreSQL (8.2.15+) or MySQL (5.7) : Choose One, JDBC Driver 5.1.47+ is required if MySQL is used\nJDK (1.8+) : Required. Double-check configure JAVA_HOME and PATH environment variables in /etc/profile\nZooKeeper (3.4.6+) : Required\npstree or psmisc : "pstree" is required for Mac OS and "psmisc" is required for Fedora/Red/Hat/CentOS/Ubuntu/Debian\nHadoop [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/cluster-deployment",
- "title": "Cluster Deployment",
- "location": [
- "Deployment Document",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Docker\nPrerequisites\n\nDocker 1.13.1+\nDocker Compose 1.11.0+\n\nHow to use this Docker image\nHere're 3 ways to quickly install DolphinScheduler\nThe First Way: Start a DolphinScheduler by docker-compose (recommended)\nIn this way, you need to install docker-compose as a prerequisite, please install it yourself according to the rich docker-compose installation guidance on the Internet\nFor Windows 7-10, you can install Docker Toolbox. For Windows 10 64-bi [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/docker-deployment",
- "title": "QuickStart in Docker",
- "location": [
- "Deployment Document",
- "Docker Deployment"
- ]
- },
- {
- "content": "QuickStart in Kubernetes\nPrerequisites\n\nHelm 3.1.0+\nKubernetes 1.12+\nPV provisioner support in the underlying infrastructure\n\nInstalling the Chart\n\nNOTE: You need to change your Chart.yaml locally to make it work. Due to a change in the Bitnami repo,\nhttps://charts.bitnami.com/bitnami was truncated only containing entries for the latest 6 months (from January 2022 on).\nThis URL: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami contain [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/kubernetes-deployment",
- "title": "QuickStart in Kubernetes",
- "location": [
- "Deployment Document",
- "Kubernetes Deployment"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password: admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate a worker group\n\n \n \n \n\n\nCreate a token\n\n \n \n\n\nLogin with regular users\n\n\n\nClick on the user name in the upper right corner to "exit&qu [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/quick-start",
- "title": "Quick Start",
- "location": [
- "User Manual",
- "Quick Start"
- ]
- },
- {
- "content": "System User Manual\nGet started quickly\n\nPlease refer to Quick Start\n\nOperation guide\n1. Home\nThe home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.\n\n\n\n2. Project management\n2.1 Create project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Su [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/system-manual",
- "title": "System User Manual",
- "location": [
- "User Manual",
- "User Manual"
- ]
- },
- {
- "content": "Open API\nBackground\nGenerally, projects and processes are created through pages, but integration with third-party systems requires API calls to manage projects and workflows.\nThe Operation Steps of DS API Calls\nCreate a token\n\nLog in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.\n\n\n \n \n\nSelect the "Expiration time" (Token validity), select [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/open-api",
- "title": "Open API",
- "location": [
- "User Manual",
- "Open API"
- ]
- },
- {
- "content": "Flink Calls Operating steps\nCreate a queue\n\nLog in to the scheduling system, click "Security", then click "Queue manage" on the left, and click "Create queue" to create a queue.\nFill in the name and value of the queue, and click "Submit"\n\n\n \n \nCreate a tenant\n1. The tenant corresponds to a Linux user, which the user worker uses to submit jobs. If Linux OS environment does not have this user, the worker will create this u [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/flink-call",
- "title": "Flink Calls Operating steps",
- "location": [
- "User Manual",
- "Flink"
- ]
- },
- {
- "content": "Dolphinscheduler parameter introduction\n1. System parameters\n\n variabledeclaration methodmeaning\n \n system.biz.date\n ${system.biz.date}\n The day before the scheduled time of the daily scheduling instance, the format is yyyyMMdd, when the data is supplemented, the date is +1\n \n \n system.biz.curdate\n ${system.biz.curdate}\n The timing time of the daily scheduling instance, the format is yyyyMMdd, when the data [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/parameters-introduction",
- "title": "Dolphinscheduler parameter introduction",
- "location": [
- "User Manual",
- "Parameter Introduction"
- ]
- },
- {
- "content": "Dolphin Scheduler 1.3 MetaData\n\nDolphin Scheduler 1.2 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nque [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/metadata-1.3",
- "title": "Dolphin Scheduler 1.3 MetaData",
- "location": [
- "Architecture Design",
- "Metadata"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the scheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the form of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until there are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/architecture-design",
- "title": "System Architecture Design",
- "location": [
- "Architecture Design",
- "Architecture Design"
- ]
- },
- {
- "content": "\nPreface\nThis document explains the DolphinScheduler application configurations according to DolphinScheduler-1.3.x versions.\nDirectory Structure\nCurrently, all the configuration files are under [conf ] directory. Please check the following simplified DolphinScheduler installation directories to have a direct view about the position [conf] directory in and configuration files inside. This document only describes DolphinScheduler configurations and other modules are no [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/configuration-file",
- "title": "Preface",
- "location": [
- "Architecture Design",
- "Configuration File"
- ]
- },
- {
- "content": "Overall Tasks Storage Structure\nAll tasks created in DolphinScheduler are saved in the t_ds_process_definition table.\nThe following shows the 't_ds_process_definition' table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not online, 1 online\n\n\n5\nprojec [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/task-structure",
- "title": "Overall Tasks Storage Structure",
- "location": [
- "Architecture Design",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker load balancing algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\nWeighted random (random)\nSmoothing polling (roundrobin)\nLinear load (lowerweight)\nThe default configuration is the linear [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/load-balance",
- "title": "Load Balance",
- "location": [
- "Architecture Design",
- "Load Balance"
- ]
- },
- {
- "content": "SkyWalking Agent\nThe dolphinscheduler-skywalking module provides SkyWalking monitor agent for the Dolphinscheduler project.\nThis document describes how to enable SkyWalking 8.4+ support with this module (recommended to use SkyWalking 8.5.0).\nInstallation\nThe following configuration is used to enable SkyWalking agent.\nThrough environment variable configuration (for Docker Compose)\nModify SkyWalking environment variables in docker/docker-swarm/config.env.sh:\nSKYWALKI [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/skywalking-agent-deployment",
- "title": "SkyWalking Agent",
- "location": [
- "Observability",
- "SkyWalking-Agent"
- ]
- },
- {
- "content": "Instructions for using the DolphinScheduler's Ambari plug-in\nNote\n\nThis document is intended for users with a basic understanding of Ambari\nThis document is a description of adding the DolphinScheduler service to the installed Ambari service\nThis document is based on version 2.5.2 of Ambari\n\nInstallation preparation\n\n\nPrepare the RPM packages\n\nIt is generated by executing the command mvn -U clean install -Prpmbuild -Dmaven.test.skip=true -X in the project root [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/ambari-integration",
- "title": "Instructions for using the DolphinScheduler's Ambari plug-in",
- "location": [
- "Integration",
- "Ambari Integration"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back Up Previous Version's Files and Database.\n2. Stop All Services of DolphinScheduler.\nsh ./script/stop-all.sh\n3. Download the New Version's Installation Package.\n\ndownload the latest version of the installation packages.\nThe following upgrade operations need to be performed in the new version's directory.\n\n4. Database Upgrade\n\n\nModify the following properties in conf/datasource.properties.\n\n\nIf you use MySQL as t [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Upgrade",
- "Upgrade"
- ]
- },
- {
- "content": "DolphinScheduler Expansion and Reduction\n1. Expansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine where the expansion master or worker node is located has already installed the scheduled service, skip to [1.4 Modify configuration] Edit the configuration fil [...]
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Expansion/Reduction",
- "Capacity expansion and reduction"
- ]
- },
- {
- "content": "Change Log\nFeature/Update\n[Feature#6022]Add standalone server module to make it easier to develop and run\nBug Fixes\n[Bug#6429] [UI] sub_process node open sub_task show empty page bug\n[Fix#6337][task] sql limit param no default value\n",
- "version": "1.3.9",
- "time": "2021-10-22",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "Apache DolphinScheduler is a cloud-native visual Big Data workflow scheduler system, committed to “solving complex big-data task dependencies and triggering relationships in data OPS orchestration so that various types of big data tasks can be used out of the box”.\nHigh Reliability\n\nDecentralized multi-master and multi-worker, HA is supported by itself, overload processing\n\nUser-Friendly\n\nAll process definition operations are visualized, Visualization process defin [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/About_DolphinScheduler/About_DolphinScheduler",
- "title": "High Reliability",
- "location": [
- "About",
- "Introduction"
- ]
- },
- {
- "content": "Hardware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux Operating System Version Requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/installation/hardware",
- "title": "Hardware Environment",
- "location": [
- "About",
- "Hardware Environment"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the\nform of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until\nthere are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/architecture/designplus",
- "title": "System Architecture Design",
- "location": [
- "About",
- "Glossary"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password: admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate a worker group\n\n \n \n \n\nCreate environment\n\n \n \n \n\n\nCreate a token\n\n \n \n\n\nLogin with regular users\n\n\n\nClick on the user name in [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/quick-start",
- "title": "Quick Start",
- "location": [
- "Quick Start",
- "Quick Start"
- ]
- },
- {
- "content": "QuickStart in Docker\nPrerequisites\n\nDocker 1.13.1+\nDocker Compose 1.11.0+\n\nHow to use this Docker image\nHere're 3 ways to quickly install DolphinScheduler\nThe First Way: Start a DolphinScheduler by docker-compose (recommended)\nIn this way, you need to install docker-compose as a prerequisite, please install it yourself according to the rich docker-compose installation guidance on the Internet\nFor Windows 7-10, you can install Docker Toolbox. For Windows 10 64-bi [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/installation/docker",
- "title": "QuickStart in Docker",
- "location": [
- "Quick Start",
- "Docker Deployment"
- ]
- },
- {
- "content": "Standalone\nStandalone only for quick look for DolphinScheduler.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in production, we recommended you follow cluster deployment or kubernetes\n\nNote: Standalone only recommends the use [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/installation/standalone",
- "title": "Standalone",
- "location": [
- "Installation",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Pseudo-Cluster Deployment\nThe purpose of pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, api server, and logger server are all on the same machine.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cl [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/installation/pseudo-cluster",
- "title": "Pseudo-Cluster Deployment",
- "location": [
- "Installation",
- "Pseudo Cluster Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\nCluster deployment is to deploy the DolphinScheduler on multiple machines for running a large number of tasks in production.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in production, we recommended you foll [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/installation/cluster",
- "title": "Cluster Deployment",
- "location": [
- "Installation",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Kubernetes\nKubernetes deployment is deploy DolphinScheduler in a Kubernetes cluster, which can schedule a large number of tasks and can be used in production.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in produc [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/installation/kubernetes",
- "title": "QuickStart in Kubernetes",
- "location": [
- "Installation",
- "Kubernetes Deployment"
- ]
- },
- {
- "content": "Workflow Overview\nThe home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.\n\n\n",
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/homepage",
- "title": "Workflow Overview",
- "location": [
- "Introduction to Functions",
- "Workflow Overview"
- ]
- },
- {
- "content": "Project\nCreate project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Submit" to create a new project.\n\n \n\n\n\nProject home\n\n\nClick the project name link on the project management page to enter the project home page, as shown in the figure below, the project home page contains the task status statistics, process status [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/project/project-list",
- "title": "Project",
- "location": [
- "Introduction to Functions",
- "Project",
- "Project List"
- ]
- },
- {
- "content": "Workflow definition\nCreate workflow definition\n\n\nClick Project Management -> Workflow -> Workflow Definition to enter the workflow definition page, and click the "Create Workflow" button to enter the workflow DAG edit page, as shown in the following figure:\n\n \n\n\n\nDrag in the toolbar Add a Shell task to the drawing board, as shown in the figure below:\n\n\n\nAdd parameter settings for this shell task:\n\n\n\nFill in the "Node Name", [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/project/workflow-definition",
- "title": "Workflow definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Definition"
- ]
- },
- {
- "content": "Workflow instance\nView workflow instance\n\nClick Project Management -> Workflow -> Workflow Instance to enter the Workflow Instance page, as shown in the figure below: \n \n \n\nClick the workflow name to enter the DAG view page to view the task execution status, as shown in the figure below.\n \n\n\n\nView task log\n\nEnter the workflow instance page, click the workflow name, enter the DAG view page, double-click the task node, as shown in the following [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/project/workflow-instance",
- "title": "Workflow instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Instance"
- ]
- },
- {
- "content": "Task instance\n\n\nClick Project Management -> Workflow -> Task Instance to enter the task instance page, as shown in the figure below, click the name of the workflow instance, you can jump to the workflow instance DAG chart to view the task status.\n \n \n \n\n\nView log：Click the "view log" button in the operation column to view the log of task execution.\n \n \n \n\n\n",
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/project/task-instance",
- "title": "Task instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Instance"
- ]
- },
- {
- "content": "Shell\nOverview\nShell task, used to create a shell-type task and execute a series of shell scripts. When the worker executed,\na temporary shell script is generated, and the Linux user with the same name as the tenant executes the script.\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/task/shell",
- "title": "Shell",
- "location": [
- "Introduction to Functions",
- "Task",
- "Shell"
- ]
- },
- {
- "content": "SubProcess\n\nThe sub-process node is to execute a certain external workflow definition as a task node.\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\n\n \n \n\nNode name: The node name in a workflow definition is unique\nRun flag: identify whether this node can be scheduled normally\nDescriptive information: describe the function of the node\nTimeout alarm: Check the timeout alarm and timeout failure. When the task ex [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/task/sub-process",
- "title": "SubProcess",
- "location": [
- "Introduction to Functions",
- "Task",
- "SubProcess"
- ]
- },
- {
- "content": "DEPENDENT\n\nDependent nodes are dependency check nodes. For example, process A depends on the successful execution of process B yesterday, and the dependent node will check whether process B has a successful execution yesterday.\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\n\n \n \n\nThe dependent node provides a logical judgment function, such as checking whether the B process was successful yesterday, or whether th [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/task/dependent",
- "title": "DEPENDENT",
- "location": [
- "Introduction to Functions",
- "Task",
- "Dependent"
- ]
- },
- {
- "content": "Stored Procedure\n\nAccording to the selected data source, execute the stored procedure.\n\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nData source: The data source type of the stored procedure supports MySQL and POSTGRESQL, select the corresponding data source\nMethod: is the method name of the stored procedure\nCustom parameters: The custom parameter types of the stored procedure support IN and OUT, and the d [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/task/stored-procedure",
- "title": "Stored Procedure",
- "location": [
- "Introduction to Functions",
- "Task",
- "Stored Procedure"
- ]
- },
- {
- "content": "SQL\nOverview\nSQL task, used to connect to database and execute SQL.\ncreate data source\nRefer to Data Source\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the drawing board.\n\nTask Parameter\n\nData source: select the corresponding data source\nsql type: supports query and non-query. The query is a select type query, which is returned [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/task/sql",
- "title": "SQL",
- "location": [
- "Introduction to Functions",
- "Task",
- "SQL"
- ]
- },
- {
- "content": "SPARK\n\nThrough the SPARK node, you can directly execute the SPARK program. For the spark node, the worker will use the spark-submit method to submit tasks\n\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nProgram type: supports JAVA, Scala and Python three languages\nThe class of the main function: is the full path of the Spark program’s entry Main Class\nMain jar package: Spark jar package\nDeployment mode: sup [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/task/spark",
- "title": "SPARK",
- "location": [
- "Introduction to Functions",
- "Task",
- "Spark"
- ]
- },
- {
- "content": "MapReduce\n\nUsing the MR node, you can directly execute the MR program. For the mr node, the worker will use the hadoop jar method to submit tasks\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\nJAVA program\n \n \n \n\nThe class of the main function: is the full path of the Main Class, the entry point of the MR program\nProgram type: select JAVA language\nMain jar package: is the MR jar package\nCommand line parameter [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/task/map-reduce",
- "title": "MapReduce",
- "location": [
- "Introduction to Functions",
- "Task",
- "MapReduce"
- ]
- },
- {
- "content": "Python\n\nUsing python nodes, you can directly execute python scripts. For python nodes, workers will use python ** to submit tasks.\n\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nScript: Python program developed by the user\nEnvironment Name: Specific which Python interpreter would be use and run Script. If you want to use Python virtualenv, you should create multiply environments for each virtualenv.\nResourc [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/task/python",
- "title": "Python",
- "location": [
- "Introduction to Functions",
- "Task",
- "Python"
- ]
- },
- {
- "content": "Flink\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n\n\nProgram type: supports JAVA, Scala and Python three languages\nThe class of the main function: is the full path of the Main Class, the entry point of the Flink program\nMain jar package: is the Flink jar package\nDeployment mode: support three modes of cluster and local\nNumber of slots: You can set the number of slots\nNumber of taskManage: You can set the numbe [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/task/flink",
- "title": "Flink",
- "location": [
- "Introduction to Functions",
- "Task",
- "Flink"
- ]
- },
- {
- "content": "HTTP\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.\nDescriptive information: describe the function of the node.\nTask priority: When the number of worker threads is insufficient, they are executed in order from [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/task/http",
- "title": "HTTP",
- "location": [
- "Introduction to Functions",
- "Task",
- "HTTP"
- ]
- },
- {
- "content": "DATAX\n\n\nDrag in the toolbarTask node into the drawing board\n\n \n\n\n\nCustom template: When you turn on the custom template switch, you can customize the content of the json configuration file of the datax node (applicable when the control configuration does not meet the requirements)\n\n\nData source: select the data source to extract the data\n\n\nsql statement: the sql statement used to extract data from the target database, the sql query column name is automatica [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/task/datax",
- "title": "DATAX",
- "location": [
- "Introduction to Functions",
- "Task",
- "DataX"
- ]
- },
- {
- "content": "Pigeon\nPigeon is general websocket service tracking task for DolphinScheduler. It can trigger, check status, get log from remote websocket service.\nCreate\nDrag in the toolbarThe task node to the drawing board to create a new Conditions task\nParameter\n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.\nDescriptive i [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/task/pigeon",
- "title": "Pigeon",
- "location": [
- "Introduction to Functions",
- "Task",
- "Pigeon"
- ]
- },
- {
- "content": "Conditions\nConditions is a condition node, determining which downstream task should be run based on the condition set to it. For now, the Conditions support multiple upstream tasks, but only two downstream tasks. When the number of upstream tasks exceeds one, complex upstream dependencies can be achieved through and and or operators.\nCreate\nDrag in the toolbarThe task node to the drawing board to create a new Conditions task, as shown in the figure below:\n \n \n \ [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/task/conditions",
- "title": "Conditions",
- "location": [
- "Introduction to Functions",
- "Task",
- "Conditions"
- ]
- },
- {
- "content": "Switch\nSwitch is a conditional judgment node, which branch should be executes according to the value of global variable and the expression result written by the user.\nCreate\nDrag the in the tool bar to create task. Note After the switch task is created, you must configure it downstream to make parameter Branch flow work.\nParameter\n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it d [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/task/switch",
- "title": "Switch",
- "location": [
- "Introduction to Functions",
- "Task",
- "Switch"
- ]
- },
- {
- "content": "Built-in Parameter\nBasic Built-in Parameter\n\n variabledeclaration methodmeaning\n \n system.biz.date\n ${system.biz.date}\n The day before the scheduled time of the daily scheduling instance, the format is yyyyMMdd, when the data is supplemented, the date is +1\n \n \n system.biz.curdate\n ${system.biz.curdate}\n The timing time of the daily scheduling instance, the format is yyyyMMdd, when the data is supplemented, [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/parameter/built-in",
- "title": "Built-in Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Built-in Parameter"
- ]
- },
- {
- "content": "Global Parameter\nScope\nThe parameters configured on the workflow definition dialog, the whole workflow is it's scope.\nUsage\nthe approach to set global parameters is, after defining the workflow, click the 'save' button, then click the '+' button below the 'Set global':\n\n \n \n\n \n \nThe global_bizdate parameter defined here can be referenced by local parameters of any other task node, and the value of global_bizdate is set to the figure obtained by referencing [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/parameter/global",
- "title": "Global Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Global Parameter"
- ]
- },
- {
- "content": "Local Parameter\nScope\nParameters configured on the task definition dialog, the scope of this parameter only for this task, but if you configured follow Parameter Context, it could passed follow task.\nUsage\nThe approach to set local parameters is, double-click on any node while defining the workflow and click the '+' button next to the 'Custom Parameters':\n\n \n\n\n \n\nIf you want to call the built-in parameter in the local parameters, fill in the value corre [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/parameter/local",
- "title": "Local Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Local Parameter"
- ]
- },
- {
- "content": "Parameter Context\nDolphinScheduler provides the ability to refer to each other between parameters, including: local parameters refer to global parameters, and upstream and downstream parameter transfer. Because of the existence of references, it involves the priority of parameters when the parameter names are the same. see also Parameter Priority\nLocal task use global parameter\nThe premise of local tasks referencing global parameters is that you have already defined Gl [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/parameter/context",
- "title": "Parameter Context",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Context"
- ]
- },
- {
- "content": "Parameter Priority\nDolphinScheduler definition of parameter values involved in may come from three types:\n\nGlobal Parameter: Parameters defined when the workflow saves page definitions\nParameter Context: Parameters passed by upstream nodes\nLocal Parameter：The node's own parameters, which is the parameters defined by the user in [Custom Parameters]. The user can define the value of this part of the parameters at the time of workflow definition.\n\nBecause the value [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/parameter/priority",
- "title": "Parameter Priority",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Priority"
- ]
- },
- {
- "content": "Data Source\nData source center supports MySQL, POSTGRESQL, HIVE/IMPALA, SPARK, CLICKHOUSE, ORACLE, SQLSERVER and other data sources\n\nClick "Data Source Center -> Create Data Source" to create different types of data sources according to requirements.\nClick "Test Connection" to test whether the data source can be successfully connected.\n\n",
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/datasource/introduction",
- "title": "Data Source",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Introduction"
- ]
- },
- {
- "content": "MySQL\n\nData source: select MYSQL\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP hostname: enter the IP to connect to MySQL\nPort: Enter the port to connect to MySQL\nUsername: Set the username for connecting to MySQL\nPassword: Set the password for connecting to MySQL\nDatabase name: Enter the name of the database connected to MySQL\nJdbc connection parameters: parameter settings for MySQL connection, filled [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/datasource/mysql",
- "title": "MySQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "MySQL"
- ]
- },
- {
- "content": "POSTGRESQL\n\nData source: select POSTGRESQL\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Host Name: Enter the IP to connect to POSTGRESQL\nPort: Enter the port to connect to POSTGRESQL\nUsername: Set the username for connecting to POSTGRESQL\nPassword: Set the password for connecting to POSTGRESQL\nDatabase name: Enter the name of the database connected to POSTGRESQL\nJdbc connection parameters: parameter s [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/datasource/postgresql",
- "title": "POSTGRESQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "PostgreSQL"
- ]
- },
- {
- "content": "HIVE\nUse HiveServer2\n \n \n \n\nData source: select HIVE\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Host Name: Enter the IP connected to HIVE\nPort: Enter the port connected to HIVE\nUsername: Set the username for connecting to HIVE\nPassword: Set the password for connecting to HIVE\nDatabase name: Enter the name of the database connected to HIVE\nJdbc connection parameters: parameter settings for HI [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/datasource/hive",
- "title": "HIVE",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "HIVE"
- ]
- },
- {
- "content": "Spark\n\n \n \n\nData source: select Spark\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Hostname: Enter the IP connected to Spark\nPort: Enter the port connected to Spark\nUsername: Set the username for connecting to Spark\nPassword: Set the password for connecting to Spark\nDatabase name: Enter the name of the database connected to Spark\nJdbc connection parameters: parameter settings for Spark connection [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/datasource/spark",
- "title": "Spark",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Spark"
- ]
- },
- {
- "content": "How to create alert plugins and alert groups\nIn version 2.0.0, users need to create alert instances, and then associate them with alert groups, and an alert group can use multiple alert instances, and we will notify them one by one.\nFirst of all, you need to go to the Security Center, select Alarm Group Management, then click Alarm Instance Management on the left, then create an alarm instance, then select the corresponding alarm plug-in and fill in the relevant alarm p [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/alert/alert_plugin_user_guide",
- "title": "How to create alert plugins and alert groups",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Alert Component User Guide "
- ]
- },
- {
- "content": "Enterprise WeChat\nIf you need to use Enterprise WeChat to alert, please modify the alert.properties file after the installation is complete, and then restart the alert service. The configuration example of enterprise WeChat is as follows\n# Enable enterprise WeChat alarm or not: ``true`` or ``false``\nenterprise.wechat.enable="true"\n\n# Enterprise WeChat corpid, each enterprise has a unique corpid, to obtain this information, you can view the "enterprise [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/alert/enterprise-wechat",
- "title": "Enterprise WeChat",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Enterprise Wechat"
- ]
- },
- {
- "content": "Resource Center\nIf you want to use the resource upload function, you can select the local file directory for a single machine(this operation does not need to deploy Hadoop). Or you can also upload to a Hadoop or MinIO cluster, at this time, you need to have Hadoop (2.6+) or MinIO and other related environments\n\nNote:\n\nIf the resource upload function is used, the deployment user in installation and deployment must to have operation authority\nIf you using Hadoop clust [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/resource",
- "title": "Resource Center",
- "location": [
- "Introduction to Functions",
- "Resource"
- ]
- },
- {
- "content": "Monitor\nService management\n\nService management is mainly to monitor and display the health status and basic information of each service in the system\n\nmaster monitoring\n\nMainly related to master information.\n\n\n \n \nworker monitoring\n\nMainly related to worker information.\n\n\n \n \nZookeeper monitoring\n\nMainly related configuration information of each worker and master in ZooKeeper.\n\n\n \n \nDB monitoring\n\nMainly the health of the DB\n\n\n \n \n [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/monitor",
- "title": "Monitor",
- "location": [
- "Introduction to Functions",
- "Monitor"
- ]
- },
- {
- "content": "Security\n\nOnly the administrator account in the security center has the authority to operate. It has functions such as queue management, tenant management, user management, alarm group management, worker group management, token management, etc. In the user management module, resources, data sources, projects, etc. Authorization\nAdministrator login, default user name and password: admin/dolphinscheduler123\n\nCreate queue\n\nQueue is used when the "queue" para [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/security",
- "title": "Security",
- "location": [
- "Introduction to Functions",
- "Security"
- ]
- },
- {
- "content": "Open API\nBackground\nGenerally, projects and processes are created through pages, but integration with third-party systems requires API calls to manage projects and workflows.\nThe Operation Steps of DS API Calls\nCreate a token\n\nLog in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.\n\n\n \n \n\nSelect the "Expiration time" (Token validity), select [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/open-api",
- "title": "Open API",
- "location": [
- "Introduction to Functions",
- "Open API"
- ]
- },
- {
- "content": "Flink Calls Operating steps\nCreate a queue\n\nLog in to the scheduling system, click "Security", then click "Queue manage" on the left, and click "Create queue" to create a queue.\nFill in the name and value of the queue, and click "Submit"\n\n\n \n \nCreate a tenant\n1. The tenant corresponds to a Linux user, which the user worker uses to submit jobs. If Linux OS environment does not have this user, the worker will create this u [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/flink-call",
- "title": "Flink Calls Operating steps",
- "location": [
- "Introduction to Functions",
- "Flink"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back Up Previous Version's Files and Database.\n2. Stop All Services of DolphinScheduler.\nsh ./script/stop-all.sh\n3. Download the New Version's Installation Package.\n\ndownload the latest version of the installation packages.\nThe following upgrade operations need to be performed in the new version's directory.\n\n4. Database Upgrade\n\n\nModify the following properties in conf/datasource.properties.\n\n\nIf you use MySQL as t [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Introduction to Functions",
- "Upgrade"
- ]
- },
- {
- "content": "\nDolphinScheduler Expansion and Reduction\n1. Expansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine where the expansion master or worker node is located has already installed the scheduled service, skip to [1.4 Modify configuration] Edit the configuration f [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Introduction to Functions",
- "Expansion and Reduction"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.System Structure\n1.1 System architecture diagram\n\n \n \n System architecture diagram\n \n\n1.2 Start process activity diagram\n\n \n \n Start process activity diagram\n \n\n1.3 Architecture description\n\n\nMasterServer\nMasterServer adopts a distributed and centerless design concept. Ma [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/architecture/design",
- "title": "System Architecture Design",
- "location": [
- "Advanced Guide",
- "Architecture Design"
- ]
- },
- {
- "content": "Dolphin Scheduler 2.0.0 MetaData\n\nDolphin Scheduler 2.0 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nq [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/architecture/metadata",
- "title": "Dolphin Scheduler 2.0.0 MetaData",
- "location": [
- "Advanced Guide",
- "Metadata"
- ]
- },
- {
- "content": "\nPreface\nThis document explains the DolphinScheduler application configurations according to DolphinScheduler-1.3.x versions.\nDirectory Structure\nCurrently, all the configuration files are under [conf ] directory. Please check the following simplified DolphinScheduler installation directories to have a direct view about the position [conf] directory in and configuration files inside. This document only describes DolphinScheduler configurations and other modules are no [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/architecture/configuration",
- "title": "Preface",
- "location": [
- "Advanced Guide",
- "Configuration File"
- ]
- },
- {
- "content": "Overall Tasks Storage Structure\nAll tasks created in DolphinScheduler are saved in the t_ds_process_definition table.\nThe following shows the 't_ds_process_definition' table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not online, 1 online\n\n\n5\nprojec [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/architecture/task-structure",
- "title": "Overall Tasks Storage Structure",
- "location": [
- "Advanced Guide",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker load balancing algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\nWeighted random (random)\nSmoothing polling (roundrobin)\nLinear load (lowerweight)\nThe default configuration is the linear [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/architecture/load-balance",
- "title": "Load Balance",
- "location": [
- "Advanced Guide",
- "Load Balance"
- ]
- },
- {
- "content": "SkyWalking Agent\nThe dolphinscheduler-skywalking module provides SkyWalking monitor agent for the Dolphinscheduler project.\nThis document describes how to enable SkyWalking 8.4+ support with this module (recommended to use SkyWalking 8.5.0).\nInstallation\nThe following configuration is used to enable SkyWalking agent.\nThrough environment variable configuration (for Docker Compose)\nModify SkyWalking environment variables in docker/docker-swarm/config.env.sh:\nSKYWALKI [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/guide/observability/skywalking-agent",
- "title": "SkyWalking Agent",
- "location": [
- "Observability",
- "SkyWalking-Agent"
- ]
- },
- {
- "content": "Change Log\nFeatures\n[#6597] supports dolphinscheduler upgrade from 1.x to 2.0\n[#6812] speed up the site loading when project exists multi process definitions\n[#6763] remove the detection of dumplicate task names\n[#6655] remove the registry monitor\nBug Fixes\n[#6613] fix bug in task retry mechanism\n[#6707] fix error when set up the global parameters in process defintion\n[#6882] fix process cannt finish and its status always be running\n[#6771] fix failover worker i [...]
- "version": "2.0.0",
- "time": "2021-11-25",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "Apache DolphinScheduler is a cloud-native visual Big Data workflow scheduler system, committed to “solving complex big-data task dependencies and triggering relationships in data OPS orchestration so that various types of big data tasks can be used out of the box”.\nHigh Reliability\n\nDecentralized multi-master and multi-worker, HA is supported by itself, overload processing\n\nUser-Friendly\n\nAll process definition operations are visualized, Visualization process defin [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/About_DolphinScheduler/About_DolphinScheduler",
- "title": "High Reliability",
- "location": [
- "About",
- "Introduction"
- ]
- },
- {
- "content": "Hardware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux Operating System Version Requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/installation/hardware",
- "title": "Hardware Environment",
- "location": [
- "About",
- "Hardware Environment"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the\nform of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until\nthere are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/architecture/designplus",
- "title": "System Architecture Design",
- "location": [
- "About",
- "Glossary"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password: admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate a worker group\n\n \n \n \n\nCreate environment\n\n \n \n \n\n\nCreate a token\n\n \n \n\n\nLogin with regular users\n\n\n\nClick on the user name in [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/quick-start",
- "title": "Quick Start",
- "location": [
- "Quick Start",
- "Quick Start"
- ]
- },
- {
- "content": "QuickStart in Docker\nPrerequisites\n\nDocker 1.13.1+\nDocker Compose 1.11.0+\n\nHow to use this Docker image\nHere're 3 ways to quickly install DolphinScheduler\nThe First Way: Start a DolphinScheduler by docker-compose (recommended)\nIn this way, you need to install docker-compose as a prerequisite, please install it yourself according to the rich docker-compose installation guidance on the Internet\nFor Windows 7-10, you can install Docker Toolbox. For Windows 10 64-bi [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/installation/docker",
- "title": "QuickStart in Docker",
- "location": [
- "Quick Start",
- "Docker Deployment"
- ]
- },
- {
- "content": "Standalone\nStandalone only for quick look for DolphinScheduler.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in production, we recommended you follow cluster deployment or kubernetes\n\nNote: Standalone only recommends the use [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/installation/standalone",
- "title": "Standalone",
- "location": [
- "Installation",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Pseudo-Cluster Deployment\nThe purpose of pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, api server, and logger server are all on the same machine.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cl [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/installation/pseudo-cluster",
- "title": "Pseudo-Cluster Deployment",
- "location": [
- "Installation",
- "Pseudo Cluster Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\nCluster deployment is to deploy the DolphinScheduler on multiple machines for running a large number of tasks in production.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in production, we recommended you foll [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/installation/cluster",
- "title": "Cluster Deployment",
- "location": [
- "Installation",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Kubernetes\nKubernetes deployment is deploy DolphinScheduler in a Kubernetes cluster, which can schedule a large number of tasks and can be used in production.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in produc [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/installation/kubernetes",
- "title": "QuickStart in Kubernetes",
- "location": [
- "Installation",
- "Kubernetes Deployment"
- ]
- },
- {
- "content": "Workflow Overview\nThe home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.\n\n\n",
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/homepage",
- "title": "Workflow Overview",
- "location": [
- "Introduction to Functions",
- "Workflow Overview"
- ]
- },
- {
- "content": "Project\nCreate project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Submit" to create a new project.\n\n \n\n\n\nProject home\n\n\nClick the project name link on the project management page to enter the project home page, as shown in the figure below, the project home page contains the task status statistics, process status [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/project/project-list",
- "title": "Project",
- "location": [
- "Introduction to Functions",
- "Project",
- "Project List"
- ]
- },
- {
- "content": "Workflow definition\nCreate workflow definition\n\n\nClick Project Management -> Workflow -> Workflow Definition to enter the workflow definition page, and click the "Create Workflow" button to enter the workflow DAG edit page, as shown in the following figure:\n\n \n\n\n\nDrag in the toolbar Add a Shell task to the drawing board, as shown in the figure below:\n\n\n\nAdd parameter settings for this shell task:\n\n\n\nFill in the "Node Name", [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/project/workflow-definition",
- "title": "Workflow definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Definition"
- ]
- },
- {
- "content": "Workflow instance\nView workflow instance\n\nClick Project Management -> Workflow -> Workflow Instance to enter the Workflow Instance page, as shown in the figure below: \n \n \n\nClick the workflow name to enter the DAG view page to view the task execution status, as shown in the figure below.\n \n\n\n\nView task log\n\nEnter the workflow instance page, click the workflow name, enter the DAG view page, double-click the task node, as shown in the following [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/project/workflow-instance",
- "title": "Workflow instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Instance"
- ]
- },
- {
- "content": "Task instance\n\n\nClick Project Management -> Workflow -> Task Instance to enter the task instance page, as shown in the figure below, click the name of the workflow instance, you can jump to the workflow instance DAG chart to view the task status.\n \n \n \n\n\nView log：Click the "view log" button in the operation column to view the log of task execution.\n \n \n \n\n\n",
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/project/task-instance",
- "title": "Task instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Instance"
- ]
- },
- {
- "content": "Shell\nOverview\nShell task, used to create a shell-type task and execute a series of shell scripts. When the worker executed,\na temporary shell script is generated, and the Linux user with the same name as the tenant executes the script.\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/task/shell",
- "title": "Shell",
- "location": [
- "Introduction to Functions",
- "Task",
- "Shell"
- ]
- },
- {
- "content": "SubProcess\n\nThe sub-process node is to execute a certain external workflow definition as a task node.\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\n\n \n \n\nNode name: The node name in a workflow definition is unique\nRun flag: identify whether this node can be scheduled normally\nDescriptive information: describe the function of the node\nTimeout alarm: Check the timeout alarm and timeout failure. When the task ex [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/task/sub-process",
- "title": "SubProcess",
- "location": [
- "Introduction to Functions",
- "Task",
- "SubProcess"
- ]
- },
- {
- "content": "DEPENDENT\n\nDependent nodes are dependency check nodes. For example, process A depends on the successful execution of process B yesterday, and the dependent node will check whether process B has a successful execution yesterday.\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\n\n \n \n\nThe dependent node provides a logical judgment function, such as checking whether the B process was successful yesterday, or whether th [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/task/dependent",
- "title": "DEPENDENT",
- "location": [
- "Introduction to Functions",
- "Task",
- "Dependent"
- ]
- },
- {
- "content": "Stored Procedure\n\nAccording to the selected data source, execute the stored procedure.\n\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nData source: The data source type of the stored procedure supports MySQL and POSTGRESQL, select the corresponding data source\nMethod: is the method name of the stored procedure\nCustom parameters: The custom parameter types of the stored procedure support IN and OUT, and the d [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/task/stored-procedure",
- "title": "Stored Procedure",
- "location": [
- "Introduction to Functions",
- "Task",
- "Stored Procedure"
- ]
- },
- {
- "content": "SQL\nOverview\nSQL task, used to connect to database and execute SQL.\ncreate data source\nRefer to Data Source\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the drawing board.\n\nTask Parameter\n\nData source: select the corresponding data source\nsql type: supports query and non-query. The query is a select type query, which is returned [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/task/sql",
- "title": "SQL",
- "location": [
- "Introduction to Functions",
- "Task",
- "SQL"
- ]
- },
- {
- "content": "SPARK\n\nThrough the SPARK node, you can directly execute the SPARK program. For the spark node, the worker will use the spark-submit method to submit tasks\n\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nProgram type: supports JAVA, Scala and Python three languages\nThe class of the main function: is the full path of the Spark program’s entry Main Class\nMain jar package: Spark jar package\nDeployment mode: sup [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/task/spark",
- "title": "SPARK",
- "location": [
- "Introduction to Functions",
- "Task",
- "Spark"
- ]
- },
- {
- "content": "MapReduce\n\nUsing the MR node, you can directly execute the MR program. For the mr node, the worker will use the hadoop jar method to submit tasks\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\nJAVA program\n \n \n \n\nThe class of the main function: is the full path of the Main Class, the entry point of the MR program\nProgram type: select JAVA language\nMain jar package: is the MR jar package\nCommand line parameter [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/task/map-reduce",
- "title": "MapReduce",
- "location": [
- "Introduction to Functions",
- "Task",
- "MapReduce"
- ]
- },
- {
- "content": "Python\n\nUsing python nodes, you can directly execute python scripts. For python nodes, workers will use python ** to submit tasks.\n\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nScript: Python program developed by the user\nEnvironment Name: Specific which Python interpreter would be use and run Script. If you want to use Python virtualenv, you should create multiply environments for each virtualenv.\nResourc [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/task/python",
- "title": "Python",
- "location": [
- "Introduction to Functions",
- "Task",
- "Python"
- ]
- },
- {
- "content": "Flink\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n\n\nProgram type: supports JAVA, Scala and Python three languages\nThe class of the main function: is the full path of the Main Class, the entry point of the Flink program\nMain jar package: is the Flink jar package\nDeployment mode: support three modes of cluster and local\nNumber of slots: You can set the number of slots\nNumber of taskManage: You can set the numbe [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/task/flink",
- "title": "Flink",
- "location": [
- "Introduction to Functions",
- "Task",
- "Flink"
- ]
- },
- {
- "content": "HTTP\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.\nDescriptive information: describe the function of the node.\nTask priority: When the number of worker threads is insufficient, they are executed in order from [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/task/http",
- "title": "HTTP",
- "location": [
- "Introduction to Functions",
- "Task",
- "HTTP"
- ]
- },
- {
- "content": "DATAX\n\n\nDrag in the toolbarTask node into the drawing board\n\n \n\n\n\nCustom template: When you turn on the custom template switch, you can customize the content of the json configuration file of the datax node (applicable when the control configuration does not meet the requirements)\n\n\nData source: select the data source to extract the data\n\n\nsql statement: the sql statement used to extract data from the target database, the sql query column name is automatica [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/task/datax",
- "title": "DATAX",
- "location": [
- "Introduction to Functions",
- "Task",
- "DataX"
- ]
- },
- {
- "content": "Pigeon\nPigeon is general websocket service tracking task for DolphinScheduler. It can trigger, check status, get log from remote websocket service.\nCreate\nDrag in the toolbarThe task node to the drawing board to create a new Conditions task\nParameter\n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.\nDescriptive i [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/task/pigeon",
- "title": "Pigeon",
- "location": [
- "Introduction to Functions",
- "Task",
- "Pigeon"
- ]
- },
- {
- "content": "Conditions\nConditions is a condition node, determining which downstream task should be run based on the condition set to it. For now, the Conditions support multiple upstream tasks, but only two downstream tasks. When the number of upstream tasks exceeds one, complex upstream dependencies can be achieved through and and or operators.\nCreate\nDrag in the toolbarThe task node to the drawing board to create a new Conditions task, as shown in the figure below:\n \n \n \ [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/task/conditions",
- "title": "Conditions",
- "location": [
- "Introduction to Functions",
- "Task",
- "Conditions"
- ]
- },
- {
- "content": "Switch\nSwitch is a conditional judgment node, which branch should be executes according to the value of global variable and the expression result written by the user.\nCreate\nDrag the in the tool bar to create task. Note After the switch task is created, you must configure it downstream to make parameter Branch flow work.\nParameter\n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it d [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/task/switch",
- "title": "Switch",
- "location": [
- "Introduction to Functions",
- "Task",
- "Switch"
- ]
- },
- {
- "content": "Built-in Parameter\nBasic Built-in Parameter\n\n variabledeclaration methodmeaning\n \n system.biz.date\n ${system.biz.date}\n The day before the scheduled time of the daily scheduling instance, the format is yyyyMMdd, when the data is supplemented, the date is +1\n \n \n system.biz.curdate\n ${system.biz.curdate}\n The timing time of the daily scheduling instance, the format is yyyyMMdd, when the data is supplemented, [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/parameter/built-in",
- "title": "Built-in Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Built-in Parameter"
- ]
- },
- {
- "content": "Global Parameter\nScope\nThe parameters configured on the workflow definition dialog, the whole workflow is it's scope.\nUsage\nthe approach to set global parameters is, after defining the workflow, click the 'save' button, then click the '+' button below the 'Set global':\n\n \n \n\n \n \nThe global_bizdate parameter defined here can be referenced by local parameters of any other task node, and the value of global_bizdate is set to the figure obtained by referencing [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/parameter/global",
- "title": "Global Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Global Parameter"
- ]
- },
- {
- "content": "Local Parameter\nScope\nParameters configured on the task definition dialog, the scope of this parameter only for this task, but if you configured follow Parameter Context, it could passed follow task.\nUsage\nThe approach to set local parameters is, double-click on any node while defining the workflow and click the '+' button next to the 'Custom Parameters':\n\n \n\n\n \n\nIf you want to call the built-in parameter in the local parameters, fill in the value corre [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/parameter/local",
- "title": "Local Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Local Parameter"
- ]
- },
- {
- "content": "Parameter Context\nDolphinScheduler provides the ability to refer to each other between parameters, including: local parameters refer to global parameters, and upstream and downstream parameter transfer. Because of the existence of references, it involves the priority of parameters when the parameter names are the same. see also Parameter Priority\nLocal task use global parameter\nThe premise of local tasks referencing global parameters is that you have already defined Gl [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/parameter/context",
- "title": "Parameter Context",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Context"
- ]
- },
- {
- "content": "Parameter Priority\nDolphinScheduler definition of parameter values involved in may come from three types:\n\nGlobal Parameter: Parameters defined when the workflow saves page definitions\nParameter Context: Parameters passed by upstream nodes\nLocal Parameter：The node's own parameters, which is the parameters defined by the user in [Custom Parameters]. The user can define the value of this part of the parameters at the time of workflow definition.\n\nBecause the value [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/parameter/priority",
- "title": "Parameter Priority",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Priority"
- ]
- },
- {
- "content": "Data Source\nData source center supports MySQL, POSTGRESQL, HIVE/IMPALA, SPARK, CLICKHOUSE, ORACLE, SQLSERVER and other data sources\n\nClick "Data Source Center -> Create Data Source" to create different types of data sources according to requirements.\nClick "Test Connection" to test whether the data source can be successfully connected.\n\n",
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/datasource/introduction",
- "title": "Data Source",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Introduction"
- ]
- },
- {
- "content": "MySQL\n\nData source: select MYSQL\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP hostname: enter the IP to connect to MySQL\nPort: Enter the port to connect to MySQL\nUsername: Set the username for connecting to MySQL\nPassword: Set the password for connecting to MySQL\nDatabase name: Enter the name of the database connected to MySQL\nJdbc connection parameters: parameter settings for MySQL connection, filled [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/datasource/mysql",
- "title": "MySQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "MySQL"
- ]
- },
- {
- "content": "POSTGRESQL\n\nData source: select POSTGRESQL\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Host Name: Enter the IP to connect to POSTGRESQL\nPort: Enter the port to connect to POSTGRESQL\nUsername: Set the username for connecting to POSTGRESQL\nPassword: Set the password for connecting to POSTGRESQL\nDatabase name: Enter the name of the database connected to POSTGRESQL\nJdbc connection parameters: parameter s [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/datasource/postgresql",
- "title": "POSTGRESQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "PostgreSQL"
- ]
- },
- {
- "content": "HIVE\nUse HiveServer2\n \n \n \n\nData source: select HIVE\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Host Name: Enter the IP connected to HIVE\nPort: Enter the port connected to HIVE\nUsername: Set the username for connecting to HIVE\nPassword: Set the password for connecting to HIVE\nDatabase name: Enter the name of the database connected to HIVE\nJdbc connection parameters: parameter settings for HI [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/datasource/hive",
- "title": "HIVE",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "HIVE"
- ]
- },
- {
- "content": "Spark\n\n \n \n\nData source: select Spark\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Hostname: Enter the IP connected to Spark\nPort: Enter the port connected to Spark\nUsername: Set the username for connecting to Spark\nPassword: Set the password for connecting to Spark\nDatabase name: Enter the name of the database connected to Spark\nJdbc connection parameters: parameter settings for Spark connection [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/datasource/spark",
- "title": "Spark",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Spark"
- ]
- },
- {
- "content": "How to create alert plugins and alert groups\nIn version 2.0.1, users need to create alert instances, and then associate them with alert groups, and an alert group can use multiple alert instances, and we will notify them one by one.\nFirst of all, you need to go to the Security Center, select Alarm Group Management, then click Alarm Instance Management on the left, then create an alarm instance, then select the corresponding alarm plug-in and fill in the relevant alarm p [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/alert/alert_plugin_user_guide",
- "title": "How to create alert plugins and alert groups",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Alert Component User Guide "
- ]
- },
- {
- "content": "Enterprise WeChat\nIf you need to use Enterprise WeChat to alert, please modify the alert.properties file after the installation is complete, and then restart the alert service. The configuration example of enterprise WeChat is as follows\n# Enable enterprise WeChat alarm or not: ``true`` or ``false``\nenterprise.wechat.enable="true"\n\n# Enterprise WeChat corpid, each enterprise has a unique corpid, to obtain this information, you can view the "enterprise [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/alert/enterprise-wechat",
- "title": "Enterprise WeChat",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Enterprise Wechat"
- ]
- },
- {
- "content": "Resource Center\nIf you want to use the resource upload function, you can select the local file directory for a single machine(this operation does not need to deploy Hadoop). Or you can also upload to a Hadoop or MinIO cluster, at this time, you need to have Hadoop (2.6+) or MinIO and other related environments\n\nNote:\n\nIf the resource upload function is used, the deployment user in installation and deployment must to have operation authority\nIf you using Hadoop clust [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/resource",
- "title": "Resource Center",
- "location": [
- "Introduction to Functions",
- "Resource"
- ]
- },
- {
- "content": "Monitor\nService management\n\nService management is mainly to monitor and display the health status and basic information of each service in the system\n\nmaster monitoring\n\nMainly related to master information.\n\n\n \n \nworker monitoring\n\nMainly related to worker information.\n\n\n \n \nZookeeper monitoring\n\nMainly related configuration information of each worker and master in ZooKeeper.\n\n\n \n \nDB monitoring\n\nMainly the health of the DB\n\n\n \n \n [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/monitor",
- "title": "Monitor",
- "location": [
- "Introduction to Functions",
- "Monitor"
- ]
- },
- {
- "content": "Security\n\nOnly the administrator account in the security center has the authority to operate. It has functions such as queue management, tenant management, user management, alarm group management, worker group management, token management, etc. In the user management module, resources, data sources, projects, etc. Authorization\nAdministrator login, default user name and password: admin/dolphinscheduler123\n\nCreate queue\n\nQueue is used when the "queue" para [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/security",
- "title": "Security",
- "location": [
- "Introduction to Functions",
- "Security"
- ]
- },
- {
- "content": "Open API\nBackground\nGenerally, projects and processes are created through pages, but integration with third-party systems requires API calls to manage projects and workflows.\nThe Operation Steps of DS API Calls\nCreate a token\n\nLog in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.\n\n\n \n \n\nSelect the "Expiration time" (Token validity), select [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/open-api",
- "title": "Open API",
- "location": [
- "Introduction to Functions",
- "Open API"
- ]
- },
- {
- "content": "Flink Calls Operating steps\nCreate a queue\n\nLog in to the scheduling system, click "Security", then click "Queue manage" on the left, and click "Create queue" to create a queue.\nFill in the name and value of the queue, and click "Submit"\n\n\n \n \nCreate a tenant\n1. The tenant corresponds to a Linux user, which the user worker uses to submit jobs. If Linux OS environment does not have this user, the worker will create this u [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/flink-call",
- "title": "Flink Calls Operating steps",
- "location": [
- "Introduction to Functions",
- "Flink"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back Up Previous Version's Files and Database.\n2. Stop All Services of DolphinScheduler.\nsh ./script/stop-all.sh\n3. Download the New Version's Installation Package.\n\nDownload the latest version of the installation packages.\nThe following upgrade operations need to be performed in the new version's directory.\n\n4. Database Upgrade\n\n\nModify the following properties in conf/config/install_config.conf.\n\n\nIf you use MySQL [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Introduction to Functions",
- "Upgrade"
- ]
- },
- {
- "content": "\nDolphinScheduler Expansion and Reduction\n1. Expansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine where the expansion master or worker node is located has already installed the scheduled service, skip to [1.4 Modify configuration] Edit the configuration f [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Introduction to Functions",
- "Expansion and Reduction"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.System Structure\n1.1 System architecture diagram\n\n \n \n System architecture diagram\n \n\n1.2 Start process activity diagram\n\n \n \n Start process activity diagram\n \n\n1.3 Architecture description\n\n\nMasterServer\nMasterServer adopts a distributed and centerless design concept. Ma [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/architecture/design",
- "title": "System Architecture Design",
- "location": [
- "Advanced Guide",
- "Architecture Design"
- ]
- },
- {
- "content": "Dolphin Scheduler 2.0.1 MetaData\n\nDolphin Scheduler 2.0 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nq [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/architecture/metadata",
- "title": "Dolphin Scheduler 2.0.1 MetaData",
- "location": [
- "Advanced Guide",
- "Metadata"
- ]
- },
- {
- "content": "\nPreface\nThis document explains the DolphinScheduler application configurations according to DolphinScheduler-1.3.x versions.\nDirectory Structure\nCurrently, all the configuration files are under [conf ] directory. Please check the following simplified DolphinScheduler installation directories to have a direct view about the position [conf] directory in and configuration files inside. This document only describes DolphinScheduler configurations and other modules are no [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/architecture/configuration",
- "title": "Preface",
- "location": [
- "Advanced Guide",
- "Configuration File"
- ]
- },
- {
- "content": "Overall Tasks Storage Structure\nAll tasks created in DolphinScheduler are saved in the t_ds_process_definition table.\nThe following shows the 't_ds_process_definition' table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not online, 1 online\n\n\n5\nprojec [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/architecture/task-structure",
- "title": "Overall Tasks Storage Structure",
- "location": [
- "Advanced Guide",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker load balancing algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\nWeighted random (random)\nSmoothing polling (roundrobin)\nLinear load (lowerweight)\nThe default configuration is the linear [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/architecture/load-balance",
- "title": "Load Balance",
- "location": [
- "Advanced Guide",
- "Load Balance"
- ]
- },
- {
- "content": "SkyWalking Agent\nThe dolphinscheduler-skywalking module provides SkyWalking monitor agent for the Dolphinscheduler project.\nThis document describes how to enable SkyWalking 8.4+ support with this module (recommended to use SkyWalking 8.5.0).\nInstallation\nThe following configuration is used to enable SkyWalking agent.\nThrough environment variable configuration (for Docker Compose)\nModify SkyWalking environment variables in docker/docker-swarm/config.env.sh:\nSKYWALKI [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/guide/observability/skywalking-agent",
- "title": "SkyWalking Agent",
- "location": [
- "Observability",
- "SkyWalking-Agent"
- ]
- },
- {
- "content": "Change Log\nDownload:\nhttps://dolphinscheduler.apache.org/en-us/download/download.html\nBug fixes\n[#7039] fix that the hive data source connection failed when using Kerberos\n[#7338] repairing standalone service startup failed\n[#7215] fix the problem that the alarm group display cannot be displayed\n[#7152] fix the problem of abnormal file upload\n[#7083] fix the problem that the switch task failed to run\n[#7070] fix the failure of workflow timeout policy\n[#6809] fix [...]
- "version": "2.0.1",
- "time": "2021-12-17",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "Apache DolphinScheduler is a cloud-native visual Big Data workflow scheduler system, committed to “solving complex big-data task dependencies and triggering relationships in data OPS orchestration so that various types of big data tasks can be used out of the box”.\nHigh Reliability\n\nDecentralized multi-master and multi-worker, HA is supported by itself, overload processing\n\nUser-Friendly\n\nAll process definition operations are visualized, Visualization process defin [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/About_DolphinScheduler/About_DolphinScheduler",
- "title": "High Reliability",
- "location": [
- "About",
- "Introduction"
- ]
- },
- {
- "content": "Hardware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux Operating System Version Requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/installation/hardware",
- "title": "Hardware Environment",
- "location": [
- "About",
- "Hardware Environment"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the\nform of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until\nthere are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/architecture/designplus",
- "title": "System Architecture Design",
- "location": [
- "About",
- "Glossary"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password: admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate a worker group\n\n \n \n \n\nCreate environment\n\n \n \n \n\n\nCreate a token\n\n \n \n\n\nLogin with regular users\n\n\n\nClick on the user name in [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/quick-start",
- "title": "Quick Start",
- "location": [
- "Quick Start",
- "Quick Start"
- ]
- },
- {
- "content": "QuickStart in Docker\nPrerequisites\n\nDocker 1.13.1+\nDocker Compose 1.11.0+\n\nHow to use this Docker image\nHere're 3 ways to quickly install DolphinScheduler\nThe First Way: Start a DolphinScheduler by docker-compose (recommended)\nIn this way, you need to install docker-compose as a prerequisite, please install it yourself according to the rich docker-compose installation guidance on the Internet\nFor Windows 7-10, you can install Docker Toolbox. For Windows 10 64-bi [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/installation/docker",
- "title": "QuickStart in Docker",
- "location": [
- "Quick Start",
- "Docker Deployment"
- ]
- },
- {
- "content": "Standalone\nStandalone only for quick look for DolphinScheduler.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in production, we recommended you follow cluster deployment or kubernetes\n\nNote: Standalone only recommends the use [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/installation/standalone",
- "title": "Standalone",
- "location": [
- "Installation",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Pseudo-Cluster Deployment\nThe purpose of pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, api server, and logger server are all on the same machine.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cl [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/installation/pseudo-cluster",
- "title": "Pseudo-Cluster Deployment",
- "location": [
- "Installation",
- "Pseudo Cluster Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\nCluster deployment is to deploy the DolphinScheduler on multiple machines for running a large number of tasks in production.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in production, we recommended you foll [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/installation/cluster",
- "title": "Cluster Deployment",
- "location": [
- "Installation",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Kubernetes\nKubernetes deployment is deploy DolphinScheduler in a Kubernetes cluster, which can schedule a large number of tasks and can be used in production.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in produc [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/installation/kubernetes",
- "title": "QuickStart in Kubernetes",
- "location": [
- "Installation",
- "Kubernetes Deployment"
- ]
- },
- {
- "content": "Workflow Overview\nThe home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.\n\n\n",
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/homepage",
- "title": "Workflow Overview",
- "location": [
- "Introduction to Functions",
- "Workflow Overview"
- ]
- },
- {
- "content": "Project\nCreate project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Submit" to create a new project.\n\n \n\n\n\nProject home\n\n\nClick the project name link on the project management page to enter the project home page, as shown in the figure below, the project home page contains the task status statistics, process status [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/project/project-list",
- "title": "Project",
- "location": [
- "Introduction to Functions",
- "Project",
- "Project List"
- ]
- },
- {
- "content": "Workflow definition\nCreate workflow definition\n\n\nClick Project Management -> Workflow -> Workflow Definition to enter the workflow definition page, and click the "Create Workflow" button to enter the workflow DAG edit page, as shown in the following figure:\n\n \n\n\n\nDrag in the toolbar Add a Shell task to the drawing board, as shown in the figure below:\n\n\n\nAdd parameter settings for this shell task:\n\n\n\nFill in the "Node Name", [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/project/workflow-definition",
- "title": "Workflow definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Definition"
- ]
- },
- {
- "content": "Workflow instance\nView workflow instance\n\nClick Project Management -> Workflow -> Workflow Instance to enter the Workflow Instance page, as shown in the figure below: \n \n \n\nClick the workflow name to enter the DAG view page to view the task execution status, as shown in the figure below.\n \n\n\n\nView task log\n\nEnter the workflow instance page, click the workflow name, enter the DAG view page, double-click the task node, as shown in the following [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/project/workflow-instance",
- "title": "Workflow instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Instance"
- ]
- },
- {
- "content": "Task instance\n\n\nClick Project Management -> Workflow -> Task Instance to enter the task instance page, as shown in the figure below, click the name of the workflow instance, you can jump to the workflow instance DAG chart to view the task status.\n \n \n \n\n\nView log：Click the "view log" button in the operation column to view the log of task execution.\n \n \n \n\n\n",
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/project/task-instance",
- "title": "Task instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Instance"
- ]
- },
- {
- "content": "Shell\nOverview\nShell task, used to create a shell-type task and execute a series of shell scripts. When the worker executed,\na temporary shell script is generated, and the Linux user with the same name as the tenant executes the script.\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/task/shell",
- "title": "Shell",
- "location": [
- "Introduction to Functions",
- "Task",
- "Shell"
- ]
- },
- {
- "content": "SubProcess\n\nThe sub-process node is to execute a certain external workflow definition as a task node.\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\n\n \n \n\nNode name: The node name in a workflow definition is unique\nRun flag: identify whether this node can be scheduled normally\nDescriptive information: describe the function of the node\nTimeout alarm: Check the timeout alarm and timeout failure. When the task ex [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/task/sub-process",
- "title": "SubProcess",
- "location": [
- "Introduction to Functions",
- "Task",
- "SubProcess"
- ]
- },
- {
- "content": "DEPENDENT\n\nDependent nodes are dependency check nodes. For example, process A depends on the successful execution of process B yesterday, and the dependent node will check whether process B has a successful execution yesterday.\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\n\n \n \n\nThe dependent node provides a logical judgment function, such as checking whether the B process was successful yesterday, or whether th [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/task/dependent",
- "title": "DEPENDENT",
- "location": [
- "Introduction to Functions",
- "Task",
- "Dependent"
- ]
- },
- {
- "content": "Stored Procedure\n\nAccording to the selected data source, execute the stored procedure.\n\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nData source: The data source type of the stored procedure supports MySQL and POSTGRESQL, select the corresponding data source\nMethod: is the method name of the stored procedure\nCustom parameters: The custom parameter types of the stored procedure support IN and OUT, and the d [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/task/stored-procedure",
- "title": "Stored Procedure",
- "location": [
- "Introduction to Functions",
- "Task",
- "Stored Procedure"
- ]
- },
- {
- "content": "SQL\nOverview\nSQL task, used to connect to database and execute SQL.\ncreate data source\nRefer to Data Source\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the drawing board.\n\nTask Parameter\n\nData source: select the corresponding data source\nsql type: supports query and non-query. The query is a select type query, which is returned [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/task/sql",
- "title": "SQL",
- "location": [
- "Introduction to Functions",
- "Task",
- "SQL"
- ]
- },
- {
- "content": "SPARK\n\nThrough the SPARK node, you can directly execute the SPARK program. For the spark node, the worker will use the spark-submit method to submit tasks\n\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nProgram type: supports JAVA, Scala and Python three languages\nThe class of the main function: is the full path of the Spark program’s entry Main Class\nMain jar package: Spark jar package\nDeployment mode: sup [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/task/spark",
- "title": "SPARK",
- "location": [
- "Introduction to Functions",
- "Task",
- "Spark"
- ]
- },
- {
- "content": "MapReduce\n\nUsing the MR node, you can directly execute the MR program. For the mr node, the worker will use the hadoop jar method to submit tasks\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\nJAVA program\n \n \n \n\nThe class of the main function: is the full path of the Main Class, the entry point of the MR program\nProgram type: select JAVA language\nMain jar package: is the MR jar package\nCommand line parameter [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/task/map-reduce",
- "title": "MapReduce",
- "location": [
- "Introduction to Functions",
- "Task",
- "MapReduce"
- ]
- },
- {
- "content": "Python\n\nUsing python nodes, you can directly execute python scripts. For python nodes, workers will use python ** to submit tasks.\n\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nScript: Python program developed by the user\nEnvironment Name: Specific which Python interpreter would be use and run Script. If you want to use Python virtualenv, you should create multiply environments for each virtualenv.\nResourc [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/task/python",
- "title": "Python",
- "location": [
- "Introduction to Functions",
- "Task",
- "Python"
- ]
- },
- {
- "content": "Flink\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n\n\nProgram type: supports JAVA, Scala and Python three languages\nThe class of the main function: is the full path of the Main Class, the entry point of the Flink program\nMain jar package: is the Flink jar package\nDeployment mode: support three modes of cluster and local\nNumber of slots: You can set the number of slots\nNumber of taskManage: You can set the numbe [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/task/flink",
- "title": "Flink",
- "location": [
- "Introduction to Functions",
- "Task",
- "Flink"
- ]
- },
- {
- "content": "HTTP\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.\nDescriptive information: describe the function of the node.\nTask priority: When the number of worker threads is insufficient, they are executed in order from [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/task/http",
- "title": "HTTP",
- "location": [
- "Introduction to Functions",
- "Task",
- "HTTP"
- ]
- },
- {
- "content": "DATAX\n\n\nDrag in the toolbarTask node into the drawing board\n\n \n\n\n\nCustom template: When you turn on the custom template switch, you can customize the content of the json configuration file of the datax node (applicable when the control configuration does not meet the requirements)\n\n\nData source: select the data source to extract the data\n\n\nsql statement: the sql statement used to extract data from the target database, the sql query column name is automatica [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/task/datax",
- "title": "DATAX",
- "location": [
- "Introduction to Functions",
- "Task",
- "DataX"
- ]
- },
- {
- "content": "Pigeon\nPigeon is general websocket service tracking task for DolphinScheduler. It can trigger, check status, get log from remote websocket service.\nCreate\nDrag in the toolbarThe task node to the drawing board to create a new Conditions task\nParameter\n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.\nDescriptive i [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/task/pigeon",
- "title": "Pigeon",
- "location": [
- "Introduction to Functions",
- "Task",
- "Pigeon"
- ]
- },
- {
- "content": "Conditions\nConditions is a condition node, determining which downstream task should be run based on the condition set to it. For now, the Conditions support multiple upstream tasks, but only two downstream tasks. When the number of upstream tasks exceeds one, complex upstream dependencies can be achieved through and and or operators.\nCreate\nDrag in the toolbarThe task node to the drawing board to create a new Conditions task, as shown in the figure below:\n \n \n \ [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/task/conditions",
- "title": "Conditions",
- "location": [
- "Introduction to Functions",
- "Task",
- "Conditions"
- ]
- },
- {
- "content": "Switch\nSwitch is a conditional judgment node, which branch should be executes according to the value of global variable and the expression result written by the user.\nCreate\nDrag the in the tool bar to create task. Note After the switch task is created, you must configure it downstream to make parameter Branch flow work.\nParameter\n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it d [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/task/switch",
- "title": "Switch",
- "location": [
- "Introduction to Functions",
- "Task",
- "Switch"
- ]
- },
- {
- "content": "Built-in Parameter\nBasic Built-in Parameter\n\n variabledeclaration methodmeaning\n \n system.biz.date\n ${system.biz.date}\n The day before the scheduled time of the daily scheduling instance, the format is yyyyMMdd\n \n \n system.biz.curdate\n ${system.biz.curdate}\n The timing time of the daily scheduling instance, the format is yyyyMMdd\n \n \n system.datetime\n ${system.datetime}\n The [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/parameter/built-in",
- "title": "Built-in Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Built-in Parameter"
- ]
- },
- {
- "content": "Global Parameter\nScope\nThe parameters configured on the workflow definition dialog, the whole workflow is it's scope.\nUsage\nthe approach to set global parameters is, after defining the workflow, click the 'save' button, then click the '+' button below the 'Set global':\n\n \n \n\n \n \nThe global_bizdate parameter defined here can be referenced by local parameters of any other task node, and the value of global_bizdate is set to the figure obtained by referencing [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/parameter/global",
- "title": "Global Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Global Parameter"
- ]
- },
- {
- "content": "Local Parameter\nScope\nParameters configured on the task definition dialog, the scope of this parameter only for this task, but if you configured follow Parameter Context, it could passed follow task.\nUsage\nThe approach to set local parameters is, double-click on any node while defining the workflow and click the '+' button next to the 'Custom Parameters':\n\n \n\n\n \n\nIf you want to call the built-in parameter in the local parameters, fill in the value corre [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/parameter/local",
- "title": "Local Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Local Parameter"
- ]
- },
- {
- "content": "Parameter Context\nDolphinScheduler provides the ability to refer to each other between parameters, including: local parameters refer to global parameters, and upstream and downstream parameter transfer. Because of the existence of references, it involves the priority of parameters when the parameter names are the same. see also Parameter Priority\nLocal task use global parameter\nThe premise of local tasks referencing global parameters is that you have already defined Gl [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/parameter/context",
- "title": "Parameter Context",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Context"
- ]
- },
- {
- "content": "Parameter Priority\nDolphinScheduler definition of parameter values involved in may come from three types:\n\nGlobal Parameter: Parameters defined when the workflow saves page definitions\nParameter Context: Parameters passed by upstream nodes\nLocal Parameter：The node's own parameters, which is the parameters defined by the user in [Custom Parameters]. The user can define the value of this part of the parameters at the time of workflow definition.\n\nBecause the value [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/parameter/priority",
- "title": "Parameter Priority",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Priority"
- ]
- },
- {
- "content": "Data Source\nData source center supports MySQL, POSTGRESQL, HIVE/IMPALA, SPARK, CLICKHOUSE, ORACLE, SQLSERVER and other data sources\n\nClick "Data Source Center -> Create Data Source" to create different types of data sources according to requirements.\nClick "Test Connection" to test whether the data source can be successfully connected.\n\n",
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/datasource/introduction",
- "title": "Data Source",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Introduction"
- ]
- },
- {
- "content": "MySQL\n\nData source: select MYSQL\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP hostname: enter the IP to connect to MySQL\nPort: Enter the port to connect to MySQL\nUsername: Set the username for connecting to MySQL\nPassword: Set the password for connecting to MySQL\nDatabase name: Enter the name of the database connected to MySQL\nJdbc connection parameters: parameter settings for MySQL connection, filled [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/datasource/mysql",
- "title": "MySQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "MySQL"
- ]
- },
- {
- "content": "POSTGRESQL\n\nData source: select POSTGRESQL\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Host Name: Enter the IP to connect to POSTGRESQL\nPort: Enter the port to connect to POSTGRESQL\nUsername: Set the username for connecting to POSTGRESQL\nPassword: Set the password for connecting to POSTGRESQL\nDatabase name: Enter the name of the database connected to POSTGRESQL\nJdbc connection parameters: parameter s [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/datasource/postgresql",
- "title": "POSTGRESQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "PostgreSQL"
- ]
- },
- {
- "content": "HIVE\nUse HiveServer2\n \n \n \n\nData source: select HIVE\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Host Name: Enter the IP connected to HIVE\nPort: Enter the port connected to HIVE\nUsername: Set the username for connecting to HIVE\nPassword: Set the password for connecting to HIVE\nDatabase name: Enter the name of the database connected to HIVE\nJdbc connection parameters: parameter settings for HI [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/datasource/hive",
- "title": "HIVE",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "HIVE"
- ]
- },
- {
- "content": "Spark\n\n \n \n\nData source: select Spark\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Hostname: Enter the IP connected to Spark\nPort: Enter the port connected to Spark\nUsername: Set the username for connecting to Spark\nPassword: Set the password for connecting to Spark\nDatabase name: Enter the name of the database connected to Spark\nJdbc connection parameters: parameter settings for Spark connection [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/datasource/spark",
- "title": "Spark",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Spark"
- ]
- },
- {
- "content": "How to create alert plugins and alert groups\nIn version 2.0.2, users need to create alert instances, and then associate them with alert groups, and an alert group can use multiple alert instances, and we will notify them one by one.\nFirst of all, you need to go to the Security Center, select Alarm Group Management, then click Alarm Instance Management on the left, then create an alarm instance, then select the corresponding alarm plug-in and fill in the relevant alarm p [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/alert/alert_plugin_user_guide",
- "title": "How to create alert plugins and alert groups",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Alert Component User Guide "
- ]
- },
- {
- "content": "Enterprise WeChat\nIf you need to use Enterprise WeChat to alert, please create an alarm Instance in warning instance manage, and then choose the wechat plugin. The configuration example of enterprise WeChat is as follows\n\nWhere send type corresponds to app and appchat respectively:\nAPP: https://work.weixin.qq.com/api/doc/90000/90135/90236\nAPPCHAT: https://work.weixin.qq.com/api/doc/90000/90135/90248\nuser.send.msg corresponds to the content in the document. The varia [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/alert/enterprise-wechat",
- "title": "Enterprise WeChat",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Enterprise Wechat"
- ]
- },
- {
- "content": "Resource Center\nIf you want to use the resource upload function, you can select the local file directory for a single machine(this operation does not need to deploy Hadoop). Or you can also upload to a Hadoop or MinIO cluster, at this time, you need to have Hadoop (2.6+) or MinIO and other related environments\n\nNote:\n\nIf the resource upload function is used, the deployment user in installation and deployment must to have operation authority\nIf you using Hadoop clust [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/resource",
- "title": "Resource Center",
- "location": [
- "Introduction to Functions",
- "Resource"
- ]
- },
- {
- "content": "Monitor\nService management\n\nService management is mainly to monitor and display the health status and basic information of each service in the system\n\nmaster monitoring\n\nMainly related to master information.\n\n\n \n \nworker monitoring\n\nMainly related to worker information.\n\n\n \n \nZookeeper monitoring\n\nMainly related configuration information of each worker and master in ZooKeeper.\n\n\n \n \nDB monitoring\n\nMainly the health of the DB\n\n\n \n \n [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/monitor",
- "title": "Monitor",
- "location": [
- "Introduction to Functions",
- "Monitor"
- ]
- },
- {
- "content": "Security\n\nOnly the administrator account in the security center has the authority to operate. It has functions such as queue management, tenant management, user management, alarm group management, worker group management, token management, etc. In the user management module, resources, data sources, projects, etc. Authorization\nAdministrator login, default user name and password: admin/dolphinscheduler123\n\nCreate queue\n\nQueue is used when the "queue" para [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/security",
- "title": "Security",
- "location": [
- "Introduction to Functions",
- "Security"
- ]
- },
- {
- "content": "Open API\nBackground\nGenerally, projects and processes are created through pages, but integration with third-party systems requires API calls to manage projects and workflows.\nThe Operation Steps of DS API Calls\nCreate a token\n\nLog in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.\n\n\n \n \n\nSelect the "Expiration time" (Token validity), select [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/open-api",
- "title": "Open API",
- "location": [
- "Introduction to Functions",
- "Open API"
- ]
- },
- {
- "content": "Flink Calls Operating steps\nCreate a queue\n\nLog in to the scheduling system, click "Security", then click "Queue manage" on the left, and click "Create queue" to create a queue.\nFill in the name and value of the queue, and click "Submit"\n\n\n \n \nCreate a tenant\n1. The tenant corresponds to a Linux user, which the user worker uses to submit jobs. If Linux OS environment does not have this user, the worker will create this u [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/flink-call",
- "title": "Flink Calls Operating steps",
- "location": [
- "Introduction to Functions",
- "Flink"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back Up Previous Version's Files and Database.\n2. Stop All Services of DolphinScheduler.\nsh ./script/stop-all.sh\n3. Download the New Version's Installation Package.\n\nDownload the latest version of the installation packages.\nThe following upgrade operations need to be performed in the new version's directory.\n\n4. Database Upgrade\n\n\nModify the following properties in conf/config/install_config.conf.\n\n\nIf you use MySQL [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Introduction to Functions",
- "Upgrade"
- ]
- },
- {
- "content": "\nDolphinScheduler Expansion and Reduction\n1. Expansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine where the expansion master or worker node is located has already installed the scheduled service, skip to [1.4 Modify configuration] Edit the configuration f [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Introduction to Functions",
- "Expansion and Reduction"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.System Structure\n1.1 System architecture diagram\n\n \n \n System architecture diagram\n \n\n1.2 Start process activity diagram\n\n \n \n Start process activity diagram\n \n\n1.3 Architecture description\n\n\nMasterServer\nMasterServer adopts a distributed and centerless design concept. Ma [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/architecture/design",
- "title": "System Architecture Design",
- "location": [
- "Advanced Guide",
- "Architecture Design"
- ]
- },
- {
- "content": "Dolphin Scheduler 2.0.2 MetaData\n\nDolphin Scheduler 2.0 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nq [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/architecture/metadata",
- "title": "Dolphin Scheduler 2.0.2 MetaData",
- "location": [
- "Advanced Guide",
- "Metadata"
- ]
- },
- {
- "content": "\nPreface\nThis document explains the DolphinScheduler application configurations according to DolphinScheduler-1.3.x versions.\nDirectory Structure\nCurrently, all the configuration files are under [conf ] directory. Please check the following simplified DolphinScheduler installation directories to have a direct view about the position [conf] directory in and configuration files inside. This document only describes DolphinScheduler configurations and other modules are no [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/architecture/configuration",
- "title": "Preface",
- "location": [
- "Advanced Guide",
- "Configuration File"
- ]
- },
- {
- "content": "Overall Tasks Storage Structure\nAll tasks created in DolphinScheduler are saved in the t_ds_process_definition table.\nThe following shows the 't_ds_process_definition' table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not online, 1 online\n\n\n5\nprojec [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/architecture/task-structure",
- "title": "Overall Tasks Storage Structure",
- "location": [
- "Advanced Guide",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker load balancing algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\nWeighted random (random)\nSmoothing polling (roundrobin)\nLinear load (lowerweight)\nThe default configuration is the linear [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/architecture/load-balance",
- "title": "Load Balance",
- "location": [
- "Advanced Guide",
- "Load Balance"
- ]
- },
- {
- "content": "SkyWalking Agent\nThe dolphinscheduler-skywalking module provides SkyWalking monitor agent for the Dolphinscheduler project.\nThis document describes how to enable SkyWalking 8.4+ support with this module (recommended to use SkyWalking 8.5.0).\nInstallation\nThe following configuration is used to enable SkyWalking agent.\nThrough environment variable configuration (for Docker Compose)\nModify SkyWalking environment variables in docker/docker-swarm/config.env.sh:\nSKYWALKI [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/guide/observability/skywalking-agent",
- "title": "SkyWalking Agent",
- "location": [
- "Observability",
- "SkyWalking-Agent"
- ]
- },
- {
- "content": "Change Log\nDownload:\nhttps://dolphinscheduler.apache.org/en-us/download/download.html\nFeatures\n[#6470] Add Python API for DolphinScheduler\n[#7460] Wechat alert support send to group chat\nImprovements\n[#7529] Init DB schema from the full sql file\nBug Fixes\n[#7661] Fix the memory leak of logger in worker\n[#7750]Compatible historical version data source connection information\n[#7748] Remove init script when gaining a new DB connection\n[#7705] Because of memory li [...]
- "version": "2.0.2",
- "time": "2022-1-5",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "About DolphinScheduler\nApache DolphinScheduler is a cloud-native visual Big Data workflow scheduler system, committed to “solving complex big-data task dependencies and triggering relationships in data OPS orchestration so that various types of big data tasks can be used out of the box”.\nHigh Reliability\n\nDecentralized multi-master and multi-worker, HA is supported by itself, overload processing\n\nUser-Friendly\n\nAll process definition operations are visualized, Vis [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/About_DolphinScheduler/About_DolphinScheduler",
- "title": "About DolphinScheduler",
- "location": [
- "About",
- "Introduction"
- ]
- },
- {
- "content": "Hardware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\nLinux Operating System Version Requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbuntu L [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/installation/hardware",
- "title": "Hardware Environment",
- "location": [
- "About",
- "Hardware Environment"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\nGlossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the\nform of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until\nthere are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/architecture/designplus",
- "title": "System Architecture Design",
- "location": [
- "About",
- "Glossary"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password: admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate a worker group\n\n \n \n \n\nCreate environment\n\n \n \n \n\n\nCreate a token\n\n \n \n\n\nLogin with regular users\n\n\n\nClick on the user name in [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/quick-start",
- "title": "Quick Start",
- "location": [
- "Quick Start",
- "Quick Start"
- ]
- },
- {
- "content": "QuickStart in Docker\nPrerequisites\n\nDocker 1.13.1+\nDocker Compose 1.11.0+\n\nHow to Use this Docker Image\nHere're 3 ways to quickly install DolphinScheduler\nThe First Way: Start a DolphinScheduler by Docker Compose (Recommended)\nIn this way, you need to install docker-compose as a prerequisite, please install it yourself according to the rich docker-compose installation guidance on the Internet\nFor Windows 7-10, you can install Docker Toolbox. For Windows 10 64-bi [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/installation/docker",
- "title": "QuickStart in Docker",
- "location": [
- "Quick Start",
- "Docker Deployment"
- ]
- },
- {
- "content": "Standalone\nStandalone only for quick look for DolphinScheduler.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in production, we recommended you follow cluster deployment or kubernetes\n\nNote: Standalone only recommends the use [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/installation/standalone",
- "title": "Standalone",
- "location": [
- "Installation",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Pseudo-Cluster Deployment\nThe purpose of pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, api server, and logger server are all on the same machine.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cl [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/installation/pseudo-cluster",
- "title": "Pseudo-Cluster Deployment",
- "location": [
- "Installation",
- "Pseudo Cluster Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\nCluster deployment is to deploy the DolphinScheduler on multiple machines for running a large number of tasks in production.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in production, we recommended you foll [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/installation/cluster",
- "title": "Cluster Deployment",
- "location": [
- "Installation",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Kubernetes\nKubernetes deployment is deploy DolphinScheduler in a Kubernetes cluster, which can schedule a large number of tasks and can be used in production.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in produc [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/installation/kubernetes",
- "title": "QuickStart in Kubernetes",
- "location": [
- "Installation",
- "Kubernetes Deployment"
- ]
- },
- {
- "content": "Workflow Overview\nThe home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.\n\n\n",
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/homepage",
- "title": "Workflow Overview",
- "location": [
- "Introduction to Functions",
- "Workflow Overview"
- ]
- },
- {
- "content": "Project\nCreate Project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Submit" to create a new project.\n\n \n\n\n\nProject Home\n\n\nClick the project name link on the project management page to enter the project home page, as shown in the figure below, the project home page contains the task status statistics, process status [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/project/project-list",
- "title": "Project",
- "location": [
- "Introduction to Functions",
- "Project",
- "Project List"
- ]
- },
- {
- "content": "Workflow Definition\nCreate workflow definition\n\n\nClick Project Management -> Workflow -> Workflow Definition to enter the workflow definition page, and click the "Create Workflow" button to enter the workflow DAG edit page, as shown in the following figure:\n\n \n\n\n\nDrag in the toolbar Add a Shell task to the drawing board, as shown in the figure below:\n\n\n\nAdd parameter settings for this shell task:\n\n\n\nFill in the "Node Name", [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/project/workflow-definition",
- "title": "Workflow Definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Definition"
- ]
- },
- {
- "content": "Workflow Instance\nView Workflow Instance\n\nClick Project Management -> Workflow -> Workflow Instance to enter the Workflow Instance page, as shown in the figure below: \n \n \n\nClick the workflow name to enter the DAG view page to view the task execution status, as shown in the figure below.\n \n\n\n\nView Task Log\n\nEnter the workflow instance page, click the workflow name, enter the DAG view page, double-click the task node, as shown in the following [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/project/workflow-instance",
- "title": "Workflow Instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Instance"
- ]
- },
- {
- "content": "Task Instance\n\n\nClick Project Management -> Workflow -> Task Instance to enter the task instance page, as shown in the figure below, click the name of the workflow instance, you can jump to the workflow instance DAG chart to view the task status.\n \n \n \n\n\nView log：Click the "view log" button in the operation column to view the log of task execution.\n \n \n \n\n\n",
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/project/task-instance",
- "title": "Task Instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Instance"
- ]
- },
- {
- "content": "Shell\nOverview\nShell task, used to create a shell-type task and execute a series of shell scripts. When the worker executed,\na temporary shell script is generated, and the Linux user with the same name as the tenant executes the script.\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/task/shell",
- "title": "Shell",
- "location": [
- "Introduction to Functions",
- "Task",
- "Shell"
- ]
- },
- {
- "content": "SubProcess\n\nThe sub-process node is to execute a certain external workflow definition as a task node.\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\n\n \n \n\nNode name: The node name in a workflow definition is unique\nRun flag: identify whether this node can be scheduled normally\nDescriptive information: describe the function of the node\nTimeout alarm: Check the timeout alarm and timeout failure. When the task ex [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/task/sub-process",
- "title": "SubProcess",
- "location": [
- "Introduction to Functions",
- "Task",
- "SubProcess"
- ]
- },
- {
- "content": "Dependent\n\nDependent nodes are dependency check nodes. For example, process A depends on the successful execution of process B yesterday, and the dependent node will check whether process B has a successful execution yesterday.\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\n\n \n \n\nThe dependent node provides a logical judgment function, such as checking whether the B process was successful yesterday, or whether th [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/task/dependent",
- "title": "Dependent",
- "location": [
- "Introduction to Functions",
- "Task",
- "Dependent"
- ]
- },
- {
- "content": "Stored Procedure\n\nAccording to the selected data source, execute the stored procedure.\n\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nData source: The data source type of the stored procedure supports MySQL and POSTGRESQL, select the corresponding data source\nMethod: is the method name of the stored procedure\nCustom parameters: The custom parameter types of the stored procedure support IN and OUT, and the d [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/task/stored-procedure",
- "title": "Stored Procedure",
- "location": [
- "Introduction to Functions",
- "Task",
- "Stored Procedure"
- ]
- },
- {
- "content": "SQL\nOverview\nSQL task, used to connect to database and execute SQL.\nCreate Data Source\nRefer to Data Source\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the drawing board.\n\nTask Parameter\n\nData source: select the corresponding data source\nsql type: supports query and non-query. The query is a select type query, which is returned [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/task/sql",
- "title": "SQL",
- "location": [
- "Introduction to Functions",
- "Task",
- "SQL"
- ]
- },
- {
- "content": "Spark\nOverview\nSpark task type for executing Spark programs. For Spark nodes, the worker submits the task by using the spark command spark submit. See spark-submit for more details.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The node name in a workflow definition is u [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/task/spark",
- "title": "Spark",
- "location": [
- "Introduction to Functions",
- "Task",
- "Spark"
- ]
- },
- {
- "content": "MapReduce\nOverview\n\nMapReduce(MR) task type for executing MapReduce programs. For MapReduce nodes, the worker submits the task by using the Hadoop command hadoop jar. See Hadoop Command Manual for more details.\n\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The node name in a work [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/task/map-reduce",
- "title": "MapReduce",
- "location": [
- "Introduction to Functions",
- "Task",
- "MapReduce"
- ]
- },
- {
- "content": "Python\n\nUsing python nodes, you can directly execute python scripts. For python nodes, workers will use python ** to submit tasks.\n\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nScript: Python program developed by the user\nEnvironment Name: Specific which Python interpreter would be use and run Script. If you want to use Python virtualenv, you should create multiply environments for each virtualenv.\nResourc [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/task/python",
- "title": "Python",
- "location": [
- "Introduction to Functions",
- "Task",
- "Python"
- ]
- },
- {
- "content": "Flink\nOverview\nFlink task type for executing Flink programs. For Flink nodes, the worker submits the task by using the flink command flink run. See flink cli for more details.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The node name in a workflow definition is unique. [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/task/flink",
- "title": "Flink",
- "location": [
- "Introduction to Functions",
- "Task",
- "Flink"
- ]
- },
- {
- "content": "HTTP\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.\nDescriptive information: describe the function of the node.\nTask priority: When the number of worker threads is insufficient, they are executed in order from [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/task/http",
- "title": "HTTP",
- "location": [
- "Introduction to Functions",
- "Task",
- "HTTP"
- ]
- },
- {
- "content": "DataX\n\n\nDrag in the toolbarTask node into the drawing board\n\n \n\n\n\nCustom template: When you turn on the custom template switch, you can customize the content of the json configuration file of the datax node (applicable when the control configuration does not meet the requirements)\n\n\nData source: select the data source to extract the data\n\n\nsql statement: the sql statement used to extract data from the target database, the sql query column name is automatica [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/task/datax",
- "title": "DataX",
- "location": [
- "Introduction to Functions",
- "Task",
- "DataX"
- ]
- },
- {
- "content": "Pigeon\nPigeon is general websocket service tracking task for DolphinScheduler. It can trigger, check status, get log from remote websocket service.\nCreate\nDrag in the toolbarThe task node to the drawing board to create a new Conditions task\nParameter\n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.\nDescriptive i [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/task/pigeon",
- "title": "Pigeon",
- "location": [
- "Introduction to Functions",
- "Task",
- "Pigeon"
- ]
- },
- {
- "content": "Conditions\nConditions is a condition node, determining which downstream task should be run based on the condition set to it. For now, the Conditions support multiple upstream tasks, but only two downstream tasks. When the number of upstream tasks exceeds one, complex upstream dependencies can be achieved through and and or operators.\nCreate\nDrag in the toolbarThe task node to the drawing board to create a new Conditions task, as shown in the figure below:\n \n \n \ [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/task/conditions",
- "title": "Conditions",
- "location": [
- "Introduction to Functions",
- "Task",
- "Conditions"
- ]
- },
- {
- "content": "Switch\nSwitch is a conditional judgment node, which branch should be executes according to the value of global variable and the expression result written by the user.\nCreate\nDrag the in the tool bar to create task. Note After the switch task is created, you must configure it downstream to make parameter Branch flow work.\nParameter\n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it d [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/task/switch",
- "title": "Switch",
- "location": [
- "Introduction to Functions",
- "Task",
- "Switch"
- ]
- },
- {
- "content": "Built-in Parameter\nBasic Built-in Parameter\n\n variabledeclaration methodmeaning\n \n system.biz.date\n ${system.biz.date}\n The day before the scheduled time of the daily scheduling instance, the format is yyyyMMdd\n \n \n system.biz.curdate\n ${system.biz.curdate}\n The timing time of the daily scheduling instance, the format is yyyyMMdd\n \n \n system.datetime\n ${system.datetime}\n The [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/parameter/built-in",
- "title": "Built-in Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Built-in Parameter"
- ]
- },
- {
- "content": "Global Parameter\nScope\nThe parameters configured on the workflow definition dialog, the whole workflow is it's scope.\nUsage\nthe approach to set global parameters is, after defining the workflow, click the 'save' button, then click the '+' button below the 'Set global':\n\n \n \n\n \n \nThe global_bizdate parameter defined here can be referenced by local parameters of any other task node, and the value of global_bizdate is set to the figure obtained by referencing [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/parameter/global",
- "title": "Global Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Global Parameter"
- ]
- },
- {
- "content": "Local Parameter\nScope\nParameters configured on the task definition dialog, the scope of this parameter only for this task, but if you configured follow Parameter Context, it could passed follow task.\nUsage\nThe approach to set local parameters is, double-click on any node while defining the workflow and click the '+' button next to the 'Custom Parameters':\n\n \n\n\n \n\nIf you want to call the built-in parameter in the local parameters, fill in the value corre [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/parameter/local",
- "title": "Local Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Local Parameter"
- ]
- },
- {
- "content": "Parameter Context\nDolphinScheduler provides the ability to refer to each other between parameters, including: local parameters refer to global parameters, and upstream and downstream parameter transfer. Because of the existence of references, it involves the priority of parameters when the parameter names are the same. see also Parameter Priority\nLocal Task Use Global Parameter\nThe premise of local tasks referencing global parameters is that you have already defined Gl [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/parameter/context",
- "title": "Parameter Context",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Context"
- ]
- },
- {
- "content": "Parameter Priority\nDolphinScheduler definition of parameter values involved in may come from three types:\n\nGlobal Parameter: Parameters defined when the workflow saves page definitions\nParameter Context: Parameters passed by upstream nodes\nLocal Parameter：The node's own parameters, which is the parameters defined by the user in [Custom Parameters]. The user can define the value of this part of the parameters at the time of workflow definition.\n\nBecause the value [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/parameter/priority",
- "title": "Parameter Priority",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Priority"
- ]
- },
- {
- "content": "Data Source\nData source center supports MySQL, POSTGRESQL, HIVE/IMPALA, SPARK, CLICKHOUSE, ORACLE, SQLSERVER and other data sources\n\nClick "Data Source Center -> Create Data Source" to create different types of data sources according to requirements.\nClick "Test Connection" to test whether the data source can be successfully connected.\n\n",
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/datasource/introduction",
- "title": "Data Source",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Introduction"
- ]
- },
- {
- "content": "MySQL\n\nData source: select MYSQL\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP hostname: enter the IP to connect to MySQL\nPort: Enter the port to connect to MySQL\nUsername: Set the username for connecting to MySQL\nPassword: Set the password for connecting to MySQL\nDatabase name: Enter the name of the database connected to MySQL\nJdbc connection parameters: parameter settings for MySQL connection, filled [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/datasource/mysql",
- "title": "MySQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "MySQL"
- ]
- },
- {
- "content": "PostgreSQL\n\nData source: select POSTGRESQL\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Host Name: Enter the IP to connect to POSTGRESQL\nPort: Enter the port to connect to POSTGRESQL\nUsername: Set the username for connecting to POSTGRESQL\nPassword: Set the password for connecting to POSTGRESQL\nDatabase name: Enter the name of the database connected to POSTGRESQL\nJdbc connection parameters: parameter s [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/datasource/postgresql",
- "title": "PostgreSQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "PostgreSQL"
- ]
- },
- {
- "content": "HIVE\nUse HiveServer2\n \n \n \n\nData source: select HIVE\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Host Name: Enter the IP connected to HIVE\nPort: Enter the port connected to HIVE\nUsername: Set the username for connecting to HIVE\nPassword: Set the password for connecting to HIVE\nDatabase name: Enter the name of the database connected to HIVE\nJdbc connection parameters: parameter settings for HI [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/datasource/hive",
- "title": "HIVE",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "HIVE"
- ]
- },
- {
- "content": "Spark\n\n \n \n\nData source: select Spark\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Hostname: Enter the IP connected to Spark\nPort: Enter the port connected to Spark\nUsername: Set the username for connecting to Spark\nPassword: Set the password for connecting to Spark\nDatabase name: Enter the name of the database connected to Spark\nJdbc connection parameters: parameter settings for Spark connection [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/datasource/spark",
- "title": "Spark",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Spark"
- ]
- },
- {
- "content": "Alert Component User Guide\nHow to Create Alert Plugins and Alert Groups\nIn version 2.0.3, users need to create alert instances, and then associate them with alert groups, and an alert group can use multiple alert instances, and we will notify them one by one.\nFirst of all, you need to go to the Security Center, select Alarm Group Management, then click Alarm Instance Management on the left, then create an alarm instance, then select the corresponding alarm plug-in and [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/alert/alert_plugin_user_guide",
- "title": "Alert Component User Guide",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Alert Component User Guide "
- ]
- },
- {
- "content": "Enterprise WeChat\nHow to Create Enterprise WeChat Alert\nIf you need to use Enterprise WeChat to alert, please create an alarm Instance in warning instance manage, and then choose the wechat plugin. The configuration example of enterprise WeChat is as follows\n\nWhere send type corresponds to app and appchat respectively:\nAPP: https://work.weixin.qq.com/api/doc/90000/90135/90236\nAPPCHAT: https://work.weixin.qq.com/api/doc/90000/90135/90248\nuser.send.msg corresponds to [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/alert/enterprise-wechat",
- "title": "Enterprise WeChat",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Enterprise Wechat"
- ]
- },
- {
- "content": "Resource Center\nIf you want to use the resource upload function, you can select the local file directory for a single machine(this operation does not need to deploy Hadoop). Or you can also upload to a Hadoop or MinIO cluster, at this time, you need to have Hadoop (2.6+) or MinIO and other related environments\n\nNote:\n\nIf the resource upload function is used, the deployment user in installation and deployment must to have operation authority\nIf you using Hadoop clust [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/resource",
- "title": "Resource Center",
- "location": [
- "Introduction to Functions",
- "Resource"
- ]
- },
- {
- "content": "Monitor\nService Management\n\nService management is mainly to monitor and display the health status and basic information of each service in the system\n\nMonitor Master Server\n\nMainly related to master information.\n\n\n \n \nMonitor Worker Server\n\nMainly related to worker information.\n\n\n \n \nMonitor ZooKeeper\n\nMainly related configuration information of each worker and master in ZooKeeper.\n\n\n \n \nMonitor DB\n\nMainly the health of the DB\n\n\n \n [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/monitor",
- "title": "Monitor",
- "location": [
- "Introduction to Functions",
- "Monitor"
- ]
- },
- {
- "content": "Security\n\nOnly the administrator account in the security center has the authority to operate. It has functions such as queue management, tenant management, user management, alarm group management, worker group management, token management, etc. In the user management module, resources, data sources, projects, etc. Authorization\nAdministrator login, default user name and password: admin/dolphinscheduler123\n\nCreate Queue\n\nQueue is used when the "queue" para [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/security",
- "title": "Security",
- "location": [
- "Introduction to Functions",
- "Security"
- ]
- },
- {
- "content": "Open API\nBackground\nGenerally, projects and processes are created through pages, but integration with third-party systems requires API calls to manage projects and workflows.\nThe Operation Steps of DS API Calls\nCreate a Token\n\nLog in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.\n\n\n \n \n\nSelect the "Expiration time" (Token validity), select [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/open-api",
- "title": "Open API",
- "location": [
- "Introduction to Functions",
- "Open API"
- ]
- },
- {
- "content": "Flink Calls Operating steps\nCreate a Queue\n\nLog in to the scheduling system, click "Security", then click "Queue manage" on the left, and click "Create queue" to create a queue.\nFill in the name and value of the queue, and click "Submit"\n\n\n \n \nCreate a Tenant\n1. The tenant corresponds to a Linux user, which the user worker uses to submit jobs. If Linux OS environment does not have this user, the worker will create this u [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/flink-call",
- "title": "Flink Calls Operating steps",
- "location": [
- "Introduction to Functions",
- "Flink"
- ]
- },
- {
- "content": "DolphinScheduler Upgrade Documentation\nBack Up Previous Version's Files and Database\nStop All Services of DolphinScheduler\nsh ./script/stop-all.sh\nDownload the New Version's Installation Package\n\nDownload the latest version of the installation packages.\nThe following upgrade operations need to be performed in the new version's directory.\n\nDatabase Upgrade\n\n\nModify the following properties in conf/config/install_config.conf.\n\n\nIf you use MySQL as the databas [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/upgrade",
- "title": "DolphinScheduler Upgrade Documentation",
- "location": [
- "Introduction to Functions",
- "Upgrade"
- ]
- },
- {
- "content": "\nDolphinScheduler Expansion and Reduction\nExpansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine where the expansion master or worker node is located has already installed the scheduled service, skip to [1.4 Modify configuration] Edit the configuration file [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Introduction to Functions",
- "Expansion and Reduction"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\nSystem Structure\nSystem Architecture Diagram\n\n \n \n System architecture diagram\n \n\nStart Process Activity Diagram\n\n \n \n Start process activity diagram\n \n\nArchitecture Description\n\n\nMasterServer\nMasterServer adopts a distributed and centerless design concept. MasterServer is [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/architecture/design",
- "title": "System Architecture Design",
- "location": [
- "Advanced Guide",
- "Architecture Design"
- ]
- },
- {
- "content": "Dolphin Scheduler 2.0.3 MetaData\n\nDolphin Scheduler 2.0 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nq [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/architecture/metadata",
- "title": "Dolphin Scheduler 2.0.3 MetaData",
- "location": [
- "Advanced Guide",
- "Metadata"
- ]
- },
- {
- "content": "\nConfiguration\nPreface\nThis document explains the DolphinScheduler application configurations according to DolphinScheduler-1.3.x versions.\nDirectory Structure\nCurrently, all the configuration files are under [conf ] directory. Please check the following simplified DolphinScheduler installation directories to have a direct view about the position [conf] directory in and configuration files inside. This document only describes DolphinScheduler configurations and other [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/architecture/configuration",
- "title": "Configuration",
- "location": [
- "Advanced Guide",
- "Configuration File"
- ]
- },
- {
- "content": "Task Structure\nOverall Tasks Storage Structure\nAll tasks created in DolphinScheduler are saved in the t_ds_process_definition table.\nThe following shows the 't_ds_process_definition' table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not online, 1 onlin [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/architecture/task-structure",
- "title": "Task Structure",
- "location": [
- "Advanced Guide",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker Load Balancing Algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\nWeighted random (random)\nSmoothing polling (roundrobin)\nLinear load (lowerweight)\nThe default configuration is the linear [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/architecture/load-balance",
- "title": "Load Balance",
- "location": [
- "Advanced Guide",
- "Load Balance"
- ]
- },
- {
- "content": "Cache\nPurpose\nDue to the master-server scheduling process, there will be a large number of database read operations, such as tenant, user, processDefinition, etc. On the one hand, it will put a lot of pressure on the DB, and on the other hand, it will slow down the entire core scheduling process.\nConsidering that this part of the business data is a scenario where more reads and less writes are performed, a cache module is introduced to reduce the DB read pressure and s [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/architecture/cache",
- "title": "Cache",
- "location": [
- "Advanced Guide",
- "Cache"
- ]
- },
- {
- "content": "SkyWalking Agent\nThe dolphinscheduler-skywalking module provides SkyWalking monitor agent for the Dolphinscheduler project.\nThis document describes how to enable SkyWalking 8.4+ support with this module (recommended to use SkyWalking 8.5.0).\nInstallation\nThe following configuration is used to enable SkyWalking agent.\nThrough Environment Variable Configuration (for Docker Compose)\nModify SkyWalking environment variables in docker/docker-swarm/config.env.sh:\nSKYWALKI [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/guide/observability/skywalking-agent",
- "title": "SkyWalking Agent",
- "location": [
- "Observability",
- "SkyWalking-Agent"
- ]
- },
- {
- "content": "Change Log\nDownload\nhttps://dolphinscheduler.apache.org/en-us/download/download.html\nFeatures\n[#7804] DingTalk alert plugin adds signature.\n[#7857] Problems caused by modifying the administrator user information.\n[#7675] Supports whether SQL is placed in one session\n[#7571] Add dependent tips about success or failure\nImprovements\n[#7990] zkRoot in conf/config/install_ config does not take effect\n[#7713] The data source password in the log is not encrypted\nBug F [...]
- "version": "2.0.3",
- "time": "2022-1-20",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "About DolphinScheduler\nApache DolphinScheduler is a cloud-native visual Big Data workflow scheduler system, committed to “solving complex big-data task dependencies and triggering relationships in data OPS orchestration so that various types of big data tasks can be used out of the box”.\nHigh Reliability\n\nDecentralized multi-master and multi-worker, HA is supported by itself, overload processing\n\nUser-Friendly\n\nAll process definition operations are visualized, Vis [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/About_DolphinScheduler/About_DolphinScheduler",
- "title": "About DolphinScheduler",
- "location": [
- "About",
- "Introduction"
- ]
- },
- {
- "content": "Hardware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux Operating System Version Requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/installation/hardware",
- "title": "Hardware Environment",
- "location": [
- "About",
- "Hardware Environment"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the\nform of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until\nthere are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/architecture/designplus",
- "title": "System Architecture Design",
- "location": [
- "About",
- "Glossary"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password: admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate a worker group\n\n \n \n \n\nCreate environment\n\n \n \n \n\n\nCreate a token\n\n \n \n\n\nLogin with regular users\n\n\n\nClick on the user name in [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/quick-start",
- "title": "Quick Start",
- "location": [
- "Quick Start",
- "Quick Start"
- ]
- },
- {
- "content": "QuickStart in Docker\nPrerequisites\n\nDocker 1.13.1+\nDocker Compose 1.11.0+\n\nHow to use this Docker image\nHere're 3 ways to quickly install DolphinScheduler\nThe First Way: Start a DolphinScheduler by docker-compose (recommended)\nIn this way, you need to install docker-compose as a prerequisite, please install it yourself according to the rich docker-compose installation guidance on the Internet\nFor Windows 7-10, you can install Docker Toolbox. For Windows 10 64-bi [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/installation/docker",
- "title": "QuickStart in Docker",
- "location": [
- "Quick Start",
- "Docker Deployment"
- ]
- },
- {
- "content": "Standalone\nStandalone only for quick look for DolphinScheduler.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in production, we recommended you follow cluster deployment or kubernetes\n\nNote: Standalone only recommends the use [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/installation/standalone",
- "title": "Standalone",
- "location": [
- "Installation",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Pseudo-Cluster Deployment\nThe purpose of pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, api server, and logger server are all on the same machine.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cl [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/installation/pseudo-cluster",
- "title": "Pseudo-Cluster Deployment",
- "location": [
- "Installation",
- "Pseudo Cluster Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\nCluster deployment is to deploy the DolphinScheduler on multiple machines for running a large number of tasks in production.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in production, we recommended you foll [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/installation/cluster",
- "title": "Cluster Deployment",
- "location": [
- "Installation",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Kubernetes\nKubernetes deployment is deploy DolphinScheduler in a Kubernetes cluster, which can schedule a large number of tasks and can be used in production.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in produc [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/installation/kubernetes",
- "title": "QuickStart in Kubernetes",
- "location": [
- "Installation",
- "Kubernetes Deployment"
- ]
- },
- {
- "content": "Workflow Overview\nThe home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.\n\n\n",
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/homepage",
- "title": "Workflow Overview",
- "location": [
- "Introduction to Functions",
- "Workflow Overview"
- ]
- },
- {
- "content": "Project\nCreate project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Submit" to create a new project.\n\n \n\n\n\nProject home\n\n\nClick the project name link on the project management page to enter the project home page, as shown in the figure below, the project home page contains the task status statistics, process status [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/project/project-list",
- "title": "Project",
- "location": [
- "Introduction to Functions",
- "Project",
- "Project List"
- ]
- },
- {
- "content": "Workflow definition\nCreate workflow definition\n\n\nClick Project Management -> Workflow -> Workflow Definition to enter the workflow definition page, and click the "Create Workflow" button to enter the workflow DAG edit page, as shown in the following figure:\n\n \n\n\n\nDrag in the toolbar Add a Shell task to the drawing board, as shown in the figure below:\n\n\n\nAdd parameter settings for this shell task:\n\n\n\nFill in the "Node Name", [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/project/workflow-definition",
- "title": "Workflow definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Definition"
- ]
- },
- {
- "content": "Workflow instance\nView workflow instance\n\nClick Project Management -> Workflow -> Workflow Instance to enter the Workflow Instance page, as shown in the figure below: \n \n \n\nClick the workflow name to enter the DAG view page to view the task execution status, as shown in the figure below.\n \n\n\n\nView task log\n\nEnter the workflow instance page, click the workflow name, enter the DAG view page, double-click the task node, as shown in the following [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/project/workflow-instance",
- "title": "Workflow instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Instance"
- ]
- },
- {
- "content": "Task instance\n\n\nClick Project Management -> Workflow -> Task Instance to enter the task instance page, as shown in the figure below, click the name of the workflow instance, you can jump to the workflow instance DAG chart to view the task status.\n \n \n \n\n\nView log：Click the "view log" button in the operation column to view the log of task execution.\n \n \n \n\n\n",
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/project/task-instance",
- "title": "Task instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Instance"
- ]
- },
- {
- "content": "Shell\nOverview\nShell task, used to create a shell-type task and execute a series of shell scripts. When the worker executed,\na temporary shell script is generated, and the Linux user with the same name as the tenant executes the script.\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/task/shell",
- "title": "Shell",
- "location": [
- "Introduction to Functions",
- "Task",
- "Shell"
- ]
- },
- {
- "content": "SubProcess\n\nThe sub-process node is to execute a certain external workflow definition as a task node.\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\n\n \n \n\nNode name: The node name in a workflow definition is unique\nRun flag: identify whether this node can be scheduled normally\nDescriptive information: describe the function of the node\nTimeout alarm: Check the timeout alarm and timeout failure. When the task ex [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/task/sub-process",
- "title": "SubProcess",
- "location": [
- "Introduction to Functions",
- "Task",
- "SubProcess"
- ]
- },
- {
- "content": "DEPENDENT\n\nDependent nodes are dependency check nodes. For example, process A depends on the successful execution of process B yesterday, and the dependent node will check whether process B has a successful execution yesterday.\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\n\n \n \n\nThe dependent node provides a logical judgment function, such as checking whether the B process was successful yesterday, or whether th [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/task/dependent",
- "title": "DEPENDENT",
- "location": [
- "Introduction to Functions",
- "Task",
- "Dependent"
- ]
- },
- {
- "content": "Stored Procedure\n\nAccording to the selected data source, execute the stored procedure.\n\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nData source: The data source type of the stored procedure supports MySQL and POSTGRESQL, select the corresponding data source\nMethod: is the method name of the stored procedure\nCustom parameters: The custom parameter types of the stored procedure support IN and OUT, and the d [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/task/stored-procedure",
- "title": "Stored Procedure",
- "location": [
- "Introduction to Functions",
- "Task",
- "Stored Procedure"
- ]
- },
- {
- "content": "SQL\nOverview\nSQL task, used to connect to database and execute SQL.\ncreate data source\nRefer to Data Source\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the drawing board.\n\nTask Parameter\n\nData source: select the corresponding data source\nsql type: supports query and non-query. The query is a select type query, which is returned [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/task/sql",
- "title": "SQL",
- "location": [
- "Introduction to Functions",
- "Task",
- "SQL"
- ]
- },
- {
- "content": "Spark\nOverview\nSpark task type for executing Spark programs. For Spark nodes, the worker submits the task by using the spark command spark submit. See spark-submit for more details.\nCreate task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The node name in a workflow definition is u [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/task/spark",
- "title": "Spark",
- "location": [
- "Introduction to Functions",
- "Task",
- "Spark"
- ]
- },
- {
- "content": "MapReduce\nOverview\n\nMapReduce(MR) task type for executing MapReduce programs. For MapReduce nodes, the worker submits the task by using the Hadoop command hadoop jar. See Hadoop Command Manual for more details.\n\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The node name in a work [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/task/map-reduce",
- "title": "MapReduce",
- "location": [
- "Introduction to Functions",
- "Task",
- "MapReduce"
- ]
- },
- {
- "content": "Python Node\nOverview\nUse Python Task to create a python-type task and execute python scripts. When the worker executes Python Task,\nit will generate a temporary python script, and executes the script by the Linux user with the same name as the tenant.\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameter\n\nNode n [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/task/python",
- "title": "Python Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Python"
- ]
- },
- {
- "content": "Flink\nOverview\nFlink task type for executing Flink programs. For Flink nodes, the worker submits the task by using the flink command flink run. See flink cli for more details.\nCreate task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The node name in a workflow definition is unique. [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/task/flink",
- "title": "Flink",
- "location": [
- "Introduction to Functions",
- "Task",
- "Flink"
- ]
- },
- {
- "content": "HTTP\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.\nDescriptive information: describe the function of the node.\nTask priority: When the number of worker threads is insufficient, they are executed in order from [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/task/http",
- "title": "HTTP",
- "location": [
- "Introduction to Functions",
- "Task",
- "HTTP"
- ]
- },
- {
- "content": "DATAX\n\n\nDrag in the toolbarTask node into the drawing board\n\n \n\n\n\nCustom template: When you turn on the custom template switch, you can customize the content of the json configuration file of the datax node (applicable when the control configuration does not meet the requirements)\n\n\nData source: select the data source to extract the data\n\n\nsql statement: the sql statement used to extract data from the target database, the sql query column name is automatica [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/task/datax",
- "title": "DATAX",
- "location": [
- "Introduction to Functions",
- "Task",
- "DataX"
- ]
- },
- {
- "content": "Pigeon\nPigeon is general websocket service tracking task for DolphinScheduler. It can trigger, check status, get log from remote websocket service.\nCreate\nDrag in the toolbarThe task node to the drawing board to create a new Conditions task\nParameter\n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.\nDescriptive i [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/task/pigeon",
- "title": "Pigeon",
- "location": [
- "Introduction to Functions",
- "Task",
- "Pigeon"
- ]
- },
- {
- "content": "Conditions\nConditions is a condition node, determining which downstream task should be run based on the condition set to it. For now, the Conditions support multiple upstream tasks, but only two downstream tasks. When the number of upstream tasks exceeds one, complex upstream dependencies can be achieved through and and or operators.\nCreate\nDrag in the toolbarThe task node to the drawing board to create a new Conditions task, as shown in the figure below:\n \n \n \ [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/task/conditions",
- "title": "Conditions",
- "location": [
- "Introduction to Functions",
- "Task",
- "Conditions"
- ]
- },
- {
- "content": "Switch\nSwitch is a conditional judgment node, which branch should be executes according to the value of global variable and the expression result written by the user.\nCreate\nDrag the in the tool bar to create task. Note After the switch task is created, you must configure it downstream to make parameter Branch flow work.\nParameter\n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it d [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/task/switch",
- "title": "Switch",
- "location": [
- "Introduction to Functions",
- "Task",
- "Switch"
- ]
- },
- {
- "content": "Built-in Parameter\nBasic Built-in Parameter\n\n variabledeclaration methodmeaning\n \n system.biz.date\n ${system.biz.date}\n The day before the scheduled time of the daily scheduling instance, the format is yyyyMMdd\n \n \n system.biz.curdate\n ${system.biz.curdate}\n The timing time of the daily scheduling instance, the format is yyyyMMdd\n \n \n system.datetime\n ${system.datetime}\n The [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/parameter/built-in",
- "title": "Built-in Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Built-in Parameter"
- ]
- },
- {
- "content": "Global Parameter\nScope\nThe parameters configured on the workflow definition dialog, the whole workflow is it's scope.\nUsage\nthe approach to set global parameters is, after defining the workflow, click the 'save' button, then click the '+' button below the 'Set global':\n\n \n \n\n \n \nThe global_bizdate parameter defined here can be referenced by local parameters of any other task node, and the value of global_bizdate is set to the figure obtained by referencing [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/parameter/global",
- "title": "Global Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Global Parameter"
- ]
- },
- {
- "content": "Local Parameter\nScope\nParameters configured on the task definition dialog, the scope of this parameter only for this task, but if you configured follow Parameter Context, it could passed follow task.\nUsage\nThe approach to set local parameters is, double-click on any node while defining the workflow and click the '+' button next to the 'Custom Parameters':\n\n \n\n\n \n\nIf you want to call the built-in parameter in the local parameters, fill in the value corre [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/parameter/local",
- "title": "Local Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Local Parameter"
- ]
- },
- {
- "content": "Parameter Context\nDolphinScheduler provides the ability to refer to each other between parameters, including: local parameters refer to global parameters, and upstream and downstream parameter transfer. Because of the existence of references, it involves the priority of parameters when the parameter names are the same. see also Parameter Priority\nLocal task use global parameter\nThe premise of local tasks referencing global parameters is that you have already defined Gl [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/parameter/context",
- "title": "Parameter Context",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Context"
- ]
- },
- {
- "content": "Parameter Priority\nDolphinScheduler definition of parameter values involved in may come from three types:\n\nGlobal Parameter: Parameters defined when the workflow saves page definitions\nParameter Context: Parameters passed by upstream nodes\nLocal Parameter：The node's own parameters, which is the parameters defined by the user in [Custom Parameters]. The user can define the value of this part of the parameters at the time of workflow definition.\n\nBecause the value [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/parameter/priority",
- "title": "Parameter Priority",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Priority"
- ]
- },
- {
- "content": "Data Source\nData source center supports MySQL, POSTGRESQL, HIVE/IMPALA, SPARK, CLICKHOUSE, ORACLE, SQLSERVER and other data sources\n\nClick "Data Source Center -> Create Data Source" to create different types of data sources according to requirements.\nClick "Test Connection" to test whether the data source can be successfully connected.\n\n",
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/datasource/introduction",
- "title": "Data Source",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Introduction"
- ]
- },
- {
- "content": "MySQL\n\nData source: select MYSQL\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP hostname: enter the IP to connect to MySQL\nPort: Enter the port to connect to MySQL\nUsername: Set the username for connecting to MySQL\nPassword: Set the password for connecting to MySQL\nDatabase name: Enter the name of the database connected to MySQL\nJdbc connection parameters: parameter settings for MySQL connection, filled [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/datasource/mysql",
- "title": "MySQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "MySQL"
- ]
- },
- {
- "content": "POSTGRESQL\n\nData source: select POSTGRESQL\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Host Name: Enter the IP to connect to POSTGRESQL\nPort: Enter the port to connect to POSTGRESQL\nUsername: Set the username for connecting to POSTGRESQL\nPassword: Set the password for connecting to POSTGRESQL\nDatabase name: Enter the name of the database connected to POSTGRESQL\nJdbc connection parameters: parameter s [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/datasource/postgresql",
- "title": "POSTGRESQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "PostgreSQL"
- ]
- },
- {
- "content": "HIVE\nUse HiveServer2\n \n \n \n\nData source: select HIVE\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Host Name: Enter the IP connected to HIVE\nPort: Enter the port connected to HIVE\nUsername: Set the username for connecting to HIVE\nPassword: Set the password for connecting to HIVE\nDatabase name: Enter the name of the database connected to HIVE\nJdbc connection parameters: parameter settings for HI [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/datasource/hive",
- "title": "HIVE",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "HIVE"
- ]
- },
- {
- "content": "Spark\n\n \n \n\nData source: select Spark\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Hostname: Enter the IP connected to Spark\nPort: Enter the port connected to Spark\nUsername: Set the username for connecting to Spark\nPassword: Set the password for connecting to Spark\nDatabase name: Enter the name of the database connected to Spark\nJdbc connection parameters: parameter settings for Spark connection [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/datasource/spark",
- "title": "Spark",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Spark"
- ]
- },
- {
- "content": "How to create alert plugins and alert groups\nIn version 2.0.5, users need to create alert instances, and then associate them with alert groups, and an alert group can use multiple alert instances, and we will notify them one by one.\nFirst of all, you need to go to the Security Center, select Alarm Group Management, then click Alarm Instance Management on the left, then create an alarm instance, then select the corresponding alarm plug-in and fill in the relevant alarm p [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/alert/alert_plugin_user_guide",
- "title": "How to create alert plugins and alert groups",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Alert Component User Guide "
- ]
- },
- {
- "content": "Enterprise WeChat\nIf you need to use Enterprise WeChat to alert, please create an alarm Instance in warning instance manage, and then choose the wechat plugin. The configuration example of enterprise WeChat is as follows\n\nWhere send type corresponds to app and appchat respectively:\nAPP: https://work.weixin.qq.com/api/doc/90000/90135/90236\nAPPCHAT: https://work.weixin.qq.com/api/doc/90000/90135/90248\nuser.send.msg corresponds to the content in the document. The varia [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/alert/enterprise-wechat",
- "title": "Enterprise WeChat",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Enterprise Wechat"
- ]
- },
- {
- "content": "DingTalk\nIf you need to use DingTalk for alerting, please create an alert instance in the alert instance management and select the DingTalk plugin. The configuration example of DingTalk is as follows:\n\nparameter configuration\n\nWebhook\n\nThe format is as follows: https://oapi.dingtalk.com/robot/send?access_token=XXXXXX\n\n\nKeyword\n\nCustom keywords for security settings\n\n\nSecret\n\nSignature of security settings\n\n\nMessageType\n\nSupport both text and markdown [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/alert/dingtalk",
- "title": "DingTalk",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Ding Talk"
- ]
- },
- {
- "content": "Resource Center\nIf you want to use the resource upload function, you can select the local file directory for a single machine(this operation does not need to deploy Hadoop). Or you can also upload to a Hadoop or MinIO cluster, at this time, you need to have Hadoop (2.6+) or MinIO and other related environments\n\nNote:\n\nIf the resource upload function is used, the deployment user in installation and deployment must to have operation authority\nIf you using Hadoop clust [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/resource",
- "title": "Resource Center",
- "location": [
- "Introduction to Functions",
- "Resource"
- ]
- },
- {
- "content": "Monitor\nService management\n\nService management is mainly to monitor and display the health status and basic information of each service in the system\n\nmaster monitoring\n\nMainly related to master information.\n\n\n \n \nworker monitoring\n\nMainly related to worker information.\n\n\n \n \nZookeeper monitoring\n\nMainly related configuration information of each worker and master in ZooKeeper.\n\n\n \n \nDB monitoring\n\nMainly the health of the DB\n\n\n \n \n [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/monitor",
- "title": "Monitor",
- "location": [
- "Introduction to Functions",
- "Monitor"
- ]
- },
- {
- "content": "Security\n\nOnly the administrator account in the security center has the authority to operate. It has functions such as queue management, tenant management, user management, alarm group management, worker group management, token management, etc. In the user management module, resources, data sources, projects, etc. Authorization\nAdministrator login, default user name and password: admin/dolphinscheduler123\n\nCreate queue\n\nQueue is used when the "queue" para [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/security",
- "title": "Security",
- "location": [
- "Introduction to Functions",
- "Security"
- ]
- },
- {
- "content": "Flink Calls Operating steps\nCreate a queue\n\nLog in to the scheduling system, click "Security", then click "Queue manage" on the left, and click "Create queue" to create a queue.\nFill in the name and value of the queue, and click "Submit"\n\n\n \n \nCreate a tenant\n1. The tenant corresponds to a Linux user, which the user worker uses to submit jobs. If Linux OS environment does not have this user, the worker will create this u [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/flink-call",
- "title": "Flink Calls Operating steps",
- "location": [
- "Introduction to Functions",
- "Flink"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back Up Previous Version's Files and Database.\n2. Stop All Services of DolphinScheduler.\nsh ./script/stop-all.sh\n3. Download the New Version's Installation Package.\n\nDownload the latest version of the installation packages.\nThe following upgrade operations need to be performed in the new version's directory.\n\n4. Database Upgrade\n\n\nModify the following properties in conf/config/install_config.conf.\n\n\nIf you use MySQL [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Introduction to Functions",
- "Upgrade"
- ]
- },
- {
- "content": "\nDolphinScheduler Expansion and Reduction\n1. Expansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine where the expansion master or worker node is located has already installed the scheduled service, skip to [1.4 Modify configuration] Edit the configuration f [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Introduction to Functions",
- "Expansion and Reduction"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.System Structure\n1.1 System architecture diagram\n\n \n \n System architecture diagram\n \n\n1.2 Start process activity diagram\n\n \n \n Start process activity diagram\n \n\n1.3 Architecture description\n\n\nMasterServer\nMasterServer adopts a distributed and centerless design concept. Ma [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/architecture/design",
- "title": "System Architecture Design",
- "location": [
- "Advanced Guide",
- "Architecture Design"
- ]
- },
- {
- "content": "Dolphin Scheduler 2.0.3 MetaData\n\nDolphin Scheduler 2.0 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nq [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/architecture/metadata",
- "title": "Dolphin Scheduler 2.0.3 MetaData",
- "location": [
- "Advanced Guide",
- "Metadata"
- ]
- },
- {
- "content": "\nPreface\nThis document explains the DolphinScheduler application configurations according to DolphinScheduler-1.3.x versions.\nDirectory Structure\nCurrently, all the configuration files are under [conf ] directory. Please check the following simplified DolphinScheduler installation directories to have a direct view about the position [conf] directory in and configuration files inside. This document only describes DolphinScheduler configurations and other modules are no [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/architecture/configuration",
- "title": "Preface",
- "location": [
- "Advanced Guide",
- "Configuration File"
- ]
- },
- {
- "content": "Overall Tasks Storage Structure\nAll tasks created in DolphinScheduler are saved in the t_ds_process_definition table.\nThe following shows the 't_ds_process_definition' table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not online, 1 online\n\n\n5\nprojec [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/architecture/task-structure",
- "title": "Overall Tasks Storage Structure",
- "location": [
- "Advanced Guide",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker load balancing algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\nWeighted random (random)\nSmoothing polling (roundrobin)\nLinear load (lowerweight)\nThe default configuration is the linear [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/architecture/load-balance",
- "title": "Load Balance",
- "location": [
- "Advanced Guide",
- "Load Balance"
- ]
- },
- {
- "content": "Cache\nPurpose\nDue to the master-server scheduling process, there will be a large number of database read operations, such as tenant, user, processDefinition, etc. On the one hand, it will put a lot of pressure on the DB, and on the other hand, it will slow down the entire core scheduling process.\nConsidering that this part of the business data is a scenario where more reads and less writes are performed, a cache module is introduced to reduce the DB read pressure and s [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/architecture/cache",
- "title": "Cache",
- "location": [
- "Advanced Guide",
- "Cache"
- ]
- },
- {
- "content": "SkyWalking Agent\nThe dolphinscheduler-skywalking module provides SkyWalking monitor agent for the Dolphinscheduler project.\nThis document describes how to enable SkyWalking 8.4+ support with this module (recommended to use SkyWalking 8.5.0).\nInstallation\nThe following configuration is used to enable SkyWalking agent.\nThrough environment variable configuration (for Docker Compose)\nModify SkyWalking environment variables in docker/docker-swarm/config.env.sh:\nSKYWALKI [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/guide/observability/skywalking-agent",
- "title": "SkyWalking Agent",
- "location": [
- "Observability",
- "SkyWalking-Agent"
- ]
- },
- {
- "content": "Change Log\nImprovements\n[#8110] The fault-tolerant process of the worker is optimized, so that when the server is under too much pressure, the worker service is interrupted, and the task can be transferred to other workers to continue.\n[#8445] Optimize the page display flag of forbidden tasks to distinguish the display of normally executed tasks\n[#8441] Add a prompt on the task box to display all the long task names.\n[#8187] The function of re uploading files is adde [...]
- "version": "2.0.5",
- "time": "2022-3-7",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "About DolphinScheduler\nApache DolphinScheduler is a cloud-native visual Big Data workflow scheduler system, committed to “solving complex big-data task dependencies and triggering relationships in data OPS orchestration so that various types of big data tasks can be used out of the box”.\nHigh Reliability\n\nDecentralized multi-master and multi-worker, HA is supported by itself, overload processing\n\nUser-Friendly\n\nAll process definition operations are visualized, Vis [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/About_DolphinScheduler/About_DolphinScheduler",
- "title": "About DolphinScheduler",
- "location": [
- "About",
- "Introduction"
- ]
- },
- {
- "content": "Hardware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux Operating System Version Requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/installation/hardware",
- "title": "Hardware Environment",
- "location": [
- "About",
- "Hardware Environment"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the\nform of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until\nthere are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/architecture/designplus",
- "title": "System Architecture Design",
- "location": [
- "About",
- "Glossary"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password: admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate a worker group\n\n \n \n \n\nCreate environment\n\n \n \n \n\n\nCreate a token\n\n \n \n\n\nLogin with regular users\n\n\n\nClick on the user name in [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/quick-start",
- "title": "Quick Start",
- "location": [
- "Quick Start",
- "Quick Start"
- ]
- },
- {
- "content": "QuickStart in Docker\nPrerequisites\n\nDocker 1.13.1+\nDocker Compose 1.11.0+\n\nHow to use this Docker image\nHere're 3 ways to quickly install DolphinScheduler\nThe First Way: Start a DolphinScheduler by docker-compose (recommended)\nIn this way, you need to install docker-compose as a prerequisite, please install it yourself according to the rich docker-compose installation guidance on the Internet\nFor Windows 7-10, you can install Docker Toolbox. For Windows 10 64-bi [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/installation/docker",
- "title": "QuickStart in Docker",
- "location": [
- "Quick Start",
- "Docker Deployment"
- ]
- },
- {
- "content": "Standalone\nStandalone only for quick look for DolphinScheduler.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in production, we recommended you follow cluster deployment or kubernetes\n\nNote: Standalone only recommends the use [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/installation/standalone",
- "title": "Standalone",
- "location": [
- "Installation",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Pseudo-Cluster Deployment\nThe purpose of pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, api server, and logger server are all on the same machine.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cl [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/installation/pseudo-cluster",
- "title": "Pseudo-Cluster Deployment",
- "location": [
- "Installation",
- "Pseudo Cluster Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\nCluster deployment is to deploy the DolphinScheduler on multiple machines for running a large number of tasks in production.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in production, we recommended you foll [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/installation/cluster",
- "title": "Cluster Deployment",
- "location": [
- "Installation",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Kubernetes\nKubernetes deployment is deploy DolphinScheduler in a Kubernetes cluster, which can schedule a large number of tasks and can be used in production.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in produc [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/installation/kubernetes",
- "title": "QuickStart in Kubernetes",
- "location": [
- "Installation",
- "Kubernetes Deployment"
- ]
- },
- {
- "content": "Workflow Overview\nThe home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.\n\n\n",
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/homepage",
- "title": "Workflow Overview",
- "location": [
- "Introduction to Functions",
- "Workflow Overview"
- ]
- },
- {
- "content": "Project\nCreate project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Submit" to create a new project.\n\n \n\n\n\nProject home\n\n\nClick the project name link on the project management page to enter the project home page, as shown in the figure below, the project home page contains the task status statistics, process status [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/project/project-list",
- "title": "Project",
- "location": [
- "Introduction to Functions",
- "Project",
- "Project List"
- ]
- },
- {
- "content": "Workflow definition\nCreate workflow definition\n\n\nClick Project Management -> Workflow -> Workflow Definition to enter the workflow definition page, and click the "Create Workflow" button to enter the workflow DAG edit page, as shown in the following figure:\n\n \n\n\n\nDrag in the toolbar Add a Shell task to the drawing board, as shown in the figure below:\n\n\n\nAdd parameter settings for this shell task:\n\n\n\nFill in the "Node Name", [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/project/workflow-definition",
- "title": "Workflow definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Definition"
- ]
- },
- {
- "content": "Workflow instance\nView workflow instance\n\nClick Project Management -> Workflow -> Workflow Instance to enter the Workflow Instance page, as shown in the figure below: \n \n \n\nClick the workflow name to enter the DAG view page to view the task execution status, as shown in the figure below.\n \n\n\n\nView task log\n\nEnter the workflow instance page, click the workflow name, enter the DAG view page, double-click the task node, as shown in the following [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/project/workflow-instance",
- "title": "Workflow instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Instance"
- ]
- },
- {
- "content": "Task instance\n\n\nClick Project Management -> Workflow -> Task Instance to enter the task instance page, as shown in the figure below, click the name of the workflow instance, you can jump to the workflow instance DAG chart to view the task status.\n \n \n \n\n\nView log：Click the "view log" button in the operation column to view the log of task execution.\n \n \n \n\n\n",
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/project/task-instance",
- "title": "Task instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Instance"
- ]
- },
- {
- "content": "Shell\nOverview\nShell task, used to create a shell-type task and execute a series of shell scripts. When the worker executed,\na temporary shell script is generated, and the Linux user with the same name as the tenant executes the script.\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/task/shell",
- "title": "Shell",
- "location": [
- "Introduction to Functions",
- "Task",
- "Shell"
- ]
- },
- {
- "content": "SubProcess\n\nThe sub-process node is to execute a certain external workflow definition as a task node.\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\n\n \n \n\nNode name: The node name in a workflow definition is unique\nRun flag: identify whether this node can be scheduled normally\nDescriptive information: describe the function of the node\nTimeout alarm: Check the timeout alarm and timeout failure. When the task ex [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/task/sub-process",
- "title": "SubProcess",
- "location": [
- "Introduction to Functions",
- "Task",
- "SubProcess"
- ]
- },
- {
- "content": "DEPENDENT\n\nDependent nodes are dependency check nodes. For example, process A depends on the successful execution of process B yesterday, and the dependent node will check whether process B has a successful execution yesterday.\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\n\n \n \n\nThe dependent node provides a logical judgment function, such as checking whether the B process was successful yesterday, or whether th [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/task/dependent",
- "title": "DEPENDENT",
- "location": [
- "Introduction to Functions",
- "Task",
- "Dependent"
- ]
- },
- {
- "content": "Stored Procedure\n\nAccording to the selected data source, execute the stored procedure.\n\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nData source: The data source type of the stored procedure supports MySQL and POSTGRESQL, select the corresponding data source\nMethod: is the method name of the stored procedure\nCustom parameters: The custom parameter types of the stored procedure support IN and OUT, and the d [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/task/stored-procedure",
- "title": "Stored Procedure",
- "location": [
- "Introduction to Functions",
- "Task",
- "Stored Procedure"
- ]
- },
- {
- "content": "SQL\nOverview\nSQL task, used to connect to database and execute SQL.\ncreate data source\nRefer to Data Source\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the drawing board.\n\nTask Parameter\n\nData source: select the corresponding data source\nsql type: supports query and non-query. The query is a select type query, which is returned [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/task/sql",
- "title": "SQL",
- "location": [
- "Introduction to Functions",
- "Task",
- "SQL"
- ]
- },
- {
- "content": "Spark\nOverview\nSpark task type for executing Spark programs. For Spark nodes, the worker submits the task by using the spark command spark submit. See spark-submit for more details.\nCreate task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The node name in a workflow definition is u [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/task/spark",
- "title": "Spark",
- "location": [
- "Introduction to Functions",
- "Task",
- "Spark"
- ]
- },
- {
- "content": "MapReduce\nOverview\n\nMapReduce(MR) task type for executing MapReduce programs. For MapReduce nodes, the worker submits the task by using the Hadoop command hadoop jar. See Hadoop Command Manual for more details.\n\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The node name in a work [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/task/map-reduce",
- "title": "MapReduce",
- "location": [
- "Introduction to Functions",
- "Task",
- "MapReduce"
- ]
- },
- {
- "content": "Python Node\nOverview\nUse Python Task to create a python-type task and execute python scripts. When the worker executes Python Task,\nit will generate a temporary python script, and executes the script by the Linux user with the same name as the tenant.\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameter\n\nNode n [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/task/python",
- "title": "Python Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Python"
- ]
- },
- {
- "content": "Flink\nOverview\nFlink task type for executing Flink programs. For Flink nodes, the worker submits the task by using the flink command flink run. See flink cli for more details.\nCreate task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The node name in a workflow definition is unique. [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/task/flink",
- "title": "Flink",
- "location": [
- "Introduction to Functions",
- "Task",
- "Flink"
- ]
- },
- {
- "content": "HTTP\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.\nDescriptive information: describe the function of the node.\nTask priority: When the number of worker threads is insufficient, they are executed in order from [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/task/http",
- "title": "HTTP",
- "location": [
- "Introduction to Functions",
- "Task",
- "HTTP"
- ]
- },
- {
- "content": "DATAX\n\n\nDrag in the toolbarTask node into the drawing board\n\n \n\n\n\nCustom template: When you turn on the custom template switch, you can customize the content of the json configuration file of the datax node (applicable when the control configuration does not meet the requirements)\n\n\nData source: select the data source to extract the data\n\n\nsql statement: the sql statement used to extract data from the target database, the sql query column name is automatica [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/task/datax",
- "title": "DATAX",
- "location": [
- "Introduction to Functions",
- "Task",
- "DataX"
- ]
- },
- {
- "content": "Pigeon\nPigeon is general websocket service tracking task for DolphinScheduler. It can trigger, check status, get log from remote websocket service.\nCreate\nDrag in the toolbarThe task node to the drawing board to create a new Conditions task\nParameter\n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.\nDescriptive i [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/task/pigeon",
- "title": "Pigeon",
- "location": [
- "Introduction to Functions",
- "Task",
- "Pigeon"
- ]
- },
- {
- "content": "Conditions\nConditions is a condition node, determining which downstream task should be run based on the condition set to it. For now, the Conditions support multiple upstream tasks, but only two downstream tasks. When the number of upstream tasks exceeds one, complex upstream dependencies can be achieved through and and or operators.\nCreate\nDrag in the toolbarThe task node to the drawing board to create a new Conditions task, as shown in the figure below:\n \n \n \ [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/task/conditions",
- "title": "Conditions",
- "location": [
- "Introduction to Functions",
- "Task",
- "Conditions"
- ]
- },
- {
- "content": "Switch\nSwitch is a conditional judgment node, which branch should be executes according to the value of global variable and the expression result written by the user.\nCreate\nDrag the in the tool bar to create task. Note After the switch task is created, you must configure it downstream to make parameter Branch flow work.\nParameter\n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it d [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/task/switch",
- "title": "Switch",
- "location": [
- "Introduction to Functions",
- "Task",
- "Switch"
- ]
- },
- {
- "content": "Built-in Parameter\nBasic Built-in Parameter\n\n variabledeclaration methodmeaning\n \n system.biz.date\n ${system.biz.date}\n The day before the scheduled time of the daily scheduling instance, the format is yyyyMMdd\n \n \n system.biz.curdate\n ${system.biz.curdate}\n The timing time of the daily scheduling instance, the format is yyyyMMdd\n \n \n system.datetime\n ${system.datetime}\n The [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/parameter/built-in",
- "title": "Built-in Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Built-in Parameter"
- ]
- },
- {
- "content": "Global Parameter\nScope\nThe parameters configured on the workflow definition dialog, the whole workflow is it's scope.\nUsage\nthe approach to set global parameters is, after defining the workflow, click the 'save' button, then click the '+' button below the 'Set global':\n\n \n \n\n \n \nThe global_bizdate parameter defined here can be referenced by local parameters of any other task node, and the value of global_bizdate is set to the figure obtained by referencing [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/parameter/global",
- "title": "Global Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Global Parameter"
- ]
- },
- {
- "content": "Local Parameter\nScope\nParameters configured on the task definition dialog, the scope of this parameter only for this task, but if you configured follow Parameter Context, it could passed follow task.\nUsage\nThe approach to set local parameters is, double-click on any node while defining the workflow and click the '+' button next to the 'Custom Parameters':\n\n \n\n\n \n\nIf you want to call the built-in parameter in the local parameters, fill in the value corre [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/parameter/local",
- "title": "Local Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Local Parameter"
- ]
- },
- {
- "content": "Parameter Context\nDolphinScheduler provides the ability to refer to each other between parameters, including: local parameters refer to global parameters, and upstream and downstream parameter transfer. Because of the existence of references, it involves the priority of parameters when the parameter names are the same. see also Parameter Priority\nLocal task use global parameter\nThe premise of local tasks referencing global parameters is that you have already defined Gl [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/parameter/context",
- "title": "Parameter Context",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Context"
- ]
- },
- {
- "content": "Parameter Priority\nDolphinScheduler definition of parameter values involved in may come from three types:\n\nGlobal Parameter: Parameters defined when the workflow saves page definitions\nParameter Context: Parameters passed by upstream nodes\nLocal Parameter：The node's own parameters, which is the parameters defined by the user in [Custom Parameters]. The user can define the value of this part of the parameters at the time of workflow definition.\n\nBecause the value [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/parameter/priority",
- "title": "Parameter Priority",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Priority"
- ]
- },
- {
- "content": "Data Source\nData source center supports MySQL, POSTGRESQL, HIVE/IMPALA, SPARK, CLICKHOUSE, ORACLE, SQLSERVER and other data sources\n\nClick "Data Source Center -> Create Data Source" to create different types of data sources according to requirements.\nClick "Test Connection" to test whether the data source can be successfully connected.\n\n",
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/datasource/introduction",
- "title": "Data Source",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Introduction"
- ]
- },
- {
- "content": "MySQL\n\nData source: select MYSQL\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP hostname: enter the IP to connect to MySQL\nPort: Enter the port to connect to MySQL\nUsername: Set the username for connecting to MySQL\nPassword: Set the password for connecting to MySQL\nDatabase name: Enter the name of the database connected to MySQL\nJdbc connection parameters: parameter settings for MySQL connection, filled [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/datasource/mysql",
- "title": "MySQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "MySQL"
- ]
- },
- {
- "content": "POSTGRESQL\n\nData source: select POSTGRESQL\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Host Name: Enter the IP to connect to POSTGRESQL\nPort: Enter the port to connect to POSTGRESQL\nUsername: Set the username for connecting to POSTGRESQL\nPassword: Set the password for connecting to POSTGRESQL\nDatabase name: Enter the name of the database connected to POSTGRESQL\nJdbc connection parameters: parameter s [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/datasource/postgresql",
- "title": "POSTGRESQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "PostgreSQL"
- ]
- },
- {
- "content": "HIVE\nUse HiveServer2\n \n \n \n\nData source: select HIVE\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Host Name: Enter the IP connected to HIVE\nPort: Enter the port connected to HIVE\nUsername: Set the username for connecting to HIVE\nPassword: Set the password for connecting to HIVE\nDatabase name: Enter the name of the database connected to HIVE\nJdbc connection parameters: parameter settings for HI [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/datasource/hive",
- "title": "HIVE",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "HIVE"
- ]
- },
- {
- "content": "Spark\n\n \n \n\nData source: select Spark\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Hostname: Enter the IP connected to Spark\nPort: Enter the port connected to Spark\nUsername: Set the username for connecting to Spark\nPassword: Set the password for connecting to Spark\nDatabase name: Enter the name of the database connected to Spark\nJdbc connection parameters: parameter settings for Spark connection [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/datasource/spark",
- "title": "Spark",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Spark"
- ]
- },
- {
- "content": "How to create alert plugins and alert groups\nIn version 2.0.6, users need to create alert instances, and then associate them with alert groups, and an alert group can use multiple alert instances, and we will notify them one by one.\nFirst of all, you need to go to the Security Center, select Alarm Group Management, then click Alarm Instance Management on the left, then create an alarm instance, then select the corresponding alarm plug-in and fill in the relevant alarm p [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/alert/alert_plugin_user_guide",
- "title": "How to create alert plugins and alert groups",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Alert Component User Guide "
- ]
- },
- {
- "content": "Enterprise WeChat\nIf you need to use Enterprise WeChat to alert, please create an alarm Instance in warning instance manage, and then choose the wechat plugin. The configuration example of enterprise WeChat is as follows\n\nWhere send type corresponds to app and appchat respectively:\nAPP: https://work.weixin.qq.com/api/doc/90000/90135/90236\nAPPCHAT: https://work.weixin.qq.com/api/doc/90000/90135/90248\nuser.send.msg corresponds to the content in the document. The varia [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/alert/enterprise-wechat",
- "title": "Enterprise WeChat",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Enterprise Wechat"
- ]
- },
- {
- "content": "DingTalk\nIf you need to use DingTalk for alerting, please create an alert instance in the alert instance management and select the DingTalk plugin. The configuration example of DingTalk is as follows:\n\nparameter configuration\n\nWebhook\n\nThe format is as follows: https://oapi.dingtalk.com/robot/send?access_token=XXXXXX\n\n\nKeyword\n\nCustom keywords for security settings\n\n\nSecret\n\nSignature of security settings\n\n\nMessageType\n\nSupport both text and markdown [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/alert/dingtalk",
- "title": "DingTalk",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Ding Talk"
- ]
- },
- {
- "content": "Resource Center\nIf you want to use the resource upload function, you can select the local file directory for a single machine(this operation does not need to deploy Hadoop). Or you can also upload to a Hadoop or MinIO cluster, at this time, you need to have Hadoop (2.6+) or MinIO and other related environments\n\nNote:\n\nIf the resource upload function is used, the deployment user in installation and deployment must to have operation authority\nIf you using Hadoop clust [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/resource",
- "title": "Resource Center",
- "location": [
- "Introduction to Functions",
- "Resource"
- ]
- },
- {
- "content": "Monitor\nService management\n\nService management is mainly to monitor and display the health status and basic information of each service in the system\n\nmaster monitoring\n\nMainly related to master information.\n\n\n \n \nworker monitoring\n\nMainly related to worker information.\n\n\n \n \nZookeeper monitoring\n\nMainly related configuration information of each worker and master in ZooKeeper.\n\n\n \n \nDB monitoring\n\nMainly the health of the DB\n\n\n \n \n [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/monitor",
- "title": "Monitor",
- "location": [
- "Introduction to Functions",
- "Monitor"
- ]
- },
- {
- "content": "Security\n\nOnly the administrator account in the security center has the authority to operate. It has functions such as queue management, tenant management, user management, alarm group management, worker group management, token management, etc. In the user management module, resources, data sources, projects, etc. Authorization\nAdministrator login, default user name and password: admin/dolphinscheduler123\n\nCreate queue\n\nQueue is used when the "queue" para [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/security",
- "title": "Security",
- "location": [
- "Introduction to Functions",
- "Security"
- ]
- },
- {
- "content": "Flink Calls Operating steps\nCreate a queue\n\nLog in to the scheduling system, click "Security", then click "Queue manage" on the left, and click "Create queue" to create a queue.\nFill in the name and value of the queue, and click "Submit"\n\n\n \n \nCreate a tenant\n1. The tenant corresponds to a Linux user, which the user worker uses to submit jobs. If Linux OS environment does not have this user, the worker will create this u [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/flink-call",
- "title": "Flink Calls Operating steps",
- "location": [
- "Introduction to Functions",
- "Flink"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back Up Previous Version's Files and Database.\n2. Stop All Services of DolphinScheduler.\nsh ./script/stop-all.sh\n3. Download the New Version's Installation Package.\n\nDownload the latest version of the installation packages.\nThe following upgrade operations need to be performed in the new version's directory.\n\n4. Database Upgrade\n\n\nModify the following properties in conf/config/install_config.conf.\n\n\nIf you use MySQL [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Introduction to Functions",
- "Upgrade"
- ]
- },
- {
- "content": "\nDolphinScheduler Expansion and Reduction\n1. Expansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine where the expansion master or worker node is located has already installed the scheduled service, skip to [1.4 Modify configuration] Edit the configuration f [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Introduction to Functions",
- "Expansion and Reduction"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.System Structure\n1.1 System architecture diagram\n\n \n \n System architecture diagram\n \n\n1.2 Start process activity diagram\n\n \n \n Start process activity diagram\n \n\n1.3 Architecture description\n\n\nMasterServer\nMasterServer adopts a distributed and centerless design concept. Ma [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/architecture/design",
- "title": "System Architecture Design",
- "location": [
- "Advanced Guide",
- "Architecture Design"
- ]
- },
- {
- "content": "Dolphin Scheduler 2.0.3 MetaData\n\nDolphin Scheduler 2.0 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nq [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/architecture/metadata",
- "title": "Dolphin Scheduler 2.0.3 MetaData",
- "location": [
- "Advanced Guide",
- "Metadata"
- ]
- },
- {
- "content": "\nPreface\nThis document explains the DolphinScheduler application configurations according to DolphinScheduler-1.3.x versions.\nDirectory Structure\nCurrently, all the configuration files are under [conf ] directory. Please check the following simplified DolphinScheduler installation directories to have a direct view about the position [conf] directory in and configuration files inside. This document only describes DolphinScheduler configurations and other modules are no [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/architecture/configuration",
- "title": "Preface",
- "location": [
- "Advanced Guide",
- "Configuration File"
- ]
- },
- {
- "content": "Overall Tasks Storage Structure\nAll tasks created in DolphinScheduler are saved in the t_ds_process_definition table.\nThe following shows the 't_ds_process_definition' table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not online, 1 online\n\n\n5\nprojec [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/architecture/task-structure",
- "title": "Overall Tasks Storage Structure",
- "location": [
- "Advanced Guide",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker load balancing algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\nWeighted random (random)\nSmoothing polling (roundrobin)\nLinear load (lowerweight)\nThe default configuration is the linear [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/architecture/load-balance",
- "title": "Load Balance",
- "location": [
- "Advanced Guide",
- "Load Balance"
- ]
- },
- {
- "content": "Cache\nPurpose\nDue to the master-server scheduling process, there will be a large number of database read operations, such as tenant, user, processDefinition, etc. On the one hand, it will put a lot of pressure on the DB, and on the other hand, it will slow down the entire core scheduling process.\nConsidering that this part of the business data is a scenario where more reads and less writes are performed, a cache module is introduced to reduce the DB read pressure and s [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/architecture/cache",
- "title": "Cache",
- "location": [
- "Advanced Guide",
- "Cache"
- ]
- },
- {
- "content": "SkyWalking Agent\nThe dolphinscheduler-skywalking module provides SkyWalking monitor agent for the Dolphinscheduler project.\nThis document describes how to enable SkyWalking 8.4+ support with this module (recommended to use SkyWalking 8.5.0).\nInstallation\nThe following configuration is used to enable SkyWalking agent.\nThrough environment variable configuration (for Docker Compose)\nModify SkyWalking environment variables in docker/docker-swarm/config.env.sh:\nSKYWALKI [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/observability/skywalking-agent",
- "title": "SkyWalking Agent",
- "location": [
- "Observability",
- "SkyWalking-Agent"
- ]
- },
- {
- "content": "Open API\nBackground\nGenerally, projects and processes are created through pages, but integration with third-party systems requires API calls to manage projects and workflows.\nThe Operation Steps of DS API Calls\nCreate a token\n\nLog in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.\n\n\n \n \n\nSelect the "Expiration time" (Token validity), select [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/guide/open-api",
- "title": "Open API",
- "location": [
- "API",
- "Open API"
- ]
- },
- {
- "content": "Change Log\n\n[Fix-8645][api] Resource Re Upload, name duplicate (#8811)\n[Feature][dag] Jump back to the list page after the workflow is saved. (#8989)\n[Fix-8929][dao]Description Failed to upload the file because the full name was too long (#9020)\n[Fix-8544][api] The folder's size can't be updated when creating, updating or deleting a resource file int the folder. (#9107)\n[BUG][alert-server]validate script before alert script (#9834)\n[Fix-10386]LDAP user create with [...]
- "version": "2.0.6",
- "time": "2022-7-8",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "About DolphinScheduler\nApache DolphinScheduler is a cloud-native visual Big Data workflow scheduler system, committed to “solving complex big-data task dependencies and triggering relationships in data OPS orchestration so that various types of big data tasks can be used out of the box”.\nHigh Reliability\n\nDecentralized multi-master and multi-worker, HA is supported by itself, overload processing\n\nUser-Friendly\n\nAll process definition operations are visualized, Vis [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/About_DolphinScheduler/About_DolphinScheduler",
- "title": "About DolphinScheduler",
- "location": [
- "About",
- "Introduction"
- ]
- },
- {
- "content": "Hardware Environment\nDolphinScheduler, as an open-source distributed workflow task scheduling system, can be well deployed and run in Intel architecture server environments and mainstream virtualization environments, and supports mainstream Linux operating system environments.\n1. Linux Operating System Version Requirements\n\n\n\nOS\nVersion\n\n\n\n\nRed Hat Enterprise Linux\n7.0 and above\n\n\nCentOS\n7.0 and above\n\n\nOracle Enterprise Linux\n7.0 and above\n\n\nUbunt [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/installation/hardware",
- "title": "Hardware Environment",
- "location": [
- "About",
- "Hardware Environment"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the\nform of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until\nthere are no subsequent nodes. Examples are as follows:\n\n \n \n [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/architecture/designplus",
- "title": "System Architecture Design",
- "location": [
- "About",
- "Glossary"
- ]
- },
- {
- "content": "Quick Start\n\n\nAdministrator user login\n\nAddress：http://localhost:12345/dolphinscheduler Username and password: admin/dolphinscheduler123\n\n\n\n\n \n \n\nCreate queue\n\n\n \n \n\nCreate tenant \n\n\n\n \n\nCreating Ordinary Users\n\n\n \n \n\nCreate an alarm group\n\n \n \n \n\nCreate a worker group\n\n \n \n \n\nCreate environment\n\n \n \n \n\n\nCreate a token\n\n \n \n\n\nLogin with regular users\n\n\n\nClick on the user name in [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/quick-start",
- "title": "Quick Start",
- "location": [
- "Quick Start",
- "Quick Start"
- ]
- },
- {
- "content": "QuickStart in Docker\nPrerequisites\n\nDocker 1.13.1+\nDocker Compose 1.11.0+\n\nHow to use this Docker image\nHere're 3 ways to quickly install DolphinScheduler\nThe First Way: Start a DolphinScheduler by docker-compose (recommended)\nIn this way, you need to install docker-compose as a prerequisite, please install it yourself according to the rich docker-compose installation guidance on the Internet\nFor Windows 7-10, you can install Docker Toolbox. For Windows 10 64-bi [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/installation/docker",
- "title": "QuickStart in Docker",
- "location": [
- "Quick Start",
- "Docker Deployment"
- ]
- },
- {
- "content": "Standalone\nStandalone only for quick look for DolphinScheduler.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in production, we recommended you follow cluster deployment or kubernetes\n\nNote: Standalone only recommends the use [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/installation/standalone",
- "title": "Standalone",
- "location": [
- "Installation",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Pseudo-Cluster Deployment\nThe purpose of pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, api server, and logger server are all on the same machine.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cl [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/installation/pseudo-cluster",
- "title": "Pseudo-Cluster Deployment",
- "location": [
- "Installation",
- "Pseudo Cluster Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\nCluster deployment is to deploy the DolphinScheduler on multiple machines for running a large number of tasks in production.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in production, we recommended you foll [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/installation/cluster",
- "title": "Cluster Deployment",
- "location": [
- "Installation",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Kubernetes\nKubernetes deployment is deploy DolphinScheduler in a Kubernetes cluster, which can schedule a large number of tasks and can be used in production.\nIf you are a green hand and want to experience DolphinScheduler, we recommended you install follow Standalone. If you want to experience more complete functions or schedule large tasks number, we recommended you install follow pseudo-cluster deployment. If you want to using DolphinScheduler in produc [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/installation/kubernetes",
- "title": "QuickStart in Kubernetes",
- "location": [
- "Installation",
- "Kubernetes Deployment"
- ]
- },
- {
- "content": "Workflow Overview\nThe home page contains task status statistics, process status statistics, and workflow definition statistics for all projects of the user.\n\n\n",
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/homepage",
- "title": "Workflow Overview",
- "location": [
- "Introduction to Functions",
- "Workflow Overview"
- ]
- },
- {
- "content": "Project\nCreate project\n\n\nClick "Project Management" to enter the project management page, click the "Create Project" button, enter the project name, project description, and click "Submit" to create a new project.\n\n \n\n\n\nProject home\n\n\nClick the project name link on the project management page to enter the project home page, as shown in the figure below, the project home page contains the task status statistics, process status [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/project/project-list",
- "title": "Project",
- "location": [
- "Introduction to Functions",
- "Project",
- "Project List"
- ]
- },
- {
- "content": "Workflow definition\nCreate workflow definition\n\n\nClick Project Management -> Workflow -> Workflow Definition to enter the workflow definition page, and click the "Create Workflow" button to enter the workflow DAG edit page, as shown in the following figure:\n\n \n\n\n\nDrag in the toolbar Add a Shell task to the drawing board, as shown in the figure below:\n\n\n\nAdd parameter settings for this shell task:\n\n\n\nFill in the "Node Name", [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/project/workflow-definition",
- "title": "Workflow definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Definition"
- ]
- },
- {
- "content": "Workflow instance\nView workflow instance\n\nClick Project Management -> Workflow -> Workflow Instance to enter the Workflow Instance page, as shown in the figure below: \n \n \n\nClick the workflow name to enter the DAG view page to view the task execution status, as shown in the figure below.\n \n\n\n\nView task log\n\nEnter the workflow instance page, click the workflow name, enter the DAG view page, double-click the task node, as shown in the following [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/project/workflow-instance",
- "title": "Workflow instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Instance"
- ]
- },
- {
- "content": "Task instance\n\n\nClick Project Management -> Workflow -> Task Instance to enter the task instance page, as shown in the figure below, click the name of the workflow instance, you can jump to the workflow instance DAG chart to view the task status.\n \n \n \n\n\nView log：Click the "view log" button in the operation column to view the log of task execution.\n \n \n \n\n\n",
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/project/task-instance",
- "title": "Task instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Instance"
- ]
- },
- {
- "content": "Shell\nOverview\nShell task, used to create a shell-type task and execute a series of shell scripts. When the worker executed,\na temporary shell script is generated, and the Linux user with the same name as the tenant executes the script.\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/task/shell",
- "title": "Shell",
- "location": [
- "Introduction to Functions",
- "Task",
- "Shell"
- ]
- },
- {
- "content": "SubProcess\n\nThe sub-process node is to execute a certain external workflow definition as a task node.\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\n\n \n \n\nNode name: The node name in a workflow definition is unique\nRun flag: identify whether this node can be scheduled normally\nDescriptive information: describe the function of the node\nTimeout alarm: Check the timeout alarm and timeout failure. When the task ex [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/task/sub-process",
- "title": "SubProcess",
- "location": [
- "Introduction to Functions",
- "Task",
- "SubProcess"
- ]
- },
- {
- "content": "DEPENDENT\n\nDependent nodes are dependency check nodes. For example, process A depends on the successful execution of process B yesterday, and the dependent node will check whether process B has a successful execution yesterday.\n\n\nDrag the task node in the toolbar to the drawing board, as shown in the following figure:\n\n\n \n \n\nThe dependent node provides a logical judgment function, such as checking whether the B process was successful yesterday, or whether th [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/task/dependent",
- "title": "DEPENDENT",
- "location": [
- "Introduction to Functions",
- "Task",
- "Dependent"
- ]
- },
- {
- "content": "Stored Procedure\n\nAccording to the selected data source, execute the stored procedure.\n\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nData source: The data source type of the stored procedure supports MySQL and POSTGRESQL, select the corresponding data source\nMethod: is the method name of the stored procedure\nCustom parameters: The custom parameter types of the stored procedure support IN and OUT, and the d [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/task/stored-procedure",
- "title": "Stored Procedure",
- "location": [
- "Introduction to Functions",
- "Task",
- "Stored Procedure"
- ]
- },
- {
- "content": "SQL\nOverview\nSQL task, used to connect to database and execute SQL.\ncreate data source\nRefer to Data Source\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the drawing board.\n\nTask Parameter\n\nData source: select the corresponding data source\nsql type: supports query and non-query. The query is a select type query, which is returned [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/task/sql",
- "title": "SQL",
- "location": [
- "Introduction to Functions",
- "Task",
- "SQL"
- ]
- },
- {
- "content": "Spark\nOverview\nSpark task type for executing Spark programs. For Spark nodes, the worker submits the task by using the spark command spark submit. See spark-submit for more details.\nCreate task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The node name in a workflow definition is u [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/task/spark",
- "title": "Spark",
- "location": [
- "Introduction to Functions",
- "Task",
- "Spark"
- ]
- },
- {
- "content": "MapReduce\nOverview\n\nMapReduce(MR) task type for executing MapReduce programs. For MapReduce nodes, the worker submits the task by using the Hadoop command hadoop jar. See Hadoop Command Manual for more details.\n\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The node name in a work [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/task/map-reduce",
- "title": "MapReduce",
- "location": [
- "Introduction to Functions",
- "Task",
- "MapReduce"
- ]
- },
- {
- "content": "Python Node\nOverview\nUse Python Task to create a python-type task and execute python scripts. When the worker executes Python Task,\nit will generate a temporary python script, and executes the script by the Linux user with the same name as the tenant.\nCreate Task\n\nClick Project Management-Project Name-Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameter\n\nNode n [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/task/python",
- "title": "Python Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Python"
- ]
- },
- {
- "content": "Flink\nOverview\nFlink task type for executing Flink programs. For Flink nodes, the worker submits the task by using the flink command flink run. See flink cli for more details.\nCreate task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the "Create Workflow" button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameter\n\nNode name: The node name in a workflow definition is unique. [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/task/flink",
- "title": "Flink",
- "location": [
- "Introduction to Functions",
- "Task",
- "Flink"
- ]
- },
- {
- "content": "HTTP\n\nDrag in the toolbarThe task node to the drawing board, as shown in the following figure:\n\n\n \n \n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.\nDescriptive information: describe the function of the node.\nTask priority: When the number of worker threads is insufficient, they are executed in order from [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/task/http",
- "title": "HTTP",
- "location": [
- "Introduction to Functions",
- "Task",
- "HTTP"
- ]
- },
- {
- "content": "DATAX\n\n\nDrag in the toolbarTask node into the drawing board\n\n \n\n\n\nCustom template: When you turn on the custom template switch, you can customize the content of the json configuration file of the datax node (applicable when the control configuration does not meet the requirements)\n\n\nData source: select the data source to extract the data\n\n\nsql statement: the sql statement used to extract data from the target database, the sql query column name is automatica [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/task/datax",
- "title": "DATAX",
- "location": [
- "Introduction to Functions",
- "Task",
- "DataX"
- ]
- },
- {
- "content": "Pigeon\nPigeon is general websocket service tracking task for DolphinScheduler. It can trigger, check status, get log from remote websocket service.\nCreate\nDrag in the toolbarThe task node to the drawing board to create a new Conditions task\nParameter\n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it does not need to be executed, you can turn on the prohibition switch.\nDescriptive i [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/task/pigeon",
- "title": "Pigeon",
- "location": [
- "Introduction to Functions",
- "Task",
- "Pigeon"
- ]
- },
- {
- "content": "Conditions\nConditions is a condition node, determining which downstream task should be run based on the condition set to it. For now, the Conditions support multiple upstream tasks, but only two downstream tasks. When the number of upstream tasks exceeds one, complex upstream dependencies can be achieved through and and or operators.\nCreate\nDrag in the toolbarThe task node to the drawing board to create a new Conditions task, as shown in the figure below:\n \n \n \ [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/task/conditions",
- "title": "Conditions",
- "location": [
- "Introduction to Functions",
- "Task",
- "Conditions"
- ]
- },
- {
- "content": "Switch\nSwitch is a conditional judgment node, which branch should be executes according to the value of global variable and the expression result written by the user.\nCreate\nDrag the in the tool bar to create task. Note After the switch task is created, you must configure it downstream to make parameter Branch flow work.\nParameter\n\nNode name: The node name in a workflow definition is unique.\nRun flag: Identifies whether this node can be scheduled normally, if it d [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/task/switch",
- "title": "Switch",
- "location": [
- "Introduction to Functions",
- "Task",
- "Switch"
- ]
- },
- {
- "content": "Built-in Parameter\nBasic Built-in Parameter\n\n variabledeclaration methodmeaning\n \n system.biz.date\n ${system.biz.date}\n The day before the scheduled time of the daily scheduling instance, the format is yyyyMMdd\n \n \n system.biz.curdate\n ${system.biz.curdate}\n The timing time of the daily scheduling instance, the format is yyyyMMdd\n \n \n system.datetime\n ${system.datetime}\n The [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/parameter/built-in",
- "title": "Built-in Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Built-in Parameter"
- ]
- },
- {
- "content": "Global Parameter\nScope\nThe parameters configured on the workflow definition dialog, the whole workflow is it's scope.\nUsage\nthe approach to set global parameters is, after defining the workflow, click the 'save' button, then click the '+' button below the 'Set global':\n\n \n \n\n \n \nThe global_bizdate parameter defined here can be referenced by local parameters of any other task node, and the value of global_bizdate is set to the figure obtained by referencing [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/parameter/global",
- "title": "Global Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Global Parameter"
- ]
- },
- {
- "content": "Local Parameter\nScope\nParameters configured on the task definition dialog, the scope of this parameter only for this task, but if you configured follow Parameter Context, it could passed follow task.\nUsage\nThe approach to set local parameters is, double-click on any node while defining the workflow and click the '+' button next to the 'Custom Parameters':\n\n \n\n\n \n\nIf you want to call the built-in parameter in the local parameters, fill in the value corre [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/parameter/local",
- "title": "Local Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Local Parameter"
- ]
- },
- {
- "content": "Parameter Context\nDolphinScheduler provides the ability to refer to each other between parameters, including: local parameters refer to global parameters, and upstream and downstream parameter transfer. Because of the existence of references, it involves the priority of parameters when the parameter names are the same. see also Parameter Priority\nLocal task use global parameter\nThe premise of local tasks referencing global parameters is that you have already defined Gl [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/parameter/context",
- "title": "Parameter Context",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Context"
- ]
- },
- {
- "content": "Parameter Priority\nDolphinScheduler definition of parameter values involved in may come from three types:\n\nGlobal Parameter: Parameters defined when the workflow saves page definitions\nParameter Context: Parameters passed by upstream nodes\nLocal Parameter：The node's own parameters, which is the parameters defined by the user in [Custom Parameters]. The user can define the value of this part of the parameters at the time of workflow definition.\n\nBecause the value [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/parameter/priority",
- "title": "Parameter Priority",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Priority"
- ]
- },
- {
- "content": "Data Source\nData source center supports MySQL, POSTGRESQL, HIVE/IMPALA, SPARK, CLICKHOUSE, ORACLE, SQLSERVER and other data sources\n\nClick "Data Source Center -> Create Data Source" to create different types of data sources according to requirements.\nClick "Test Connection" to test whether the data source can be successfully connected.\n\n",
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/datasource/introduction",
- "title": "Data Source",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Introduction"
- ]
- },
- {
- "content": "MySQL\n\nData source: select MYSQL\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP hostname: enter the IP to connect to MySQL\nPort: Enter the port to connect to MySQL\nUsername: Set the username for connecting to MySQL\nPassword: Set the password for connecting to MySQL\nDatabase name: Enter the name of the database connected to MySQL\nJdbc connection parameters: parameter settings for MySQL connection, filled [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/datasource/mysql",
- "title": "MySQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "MySQL"
- ]
- },
- {
- "content": "POSTGRESQL\n\nData source: select POSTGRESQL\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Host Name: Enter the IP to connect to POSTGRESQL\nPort: Enter the port to connect to POSTGRESQL\nUsername: Set the username for connecting to POSTGRESQL\nPassword: Set the password for connecting to POSTGRESQL\nDatabase name: Enter the name of the database connected to POSTGRESQL\nJdbc connection parameters: parameter s [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/datasource/postgresql",
- "title": "POSTGRESQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "PostgreSQL"
- ]
- },
- {
- "content": "HIVE\nUse HiveServer2\n \n \n \n\nData source: select HIVE\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Host Name: Enter the IP connected to HIVE\nPort: Enter the port connected to HIVE\nUsername: Set the username for connecting to HIVE\nPassword: Set the password for connecting to HIVE\nDatabase name: Enter the name of the database connected to HIVE\nJdbc connection parameters: parameter settings for HI [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/datasource/hive",
- "title": "HIVE",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "HIVE"
- ]
- },
- {
- "content": "Spark\n\n \n \n\nData source: select Spark\nData source name: enter the name of the data source\nDescription: Enter a description of the data source\nIP/Hostname: Enter the IP connected to Spark\nPort: Enter the port connected to Spark\nUsername: Set the username for connecting to Spark\nPassword: Set the password for connecting to Spark\nDatabase name: Enter the name of the database connected to Spark\nJdbc connection parameters: parameter settings for Spark connection [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/datasource/spark",
- "title": "Spark",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Spark"
- ]
- },
- {
- "content": "How to create alert plugins and alert groups\nafter version 2.0.0, users need to create alert instances, and then associate them with alert groups, and an alert group can use multiple alert instances, and we will notify them one by one.\nFirst of all, you need to go to the Security Center, select Alarm Group Management, then click Alarm Instance Management on the left, then create an alarm instance, then select the corresponding alarm plug-in and fill in the relevant alar [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/alert/alert_plugin_user_guide",
- "title": "How to create alert plugins and alert groups",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Alert Component User Guide "
- ]
- },
- {
- "content": "Enterprise WeChat\nIf you need to use Enterprise WeChat to alert, please create an alarm Instance in warning instance manage, and then choose the wechat plugin. The configuration example of enterprise WeChat is as follows\n\nWhere send type corresponds to app and appchat respectively:\nAPP: https://work.weixin.qq.com/api/doc/90000/90135/90236\nAPPCHAT: https://work.weixin.qq.com/api/doc/90000/90135/90248\nuser.send.msg corresponds to the content in the document. The varia [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/alert/enterprise-wechat",
- "title": "Enterprise WeChat",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Enterprise Wechat"
- ]
- },
- {
- "content": "DingTalk\nIf you need to use DingTalk for alerting, please create an alert instance in the alert instance management and select the DingTalk plugin. The configuration example of DingTalk is as follows:\n\nparameter configuration\n\nWebhook\n\nThe format is as follows: https://oapi.dingtalk.com/robot/send?access_token=XXXXXX\n\n\nKeyword\n\nCustom keywords for security settings\n\n\nSecret\n\nSignature of security settings\n\n\nMessageType\n\nSupport both text and markdown [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/alert/dingtalk",
- "title": "DingTalk",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Ding Talk"
- ]
- },
- {
- "content": "Resource Center\nIf you want to use the resource upload function, you can select the local file directory for a single machine(this operation does not need to deploy Hadoop). Or you can also upload to a Hadoop or MinIO cluster, at this time, you need to have Hadoop (2.6+) or MinIO and other related environments\n\nNote:\n\nIf the resource upload function is used, the deployment user in installation and deployment must to have operation authority\nIf you using Hadoop clust [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/resource",
- "title": "Resource Center",
- "location": [
- "Introduction to Functions",
- "Resource"
- ]
- },
- {
- "content": "Monitor\nService management\n\nService management is mainly to monitor and display the health status and basic information of each service in the system\n\nmaster monitoring\n\nMainly related to master information.\n\n\n \n \nworker monitoring\n\nMainly related to worker information.\n\n\n \n \nZookeeper monitoring\n\nMainly related configuration information of each worker and master in ZooKeeper.\n\n\n \n \nDB monitoring\n\nMainly the health of the DB\n\n\n \n \n [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/monitor",
- "title": "Monitor",
- "location": [
- "Introduction to Functions",
- "Monitor"
- ]
- },
- {
- "content": "Security\n\nOnly the administrator account in the security center has the authority to operate. It has functions such as queue management, tenant management, user management, alarm group management, worker group management, token management, etc. In the user management module, resources, data sources, projects, etc. Authorization\nAdministrator login, default user name and password: admin/dolphinscheduler123\n\nCreate queue\n\nQueue is used when the "queue" para [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/security",
- "title": "Security",
- "location": [
- "Introduction to Functions",
- "Security"
- ]
- },
- {
- "content": "Flink Calls Operating steps\nCreate a queue\n\nLog in to the scheduling system, click "Security", then click "Queue manage" on the left, and click "Create queue" to create a queue.\nFill in the name and value of the queue, and click "Submit"\n\n\n \n \nCreate a tenant\n1. The tenant corresponds to a Linux user, which the user worker uses to submit jobs. If Linux OS environment does not have this user, the worker will create this u [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/flink-call",
- "title": "Flink Calls Operating steps",
- "location": [
- "Introduction to Functions",
- "Flink"
- ]
- },
- {
- "content": "DolphinScheduler upgrade documentation\n1. Back Up Previous Version's Files and Database.\n2. Stop All Services of DolphinScheduler.\nsh ./script/stop-all.sh\n3. Download the New Version's Installation Package.\n\nDownload the latest version of the installation packages.\nThe following upgrade operations need to be performed in the new version's directory.\n\n4. Database Upgrade\n\n\nModify the following properties in conf/config/install_config.conf.\n\n\nIf you use MySQL [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/upgrade",
- "title": "DolphinScheduler upgrade documentation",
- "location": [
- "Introduction to Functions",
- "Upgrade"
- ]
- },
- {
- "content": "\nDolphinScheduler Expansion and Reduction\n1. Expansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine where the expansion master or worker node is located has already installed the scheduled service, skip to [1.4 Modify configuration] Edit the configuration f [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Introduction to Functions",
- "Expansion and Reduction"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.System Structure\n1.1 System architecture diagram\n\n \n \n System architecture diagram\n \n\n1.2 Start process activity diagram\n\n \n \n Start process activity diagram\n \n\n1.3 Architecture description\n\n\nMasterServer\nMasterServer adopts a distributed and centerless design concept. Ma [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/architecture/design",
- "title": "System Architecture Design",
- "location": [
- "Advanced Guide",
- "Architecture Design"
- ]
- },
- {
- "content": "Dolphin Scheduler 2.0.3 MetaData\n\nDolphin Scheduler 2.0 DB Table Overview\n\n\n\nTable Name\nComment\n\n\n\n\nt_ds_access_token\ntoken for access ds backend\n\n\nt_ds_alert\nalert detail\n\n\nt_ds_alertgroup\nalert group\n\n\nt_ds_command\ncommand detail\n\n\nt_ds_datasource\ndata source\n\n\nt_ds_error_command\nerror command detail\n\n\nt_ds_process_definition\nprocess difinition\n\n\nt_ds_process_instance\nprocess instance\n\n\nt_ds_project\nproject\n\n\nt_ds_queue\nq [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/architecture/metadata",
- "title": "Dolphin Scheduler 2.0.3 MetaData",
- "location": [
- "Advanced Guide",
- "Metadata"
- ]
- },
- {
- "content": "\nPreface\nThis document explains the DolphinScheduler application configurations according to DolphinScheduler-1.3.x versions.\nDirectory Structure\nCurrently, all the configuration files are under [conf ] directory. Please check the following simplified DolphinScheduler installation directories to have a direct view about the position [conf] directory in and configuration files inside. This document only describes DolphinScheduler configurations and other modules are no [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/architecture/configuration",
- "title": "Preface",
- "location": [
- "Advanced Guide",
- "Configuration File"
- ]
- },
- {
- "content": "Overall Tasks Storage Structure\nAll tasks created in DolphinScheduler are saved in the t_ds_process_definition table.\nThe following shows the 't_ds_process_definition' table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not online, 1 online\n\n\n5\nprojec [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/architecture/task-structure",
- "title": "Overall Tasks Storage Structure",
- "location": [
- "Advanced Guide",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker load balancing algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\nWeighted random (random)\nSmoothing polling (roundrobin)\nLinear load (lowerweight)\nThe default configuration is the linear [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/architecture/load-balance",
- "title": "Load Balance",
- "location": [
- "Advanced Guide",
- "Load Balance"
- ]
- },
- {
- "content": "Cache\nPurpose\nDue to the master-server scheduling process, there will be a large number of database read operations, such as tenant, user, processDefinition, etc. On the one hand, it will put a lot of pressure on the DB, and on the other hand, it will slow down the entire core scheduling process.\nConsidering that this part of the business data is a scenario where more reads and less writes are performed, a cache module is introduced to reduce the DB read pressure and s [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/architecture/cache",
- "title": "Cache",
- "location": [
- "Advanced Guide",
- "Cache"
- ]
- },
- {
- "content": "SkyWalking Agent\nThe dolphinscheduler-skywalking module provides SkyWalking monitor agent for the Dolphinscheduler project.\nThis document describes how to enable SkyWalking 8.4+ support with this module (recommended to use SkyWalking 8.5.0).\nInstallation\nThe following configuration is used to enable SkyWalking agent.\nThrough environment variable configuration (for Docker Compose)\nModify SkyWalking environment variables in docker/docker-swarm/config.env.sh:\nSKYWALKI [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/observability/skywalking-agent",
- "title": "SkyWalking Agent",
- "location": [
- "Observability",
- "SkyWalking-Agent"
- ]
- },
- {
- "content": "Open API\nBackground\nGenerally, projects and processes are created through pages, but integration with third-party systems requires API calls to manage projects and workflows.\nThe Operation Steps of DS API Calls\nCreate a token\n\nLog in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.\n\n\n \n \n\nSelect the "Expiration time" (Token validity), select [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/guide/open-api",
- "title": "Open API",
- "location": [
- "API",
- "Open API"
- ]
- },
- {
- "content": "Change Log\nBug Fixes\n\nMonitoring: Added disk available capacity monitoring (#10030)\nfix Parameter transfer problem when recover failed task (#10694)\nfix copy bug (#10940)\nfix dependent task when the dependent on process have forbidden tasks (#10952)\ncorrect post body for http alert plugin (#11946)\n[bugfix-10116]fix complement data bug (#12376)\n[Fix-12167][masterserver] The retry task was submitted in advance (#12291)\n[Bug][api] fix Circular dependencies are not [...]
- "version": "2.0.7",
- "time": "2022-10-24",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "About DolphinScheduler\nApache DolphinScheduler provides a distributed and easy to expand visual workflow task scheduling open-source platform. It is suitable for enterprise-level scenarios. It provides a solution to visualize operation tasks, workflows, and the entire data processing procedures.\nApache DolphinScheduler aims to solve complex big data task dependencies and to trigger relationships in data OPS orchestration for various big data applications. Solves the int [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/about/introduction",
- "title": "About DolphinScheduler",
- "location": [
- "About",
- "Introduction"
- ]
- },
- {
- "content": "Features\nSimple to Use\n\nVisual DAG: User-friendly drag-and-drop workflow definition and facility for run-time control.\nModular Operation: Modularity facilitates easy customization and maintenance.\n\nRich Scenarios\n\nMultiple Task Type Support: Supports more than 10 task types, like Shell, MR, Spark, SQL, etc., with cross-language support making it easy to extend\nWorkflow Ops: Workflow can be timed, paused, resumed and stopped, enabling easy maintenance and control [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/about/features",
- "title": "Features",
- "location": [
- "About",
- "Features"
- ]
- },
- {
- "content": "Hardware Environment\nThis section briefs about the hardware requirements for DolphinScheduler. DolphinScheduler works as an open-source distributed workflow task scheduling system. It can deploy and run smoothly in Intel architecture server environments and mainstream virtualization environments. It also supports mainstream Linux operating system environments and ARM architecture.\nLinux Operating System Version Requirements\nThe Linux operating systems specified below c [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/about/hardware",
- "title": "Hardware Environment",
- "location": [
- "About",
- "Hardware Environment"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the\nform of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until\nthere are no subsequent nodes. Examples are as follows:\n\nProcess d [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/about/glossary",
- "title": "System Architecture Design",
- "location": [
- "About",
- "Glossary"
- ]
- },
- {
- "content": "Quick Start\nThis is a Quick Start guide to help you get a basic idea of working with Apache DolphinScheduler. Once you've gone through the tutorial video, we encourage you to find out more about Apache DolphinScheduler functions and examples in .\n \n \n\nAdministrator User Login\n\nAddress: http://localhost:12345/dolphinscheduler\nUsername and password: admin/dolphinscheduler123\n\n\nCreate a queue\n\nCreate a tenant\n\nCreate Ordinary Users\n\nCreate an alarm instance [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/start/quick-start",
- "title": "Quick Start",
- "location": [
- "Quick Start",
- "Quick Start"
- ]
- },
- {
- "content": "Docker Quick Start\nThere are three ways to start DolphinScheduler with Docker, Standalone-server is the way you\nfind if you just want to start and try DolphinScheduler as a beginner. docker-compose is for\nsome who want to deploy DolphinScheduler in small or event middle scale workflows in their daily work.\nUsing exists postgresql and zookeeper server is for users who want to reuse the database\nor zookeeper server already exists.\nPrepare\n\nDocker 1.13.1+\nDocker Com [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/start/docker",
- "title": "Docker Quick Start",
- "location": [
- "Quick Start",
- "Docker Deployment"
- ]
- },
- {
- "content": "Home Page\nApache DolphinScheduler home page lets you see task state statistics, workflow state statistics, and project statistics for all projects of users. It is the best way to observe status of your system as a whole as well as diving into individual process to check each status of task and task logs.\n\n",
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/homepage",
- "title": "Home Page",
- "location": [
- "Introduction to Functions",
- "Workflow Overview"
- ]
- },
- {
- "content": "Project\nThis page describes details regarding Project screen in Apache DolphinScheduler. Here, you will see all the functions which can be handled in this screen. The following table explains commonly used terms in Apache DolphinScheduler:\n\n\n\nGlossary\ndescription\n\n\n\n\nDAG\nTasks in a workflow are assembled in form of Directed Acyclic Graph (DAG). A topological traversal is performed from nodes with zero degrees of entry until there are no subsequent nodes.\n\n\n [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/project/project-list",
- "title": "Project",
- "location": [
- "Introduction to Functions",
- "Project",
- "Project List"
- ]
- },
- {
- "content": "Workflow Definition\nOverview\nWork Definition screen shows list of existing workflows and utility to create or import workflows.\n\nCreate workflow definition\nClick Project Management -> Workflow -> Workflow Definition, enter the Workflow Definition page, and click the Create Workflow button to enter the workflow DAG edit page, as shown in the following figure:\n\nCreate Tasks to Workflow\nDrag from the toolbar to the canvas, to add a shell task to the canvas, as [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/project/workflow-definition",
- "title": "Workflow Definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Definition"
- ]
- },
- {
- "content": "Workflow Instance\nView Workflow Instance\nClick Project Management -> Workflow -> Workflow Instance, enter the Workflow Instance page, as shown in the following figure:\n\nClick the workflow name to enter the DAG view page, and check the task execution status, as shown in the following figure:\n\nView Task Log\nEnter the workflow instance page, click the workflow name, enter the DAG view page, double-click the task node, as shown in the following figure:\n\nClick & [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/project/workflow-instance",
- "title": "Workflow Instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Instance"
- ]
- },
- {
- "content": "Task Instance\nCreate Task Instance\nClick Project Management -> Workflow -> Task Instance to enter the task instance page, as shown in the figure below, click the name of the workflow instance to jump to the DAG diagram of the workflow instance to view the task status.\n\nView Log\nClick the View Log button in the operation column to view the log of the task execution\n\n",
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/project/task-instance",
- "title": "Task Instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Instance"
- ]
- },
- {
- "content": "Task Definition\nTask definition allows to modify or operate tasks at the task level rather than modifying them in the workflow definition.\nWe already have workflow level task editor in workflow definition which you can click the specific\nworkflow and then edit its task definition. It is depressing when you want to edit the task definition but do not remember\nwhich workflow it belongs to. So we decide to add Task Definition view under Task menu.\n\nIn this view, you ca [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/project/task-definition",
- "title": "Task Definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Definition"
- ]
- },
- {
- "content": "Shell\nOverview\nShell task type, used to create a shell type task and execute a series of shell scripts. When the worker executes this task, a temporary shell script is generated and executed using the linux user with the same name as the tenant.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameters\n\n\n\nParamet [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/shell",
- "title": "Shell",
- "location": [
- "Introduction to Functions",
- "Task",
- "Shell"
- ]
- },
- {
- "content": "SubProcess Node\nOverview\nThe sub-process node is to execute an external workflow definition as a task node.\nCreate Task\n\nClick Project Management-> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar task node to canvas to create a new SubProcess task.\n\nTask Parameter\n\n\n\nParameter\nDescription\n\n\n\n\nNode name\nUnique name of node in workflow definition.\n\n\nRun flag\nIdentifi [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/sub-process",
- "title": "SubProcess Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "SubProcess"
- ]
- },
- {
- "content": "Dependent\nOverview\nDependent nodes are dependency check nodes. For example, process A depends on the successful execution of process B from yesterday, and the dependent node will check whether process B run successful yesterday.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar task node to canvas.\n\nTask Parameter\n\n\n\nParameter\nDescripti [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/dependent",
- "title": "Dependent",
- "location": [
- "Introduction to Functions",
- "Task",
- "Dependent"
- ]
- },
- {
- "content": "Stored Procedure\n\nExecute the stored procedure according to the selected DataSource.\n\n\nDrag from the toolbar task node into the canvas, as shown in the figure below:\n\n\n \n \nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nNode Name\nSet the name of the task. Node names within a workflow definition are unique.\n\n\nRun flag\nIndicates whether the node can be scheduled normally. If it is not necessary to execute, you can turn on the prohibiting execution s [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/stored-procedure",
- "title": "Stored Procedure",
- "location": [
- "Introduction to Functions",
- "Task",
- "Stored Procedure"
- ]
- },
- {
- "content": "SQL\nOverview\nSQL task type used to connect to databases and execute SQL.\nCreate DataSource\nRefer to datasource-setting DataSource Center section\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameter\n\n\n\nParameter\nDescription\n\n\n\n\nData source\nSelect the corresponding DataSource.\n\n\nSQL type\nSupports qu [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/sql",
- "title": "SQL",
- "location": [
- "Introduction to Functions",
- "Task",
- "SQL"
- ]
- },
- {
- "content": "Spark Node\nOverview\nSpark task type for executing Spark application. When executing the Spark task, the worker will submits a job to the Spark cluster by following commands:\n(1) spark submit method to submit tasks. See spark-submit for more details.\n(2) spark sql method to submit tasks. See spark sql for more details.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing pag [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/spark",
- "title": "Spark Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Spark"
- ]
- },
- {
- "content": "MapReduce Node\nOverview\nMapReduce(MR) task type used for executing MapReduce programs. For MapReduce nodes, the worker submits the task by using the Hadoop command hadoop jar. See Hadoop Command Manual for more details.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameters\nGeneral\n\n\n\nParameter\nDescription\n\ [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/map-reduce",
- "title": "MapReduce Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "MapReduce"
- ]
- },
- {
- "content": "Python Node\nOverview\nUse Python Task to create a python-type task and execute python scripts. When the worker executes Python Task,\nit will generate a temporary python script, and executes the script by the Linux user with the same name as the tenant.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameter\n\n\n\nPa [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/python",
- "title": "Python Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Python"
- ]
- },
- {
- "content": "Flink Node\nOverview\nFlink task type, used to execute Flink programs. For Flink nodes:\n\n\nWhen the program type is Java, Scala or Python, the worker submits the task flink run using the Flink command. See flink cli for more details.\n\n\nWhen the program type is SQL, the worker submit tasks using sql-client.sh. See flink sql client for more details.\n\n\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/flink",
- "title": "Flink Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Flink"
- ]
- },
- {
- "content": "HTTP Node\nOverview\nThis node is used to perform http type tasks such as the common POST and GET request types, and also supports http request validation and other functions.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nNode Name\nSet the name of the task. [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/http",
- "title": "HTTP Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "HTTP"
- ]
- },
- {
- "content": "DataX\nOverview\nDataX task type for executing DataX programs. For DataX nodes, the worker will execute ${DATAX_HOME}/bin/datax.py to analyze the input json file.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nNode name\nThe node name in a workflow definition [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/datax",
- "title": "DataX",
- "location": [
- "Introduction to Functions",
- "Task",
- "DataX"
- ]
- },
- {
- "content": "Pigeon\nOverview\nPigeon is a task used to trigger remote tasks, acquire logs or status by calling remote WebSocket service. It is DolphinScheduler uses a remote WebSocket service to call tasks.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas to create a new Pigeon task.\n\nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nN [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/pigeon",
- "title": "Pigeon",
- "location": [
- "Introduction to Functions",
- "Task",
- "Pigeon"
- ]
- },
- {
- "content": "Conditions\nCondition is a conditional node, that determines which downstream task should run based on the condition of the upstream task. Currently, the Conditions support multiple upstream tasks, but only two downstream tasks. When the number of upstream tasks exceeds one, achieve complex upstream dependencies by through and and or operators.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to en [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/conditions",
- "title": "Conditions",
- "location": [
- "Introduction to Functions",
- "Task",
- "Conditions"
- ]
- },
- {
- "content": "Switch\nOverview\nThe switch is a conditional judgment node, decide the branch executes according to the value of global variable and the expression result written by the user.\nNote: Execute expressions using javax.script.ScriptEngine.eval.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar task node to canvas to create a task.\n\nNote: After cr [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/switch",
- "title": "Switch",
- "location": [
- "Introduction to Functions",
- "Task",
- "Switch"
- ]
- },
- {
- "content": "Amazon EMR\nOverview\nAmazon EMR task type, for creating EMR clusters on AWS and running computing tasks. Using aws-java-sdk in the background code, to transfer JSON parameters to RunJobFlowRequest object and submit to AWS.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, click the Create Workflow button to enter the DAG editing page.\nDrag AmazonEMR task from the toolbar to the artboard to complete the creation.\n\nTask Parameters\n [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/emr",
- "title": "Amazon EMR",
- "location": [
- "Introduction to Functions",
- "Task",
- "Amazon EMR"
- ]
- },
- {
- "content": "Apache Zeppelin\nOverview\nUse Zeppelin Task to create a zeppelin-type task and execute zeppelin notebook paragraphs. When the worker executes Zeppelin Task,\nit will call Zeppelin Client API to trigger zeppelin notebook paragraph. Click here for details about Apache Zeppelin Notebook.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the can [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/task/zeppelin",
- "title": "Apache Zeppelin",
- "location": [
- "Introduction to Functions",
- "Task",
- "Apache Zeppelin"
- ]
- },
- {
- "content": "Built-in Parameter\nBasic Built-in Parameter\n\n\n\nVariable\nDeclaration Method\nMeaning\n\n\n\n\nsystem.biz.date\n${system.biz.date}\nThe day before the schedule time of the daily scheduling instance, the format is yyyyMMdd\n\n\nsystem.biz.curdate\n${system.biz.curdate}\nThe schedule time of the daily scheduling instance, the format is yyyyMMdd\n\n\nsystem.datetime\n${system.datetime}\nThe schedule time of the daily scheduling instance, the format is yyyyMMddHHmmss\n\n\ [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/parameter/built-in",
- "title": "Built-in Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Built-in Parameter"
- ]
- },
- {
- "content": "Global Parameter\nScope\nGlobal parameters are parameters that are valid for all task nodes of the entire workflow. It can be configured on the workflow definition page.\nUsage\nThe specific use method can be determined according to the actual production situation. This example uses a shell task to print out the date value of yesterday.\nExample\nCreate a Shell task\nCreate a shell task and enter echo ${dt} in the script content. In this case, dt is the global parameter w [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/parameter/global",
- "title": "Global Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Global Parameter"
- ]
- },
- {
- "content": "Local Parameter\nScope\nParameters configured on the task definition page, the scope of this parameter is inside this task only. But if you configure according to Refer to Parameter Context, it could pass to downstream tasks.\nUsage\n\nIf you want to use parameter in single task, see local parameter with custom parameter.\nIf you want to use set parameter in task and use them in downstream tasks:\n\nFor simple use without using custom parameter, see export parameter via s [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/parameter/local",
- "title": "Local Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Local Parameter"
- ]
- },
- {
- "content": "Parameter Context\nDolphinScheduler provides the ability to refer to each other between parameters, including local parameters refer to global parameters and upstream and downstream parameter transfer. Due to the existence of references, it involves the priority of parameters when the parameter names are the same. see also Parameter Priority\nLocal Task Refers to Global Parameter\nThe premise of local tasks referring global parameters is that you have already defined Glob [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/parameter/context",
- "title": "Parameter Context",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Context"
- ]
- },
- {
- "content": "Parameter Priority\nDolphinScheduler has three parameter types:\n\nGlobal Parameter: parameters defined at the workflow define page.\nParameter Context: parameters passed by upstream task nodes.\nLocal Parameter: parameters belong to its node, which is the parameters defined by the user in [Custom Parameters].\n\nThe user can define part of the parameters when creating workflow definitions.\nAs there are multiple sources of the parameter value, it will raise parameter pri [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/parameter/priority",
- "title": "Parameter Priority",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Priority"
- ]
- },
- {
- "content": "MySQL\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect MYSQL.\n\n\nDatasource name\nEnter the name of the DataSource.\n\n\nDescription\nEnter a description of the DataSource.\n\n\nIP/Host Name\nEnter the MYSQL service IP.\n\n\nPort\nEnter the MYSQL service port.\n\n\nUsername\nSet the username for MYSQL connection.\n\n\nPassword\nSet the password for MYSQL connection.\n\n\nDatabase name\nEnter the database name of the MYSQL connection.\n [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/datasource/mysql",
- "title": "MySQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "MySQL"
- ]
- },
- {
- "content": "PostgreSQL\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect POSTGRESQL.\n\n\nDatasource name\nEnter the name of the DataSource.\n\n\nDescription\nEnter a description of the DataSource.\n\n\nIP/Host Name\nEnter the PostgreSQL service IP.\n\n\nPort\nEnter the PostgreSQL service port.\n\n\nUsername\nSet the username for PostgreSQL connection.\n\n\nPassword\nSet the password for PostgreSQL connection.\n\n\nDatabase name\nEnter the database n [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/datasource/postgresql",
- "title": "PostgreSQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "PostgreSQL"
- ]
- },
- {
- "content": "HIVE\nUse HiveServer2\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect HIVE.\n\n\nDatasource name\nEnter the name of the DataSource.\n\n\nDescription\nEnter a description of the DataSource.\n\n\nIP/Host Name\nEnter the HIVE service IP.\n\n\nPort\nEnter the HIVE service port.\n\n\nUsername\nSet the username for HIVE connection.\n\n\nPassword\nSet the password for HIVE connection.\n\n\nDatabase name\nEnter the database name of the HIVE con [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/datasource/hive",
- "title": "HIVE",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "HIVE"
- ]
- },
- {
- "content": "Spark\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect Spark.\n\n\nDatasource name\nEnter the name of the DataSource.\n\n\nDescription\nEnter a description of the DataSource.\n\n\nIP/Host Name\nEnter the Spark service IP.\n\n\nPort\nEnter the Spark service port.\n\n\nUsername\nSet the username for Spark connection.\n\n\nPassword\nSet the password for Spark connection.\n\n\nDatabase name\nEnter the database name of the Spark connection.\n [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/datasource/spark",
- "title": "Spark",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Spark"
- ]
- },
- {
- "content": "Presto\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect Presto.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the Presto service IP.\n\n\nPort\nEnter the Presto service port.\n\n\nUsername\nSet the username for Presto connection.\n\n\nPassword\nSet the password for Presto connection.\n\n\nDatabase Name\nEnter the database name of the Presto connec [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/datasource/presto",
- "title": "Presto",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Presto"
- ]
- },
- {
- "content": "SQL Server\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect SQLSERVER.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the SQLSERVER service IP.\n\n\nPort\nEnter the SQLSERVER service port.\n\n\nUsername\nSet the username for SQLSERVER connection.\n\n\nPassword\nSet the password for SQLSERVER connection.\n\n\nDatabase Name\nEnter the database name o [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/datasource/sqlserver",
- "title": "SQL Server",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "SQL SERVER"
- ]
- },
- {
- "content": "Amazon Redshift\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect Redshift.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the Redshift service IP.\n\n\nPort\nEnter the Redshift service port.\n\n\nUsername\nSet the username for Redshift connection.\n\n\nPassword\nSet the password for Redshift connection.\n\n\nDatabase Name\nEnter the database name o [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/datasource/redshift",
- "title": "Amazon Redshift",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Amazon Redshift"
- ]
- },
- {
- "content": "ClickHouse\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect CLICKHOUSE.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the CLICKHOUSE service IP.\n\n\nPort\nEnter the CLICKHOUSE service port.\n\n\nUsername\nSet the username for CLICKHOUSE connection.\n\n\nPassword\nSet the password for CLICKHOUSE connection.\n\n\nDatabase Name\nEnter the database n [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/datasource/clickhouse",
- "title": "ClickHouse",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "ClickHouse"
- ]
- },
- {
- "content": "DB2\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect DB2.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the DB2 service IP.\n\n\nPort\nEnter the DB2 service port.\n\n\nUsername\nSet the username for DB2 connection.\n\n\nPassword\nSet the password for DB2 connection.\n\n\nDatabase Name\nEnter the database name of the DB2 connection.\n\n\njdbc conne [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/datasource/db2",
- "title": "DB2",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "IBM DB2"
- ]
- },
- {
- "content": "Oracle\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect Oracle.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the Oracle service IP.\n\n\nPort\nEnter the Oracle service port.\n\n\nUsername\nSet the username for Oracle connection.\n\n\nPassword\nSet the password for Oracle connection.\n\n\nDatabase Name\nEnter the database name of the Oracle connec [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/datasource/oracle",
- "title": "Oracle",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Oracle"
- ]
- },
- {
- "content": "Alert Component User Guide\nAlert Plugins and Alert Groups Creation\nIn version 2.0.0, users need to create alert instances, and needs to choose an alarm policy when defining an alarm instance, there are three options: send if the task succeeds, send on failure, and send on both success and failure. when the workflow or task is executed, if an alarm is triggered, calling the alarm instance send method needs a logical judgment, which matches the alarm instance with the tas [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/alert/alert_plugin_user_guide",
- "title": "Alert Component User Guide",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Alert Component User Guide "
- ]
- },
- {
- "content": "Email\nIf you need to use Email for alerting, create an alert instance in the alert instance management and select the Email plugin.\nThe following shows the Email configuration example:\n\n\n\n",
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/alert/email",
- "title": "Email",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Email"
- ]
- },
- {
- "content": "Telegram\nIf you need Telegram to alert, create an alert instance in the alert instance management, and choose the Telegram plugin.\nThe following shows the Telegram configuration example:\n\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nWebHook\nThe WebHook of Telegram when use robot to send message.\n\n\nbotToken\nThe access token of robot.\n\n\nchatId\nSub Telegram Channel.\n\n\nparseMode\nMessage parse type (support txt, markdown, markdownV2, html).\n [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/alert/telegram",
- "title": "Telegram",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Telegram"
- ]
- },
- {
- "content": "DingTalk\nIf you need to use DingTalk for alerting, create an alert instance in the alert instance management and select the DingTalk plugin.\nThe following shows the DingTalk configuration example:\n\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nWarning Type\nAlert on success or failure or both.\n\n\nWebHook\nThe format is: https://oapi.dingtalk.com/robot/send?access_token=XXXXXX\n\n\nKeyword\nCustom keywords for security settings.\n\n\nSecret\nSignatur [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/alert/dingtalk",
- "title": "DingTalk",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Ding Talk"
- ]
- },
- {
- "content": "Enterprise WeChat\nIf you need to use Enterprise WeChat to alert, create an alert instance in the alert instance management, and choose the WeChat plugin.\nThe following is the WeChat configuration example:\n\nSend Type\nThe parameter send.type corresponds to sending messages to Enterprise WeChat customized APP and group chat created by API respectively.\nAPP\nThe APP sends type means to notify the alert results via Enterprise WeChat customized APPs, supports sending mess [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/alert/enterprise-wechat",
- "title": "Enterprise WeChat",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Enterprise Wechat"
- ]
- },
- {
- "content": "Webex Teams\nIf you need to use Webex Teams to alert, create an alert instance in the alert instance management, and choose the WebexTeams plugin. You can pick private alert or room group chat alert.\nThe following is the WebexTeams configuration example:\n\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nbotAccessToken\nThe access token of robot.\n\n\nroomID\nThe ID of the room that receives message (only support one room ID).\n\n\ntoPersonId\nThe person I [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/alert/enterprise-webexteams",
- "title": "Webex Teams",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Enterprise Webexteams"
- ]
- },
- {
- "content": "Script\nIf you need to use Shell script for alerting, create an alert instance in the alert instance management and select the Script plugin.\nThe following shows the Script configuration example:\n\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nUser Params\nUser defined parameters will pass to the script.\n\n\nScript Path\nThe file location path in the server.\n\n\nType\nSupport Shell script.\n\n\n\nNote\nConsider the script file access privileges with t [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/alert/script",
- "title": "Script",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Script"
- ]
- },
- {
- "content": "HTTP\nIf you need to use Http script for alerting, create an alert instance in the alert instance management and select the Http plugin.\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nURL\nThe Http request URL needs to contain protocol, host, path and parameters if the method is GET.\n\n\nRequest Type\nSelect the request type from POST or GET.\n\n\nHeaders\nThe headers of the Http request in JSON format.\n\n\nBody\nThe request body of the Http request in [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/alert/http",
- "title": "HTTP",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Http"
- ]
- },
- {
- "content": "Feishu\nIf you need to use Feishu for alerting, create an alert instance in the alert instance management and select\nthe Feishu plugin.\nThe following shows the Feishu configuration example:\n\nParameter Configuration\n\n\nWebhook\n\nCopy the robot webhook URL shown below:\n\n\n\n\nFeishu：Use bots in groups\n",
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/alert/feishu",
- "title": "Feishu",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Feishu"
- ]
- },
- {
- "content": "Resource Center Introduction\nThe Resource Center is typically used for uploading files, UDF functions, and task group management. For a stand-alone\nenvironment, you can select the local file directory as the upload folder (this operation does not require Hadoop or HDFS deployment).\nOf course, you can also choose to upload to Hadoop or MinIO cluster. In this case, you need to have Hadoop (2.6+) or MinIOn and other related environments.\n",
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/resource/intro",
- "title": "Resource Center Introduction",
- "location": [
- "Introduction to Functions",
- "Resource",
- "Introduction"
- ]
- },
- {
- "content": "HDFS Resource Configuration\nWhen it is necessary to use the Resource Center to create or upload relevant files, all files and resources will be stored on HDFS. Therefore the following configuration is required.\nLocal File Resource Configuration\nFor a single machine, you can choose to use local file directory as the upload directory (no need to deploy Hadoop) by making the following configuration.\nConfiguring the common.properties\nConfigure the file in the following p [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/resource/configuration",
- "title": "HDFS Resource Configuration",
- "location": [
- "Introduction to Functions",
- "Resource",
- "Configuration"
- ]
- },
- {
- "content": "File Management\nWhen the third-party jar needs to be used in the scheduling process or the user needs to customize the script, the related operations can be completed on this page. The file types that can be created include: txt, log, sh, conf, py, java etc. And you can edit, rename, download and delete files.\n\n\nNote:\n\nWhen you manage files as admin, remember to set up tenant for admin first.\n\n\nBasic Operations\nCreate File\nThe file format supports the following [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/resource/file-manage",
- "title": "File Management",
- "location": [
- "Introduction to Functions",
- "Resource",
- "File Manage"
- ]
- },
- {
- "content": "UDF Manage\nResource Management\n\nThe resource management and file management functions are similar. The difference is that the resource management is the UDF upload function, and the file management uploads the user programs, scripts and configuration files.\nIt mainly includes the following operations: rename, download, delete, etc.\nUpload UDF resources: Same as uploading files.\n\nFunction Management\nCreate UDF function\nClick Create UDF Function, enter the UDF func [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/resource/udf-manage",
- "title": "UDF Manage",
- "location": [
- "Introduction to Functions",
- "Resource",
- "UDF Manage"
- ]
- },
- {
- "content": "Task Group Settings\nThe task group is mainly used to control the concurrency of task instances, and is designed to control the pressure of other resources (it can also control the pressure of the Hadoop cluster, the cluster will have queue control it). When creating a new task definition, you can configure the corresponding task group and configure the priority of the task running in the task group.\nTask Group Configuration\nCreate Task Group\n\nThe user clicks Resource [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/resource/task-group",
- "title": "Task Group Settings",
- "location": [
- "Introduction to Functions",
- "Resource",
- "Task Group Manage"
- ]
- },
- {
- "content": "Monitor\nService Management\n\nService management is mainly to monitor and display the health status and basic information of each service in the system.\n\nMaster Server\n\nMainly related to master information.\n\n\nWorker Server\n\nMainly related to worker information.\n\n\nDatabase\n\nMainly the health status of the DB.\n\n\nStatistics Management\nStatistics\n\n\n\n\nParameter\nDescription\n\n\n\n\nNumber of commands wait to be executed\nStatistics of the t_ds_command [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/monitor",
- "title": "Monitor",
- "location": [
- "Introduction to Functions",
- "Monitor"
- ]
- },
- {
- "content": "Security (Authorization System)\nOnly the administrator account in the security center has permission to operate, which includes queue management, tenant management, user management, alarm group management, worker group management, token management and other functions. In the user management module, you can authorize resources, data sources, projects, etc.\nAdministrator login, default username/password: admin/dolphinscheduler123\nCreate Queue\n\nThe queue is used when ex [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/security",
- "title": "Security (Authorization System)",
- "location": [
- "Introduction to Functions",
- "Security"
- ]
- },
- {
- "content": "Introduction\nApache DolphinScheduler has export some metrics to monitor the system. We use micrometer for the exporter facade, and\nthe default exporter is prometheus, more exporter is coming soon.\nQuick Start\nYou can add the following config in master/worker/alert/api's yaml file to open the metrics exporter.\nmetrics:\n enabled: true\n\nOnce you open the metrics exporter, you can access the metrics by the url: http://ip:port/actuator/prometheus\nThe exporter port is [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/metrics/metrics",
- "title": "Introduction",
- "location": [
- "Introduction to Functions",
- "Metrics",
- "Metrics"
- ]
- },
- {
- "content": "Data Quality\nIntroduction\nThe data quality task is used to check the data accuracy during the integration and processing of data. Data quality tasks in this release include single-table checking, single-table custom SQL checking, multi-table accuracy, and two-table value comparisons. The running environment of the data quality task is Spark 2.4.0, and other versions have not been verified, and users can verify by themselves.\nThe execution logic of the data quality task [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/data-quality",
- "title": "Data Quality",
- "location": [
- "Introduction to Functions",
- "Data Quality"
- ]
- },
- {
- "content": "DolphinScheduler Upgrade\nPrepare\nBackup Previous Version's Files and Database\nTo prevent data loss by some miss-operation, it is recommended to back up data before upgrading. The backup way according to your environment.\nDownload the Latest Version Installation Package\nDownload the latest binary distribute package from download and then put it in the different\ndirectory where current service running. And all below command is running in this directory.\nUpgrade\nStop [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/upgrade",
- "title": "DolphinScheduler Upgrade",
- "location": [
- "Introduction to Functions",
- "Upgrade"
- ]
- },
- {
- "content": "DolphinScheduler Expansion and Reduction\nExpansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine which locate the expansion master or worker node has already installed the scheduled service, check the [1.4 Modify configuration] and edit the configuration file [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Introduction to Functions",
- "Expansion and Reduction"
- ]
- },
- {
- "content": "Standalone\nStandalone only for quick experience for DolphinScheduler.\nIf you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow Standalone deployment. If you want to experience more complete functions and schedule massive tasks, we recommend you install follow pseudo-cluster deployment. If you want to deploy DolphinScheduler in production, we recommend you follow cluster deployment or Kubernetes deployment.\n\nNote: Standal [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/installation/standalone",
- "title": "Standalone",
- "location": [
- "Installation",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Pseudo-Cluster Deployment\nThe purpose of the pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, API server, are all on the same machine.\nIf you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow Standalone deployment. If you want to experience more complete functions and schedule massive tasks, we recommend you install follow[pseudo-clust [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/installation/pseudo-cluster",
- "title": "Pseudo-Cluster Deployment",
- "location": [
- "Installation",
- "Pseudo Cluster Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\nCluster deployment is to deploy the DolphinScheduler on multiple machines for running massive tasks in production.\nIf you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow Standalone deployment. If you want to experience more complete functions and schedule massive tasks, we recommend you install follow pseudo-cluster deployment. If you want to deploy DolphinScheduler in production, we recommend you foll [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/installation/cluster",
- "title": "Cluster Deployment",
- "location": [
- "Installation",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Kubernetes\nKubernetes deployment is DolphinScheduler deployment in a Kubernetes cluster, which can schedule massive tasks and can be used in production.\nIf you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow Standalone deployment. If you want to experience more complete functions and schedule massive tasks, we recommend you install follow pseudo-cluster deployment. If you want to deploy DolphinScheduler in [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/installation/kubernetes",
- "title": "QuickStart in Kubernetes",
- "location": [
- "Installation",
- "Kubernetes Deployment"
- ]
- },
- {
- "content": "System Architecture Design\nSystem Structure\nSystem Architecture Diagram\n\n \n \n System architecture diagram\n \n\nStart Process Activity Diagram\n\n \n \n Start process activity diagram\n \n\nArchitecture Description\n\n\nMasterServer\nMasterServer adopts a distributed and decentralized design concept. MasterServer is mainly responsible for DAG task segmentation, task submission monitoring, and monitoring the health status of other MasterServer and [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/architecture/design",
- "title": "System Architecture Design",
- "location": [
- "Architecture Guide",
- "Architecture Design"
- ]
- },
- {
- "content": "MetaData\nTable Schema\nsee sql files in dolphinscheduler/dolphinscheduler-dao/src/main/resources/sql\n\nE-R Diagram\nUser Queue DataSource\n\n\nOne tenant can own Multiple users.\nThe queue field in the t_ds_user table stores the queue_name information in the t_ds_queue table, t_ds_tenant stores queue information using queue_id column. During the execution of the process definition, the user queue has the highest priority. If the user queue is null, use the tenant queue. [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/architecture/metadata",
- "title": "MetaData",
- "location": [
- "Architecture Guide",
- "Metadata"
- ]
- },
- {
- "content": "\nConfiguration\nPreface\nThis document explains the DolphinScheduler application configurations.\nDirectory Structure\nThe directory structure of DolphinScheduler is as follows:\n├── LICENSE\n│\n├── NOTICE\n│\n├── licenses directory of licenses\n│\n├── bin directory of DolphinScheduler application commands, configrations scripts \n│ ├── dolphinscheduler-daemon.sh script to start or [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/architecture/configuration",
- "title": "Configuration",
- "location": [
- "Architecture Guide",
- "Configuration File"
- ]
- },
- {
- "content": "Task Structure\nOverall Tasks Storage Structure\nAll tasks in DolphinScheduler are saved in the t_ds_process_definition table.\nThe following shows the t_ds_process_definition table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not released, 1 released\n\n\ [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/architecture/task-structure",
- "title": "Task Structure",
- "location": [
- "Architecture Guide",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker Load Balancing Algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\n\n\nWeighted random (random)\n\n\nSmoothing polling (round-robin)\n\n\nLinear load (lower weight)\n\n\nThe default configurat [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/architecture/load-balance",
- "title": "Load Balance",
- "location": [
- "Architecture Guide",
- "Load Balance"
- ]
- },
- {
- "content": "Cache\nPurpose\nDue to the large database read operations during the master-server scheduling process. Such as read tables like tenant, user, processDefinition, etc. Operations stress read pressure to the DB, and slow down the entire core scheduling process.\nBy considering this part of the business data is a high-read and low-write scenario, a cache module is introduced to reduce the DB read pressure and speed up the core scheduling process.\nCache Settings\nspring:\n c [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/architecture/cache",
- "title": "Cache",
- "location": [
- "Architecture Guide",
- "Cache"
- ]
- },
- {
- "content": "Open API\nBackground\nGenerally, projects and processes are created through pages, but considering the integration with third-party systems requires API calls to manage projects and workflows.\nThe Operation Steps of DolphinScheduler API Calls\nCreate a Token\n\nLog in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.\n\n\n\nSelect the "Expiration time" (To [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/open-api",
- "title": "Open API",
- "location": [
- "API",
- "Open API"
- ]
- },
- {
- "content": "Health Check\nBackground\nHealth check are designed to provide a unique way to check the health of the dolphinscheduler service. It includes the health status of modules, such as DB, cache, network, etc.\nEndpoint\nAPI-Server\ncurl --request GET 'http://localhost:12345/dolphinscheduler/actuator/health'\n\n{"status":"UP","components":{"db":{"status":"UP","details":{"database":"H2 [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/guide/healthcheck",
- "title": "Health Check",
- "location": [
- "API",
- "Health Check"
- ]
- },
- {
- "content": "DSIP\nDolphinScheduler Improvement Proposal (DSIP) 是对 Apache DolphinScheduler 代码库进行的重大改进。它不是为了小修小补存在的，\nDSIP 的目的是通知社区完成或即将完成的重大变更。\n怎样的修改应该被认定为 DSIP\n\n任何重大的新功能、重大改进、引入或删除组件\n任何公共接口的任何重大变化，例如 API接口、web ui 巨大变化\n\n当一个 PR 或者 Issue 是否应该被认定为 DSIP 存疑时，如果有 committer 认为他应该纳入 DAIP 的范畴，那它就应该是 DSIP。\n我们使用 GitHub Issue 和 Apache 邮件列表来记录和保存 DSIP，想要了解更多相关信息，您可以跳转到 当前的 DSIPs 以及 past DSIPs\n作为 DSIP，它应该包含如下部分：\n\n在 dev@dolphinscheduler.apache.org 中有一个以 [DISCUSS][DSIP 为开头的邮件。\n有一个打了 " [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/DSIP",
- "title": "DSIP",
- "location": [
- "DSIP",
- "DSIP"
- ]
- },
- {
- "content": "Change Log\nFeature\n\nAdd Task group feature to limit the concurrency of taskInstance (#6422)\n[UI Next] A new version of the UI proposal (#7332)\n[UI Next] Data Quality (#8275)\nSupport custom timezone (#8310)\nDependent tasks can re-run automatically in the case of complement (#8373)\nOptimize workflowlieage query speed (#8499)\nAdd dataSource for Amazon Redshift (#8527)\nRefactor cli config dict mechnism (#8593)\nE2E test adapt M1 chip (#9039)\nSupport output paramete [...]
- "version": "3.0.0",
- "time": "2022-8-9",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "About DolphinScheduler\nApache DolphinScheduler provides a distributed and easy to expand visual workflow task scheduling open-source platform. It is suitable for enterprise-level scenarios. It provides a solution to visualize operation tasks, workflows, and the entire data processing procedures.\nApache DolphinScheduler aims to solve complex big data task dependencies and to trigger relationships in data OPS orchestration for various big data applications. Solves the int [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/about/introduction",
- "title": "About DolphinScheduler",
- "location": [
- "About",
- "Introduction"
- ]
- },
- {
- "content": "Features\nSimple to Use\n\nVisual DAG: User-friendly drag-and-drop workflow definition and facility for run-time control.\nModular Operation: Modularity facilitates easy customization and maintenance.\n\nRich Scenarios\n\nMultiple Task Type Support: Supports more than 10 task types, like Shell, MR, Spark, SQL, etc., with cross-language support making it easy to extend\nWorkflow Ops: Workflow can be timed, paused, resumed and stopped, enabling easy maintenance and control [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/about/features",
- "title": "Features",
- "location": [
- "About",
- "Features"
- ]
- },
- {
- "content": "Hardware Environment\nThis section briefs about the hardware requirements for DolphinScheduler. DolphinScheduler works as an open-source distributed workflow task scheduling system. It can deploy and run smoothly in Intel architecture server environments and mainstream virtualization environments. It also supports mainstream Linux operating system environments and ARM architecture.\nLinux Operating System Version Requirements\nThe Linux operating systems specified below c [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/about/hardware",
- "title": "Hardware Environment",
- "location": [
- "About",
- "Hardware Environment"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the\nform of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until\nthere are no subsequent nodes. Examples are as follows:\n\nProcess d [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/about/glossary",
- "title": "System Architecture Design",
- "location": [
- "About",
- "Glossary"
- ]
- },
- {
- "content": "Quick Start\nThis is a Quick Start guide to help you get a basic idea of working with Apache DolphinScheduler. Once you've gone through the tutorial video, we encourage you to find out more about Apache DolphinScheduler functions and examples in .\n \n \n\nAdministrator User Login\n\nAddress: http://localhost:12345/dolphinscheduler\nUsername and password: admin/dolphinscheduler123\n\n\nCreate a queue\n\nCreate a tenant\n\nCreate Ordinary Users\n\nCreate an alarm instance [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/start/quick-start",
- "title": "Quick Start",
- "location": [
- "Quick Start",
- "Quick Start"
- ]
- },
- {
- "content": "Docker Quick Start\nThere are three ways to start DolphinScheduler with Docker, Standalone-server is the way you\nfind if you just want to start and try DolphinScheduler as a beginner. docker-compose is for\nsome who want to deploy DolphinScheduler in small or event middle scale workflows in their daily work.\nUsing exists postgresql and zookeeper server is for users who want to reuse the database\nor zookeeper server already exists.\nPrepare\n\nDocker 1.13.1+\nDocker Com [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/start/docker",
- "title": "Docker Quick Start",
- "location": [
- "Quick Start",
- "Docker Deployment"
- ]
- },
- {
- "content": "Home Page\nApache DolphinScheduler home page lets you see task state statistics, workflow state statistics, and project statistics for all projects of users. It is the best way to observe status of your system as a whole as well as diving into individual process to check each status of task and task logs.\n\n",
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/homepage",
- "title": "Home Page",
- "location": [
- "Introduction to Functions",
- "Workflow Overview"
- ]
- },
- {
- "content": "Project\nThis page describes details regarding Project screen in Apache DolphinScheduler. Here, you will see all the functions which can be handled in this screen. The following table explains commonly used terms in Apache DolphinScheduler:\n\n\n\nGlossary\ndescription\n\n\n\n\nDAG\nTasks in a workflow are assembled in form of Directed Acyclic Graph (DAG). A topological traversal is performed from nodes with zero degrees of entry until there are no subsequent nodes.\n\n\n [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/project/project-list",
- "title": "Project",
- "location": [
- "Introduction to Functions",
- "Project",
- "Project List"
- ]
- },
- {
- "content": "Workflow Definition\nOverview\nWork Definition screen shows list of existing workflows and utility to create or import workflows.\n\nCreate workflow definition\nClick Project Management -> Workflow -> Workflow Definition, enter the Workflow Definition page, and click the Create Workflow button to enter the workflow DAG edit page, as shown in the following figure:\n\nCreate Tasks to Workflow\nDrag from the toolbar to the canvas, to add a shell task to the canvas, as [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/project/workflow-definition",
- "title": "Workflow Definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Definition"
- ]
- },
- {
- "content": "Workflow Instance\nView Workflow Instance\nClick Project Management -> Workflow -> Workflow Instance, enter the Workflow Instance page, as shown in the following figure:\n\nClick the workflow name to enter the DAG view page, and check the task execution status, as shown in the following figure:\n\nView Task Log\nEnter the workflow instance page, click the workflow name, enter the DAG view page, double-click the task node, as shown in the following figure:\n\nClick & [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/project/workflow-instance",
- "title": "Workflow Instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Instance"
- ]
- },
- {
- "content": "Task Instance\nCreate Task Instance\nClick Project Management -> Workflow -> Task Instance to enter the task instance page, as shown in the figure below, click the name of the workflow instance to jump to the DAG diagram of the workflow instance to view the task status.\n\nView Log\nClick the View Log button in the operation column to view the log of the task execution\n\n",
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/project/task-instance",
- "title": "Task Instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Instance"
- ]
- },
- {
- "content": "Task Definition\nTask definition allows to modify or operate tasks at the task level rather than modifying them in the workflow definition.\nWe already have workflow level task editor in workflow definition which you can click the specific\nworkflow and then edit its task definition. It is depressing when you want to edit the task definition but do not remember\nwhich workflow it belongs to. So we decide to add Task Definition view under Task menu.\n\nIn this view, you ca [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/project/task-definition",
- "title": "Task Definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Definition"
- ]
- },
- {
- "content": "Shell\nOverview\nShell task type, used to create a shell type task and execute a series of shell scripts. When the worker executes this task, a temporary shell script is generated and executed using the linux user with the same name as the tenant.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameters\n\n\n\nParamet [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/shell",
- "title": "Shell",
- "location": [
- "Introduction to Functions",
- "Task",
- "Shell"
- ]
- },
- {
- "content": "SubProcess Node\nOverview\nThe sub-process node is to execute an external workflow definition as a task node.\nCreate Task\n\nClick Project Management-> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar task node to canvas to create a new SubProcess task.\n\nTask Parameter\n\n\n\nParameter\nDescription\n\n\n\n\nNode name\nUnique name of node in workflow definition.\n\n\nRun flag\nIdentifi [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/sub-process",
- "title": "SubProcess Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "SubProcess"
- ]
- },
- {
- "content": "Dependent\nOverview\nDependent nodes are dependency check nodes. For example, process A depends on the successful execution of process B from yesterday, and the dependent node will check whether process B run successful yesterday.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar task node to canvas.\n\nTask Parameter\n\n\n\nParameter\nDescripti [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/dependent",
- "title": "Dependent",
- "location": [
- "Introduction to Functions",
- "Task",
- "Dependent"
- ]
- },
- {
- "content": "Stored Procedure\n\nExecute the stored procedure according to the selected DataSource.\n\n\nDrag from the toolbar task node into the canvas, as shown in the figure below:\n\n\n \n \nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nNode Name\nSet the name of the task. Node names within a workflow definition are unique.\n\n\nRun flag\nIndicates whether the node can be scheduled normally. If it is not necessary to execute, you can turn on the prohibiting execution s [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/stored-procedure",
- "title": "Stored Procedure",
- "location": [
- "Introduction to Functions",
- "Task",
- "Stored Procedure"
- ]
- },
- {
- "content": "SQL\nOverview\nSQL task type used to connect to databases and execute SQL.\nCreate DataSource\nRefer to datasource-setting DataSource Center section\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameter\n\n\n\nParameter\nDescription\n\n\n\n\nData source\nSelect the corresponding DataSource.\n\n\nSQL type\nSupports qu [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/sql",
- "title": "SQL",
- "location": [
- "Introduction to Functions",
- "Task",
- "SQL"
- ]
- },
- {
- "content": "Spark Node\nOverview\nSpark task type for executing Spark application. When executing the Spark task, the worker will submits a job to the Spark cluster by following commands:\n(1) spark submit method to submit tasks. See spark-submit for more details.\n(2) spark sql method to submit tasks. See spark sql for more details.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing pag [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/spark",
- "title": "Spark Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Spark"
- ]
- },
- {
- "content": "MapReduce Node\nOverview\nMapReduce(MR) task type used for executing MapReduce programs. For MapReduce nodes, the worker submits the task by using the Hadoop command hadoop jar. See Hadoop Command Manual for more details.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameters\nGeneral\n\n\n\nParameter\nDescription\n\ [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/map-reduce",
- "title": "MapReduce Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "MapReduce"
- ]
- },
- {
- "content": "Python Node\nOverview\nUse Python Task to create a python-type task and execute python scripts. When the worker executes Python Task,\nit will generate a temporary python script, and executes the script by the Linux user with the same name as the tenant.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameter\n\n\n\nPa [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/python",
- "title": "Python Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Python"
- ]
- },
- {
- "content": "Flink Node\nOverview\nFlink task type, used to execute Flink programs. For Flink nodes:\n\n\nWhen the program type is Java, Scala or Python, the worker submits the task flink run using the Flink command. See flink cli for more details.\n\n\nWhen the program type is SQL, the worker submit tasks using sql-client.sh. See flink sql client for more details.\n\n\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/flink",
- "title": "Flink Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Flink"
- ]
- },
- {
- "content": "HTTP Node\nOverview\nThis node is used to perform http type tasks such as the common POST and GET request types, and also supports http request validation and other functions.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nNode Name\nSet the name of the task. [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/http",
- "title": "HTTP Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "HTTP"
- ]
- },
- {
- "content": "DataX\nOverview\nDataX task type for executing DataX programs. For DataX nodes, the worker will execute ${DATAX_HOME}/bin/datax.py to analyze the input json file.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nNode name\nThe node name in a workflow definition [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/datax",
- "title": "DataX",
- "location": [
- "Introduction to Functions",
- "Task",
- "DataX"
- ]
- },
- {
- "content": "Pigeon\nOverview\nPigeon is a task used to trigger remote tasks, acquire logs or status by calling remote WebSocket service. It is DolphinScheduler uses a remote WebSocket service to call tasks.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas to create a new Pigeon task.\n\nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nN [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/pigeon",
- "title": "Pigeon",
- "location": [
- "Introduction to Functions",
- "Task",
- "Pigeon"
- ]
- },
- {
- "content": "Conditions\nCondition is a conditional node, that determines which downstream task should run based on the condition of the upstream task. Currently, the Conditions support multiple upstream tasks, but only two downstream tasks. When the number of upstream tasks exceeds one, achieve complex upstream dependencies by through and and or operators.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to en [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/conditions",
- "title": "Conditions",
- "location": [
- "Introduction to Functions",
- "Task",
- "Conditions"
- ]
- },
- {
- "content": "Switch\nOverview\nThe switch is a conditional judgment node, decide the branch executes according to the value of global variable and the expression result written by the user.\nNote: Execute expressions using javax.script.ScriptEngine.eval.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar task node to canvas to create a task.\n\nNote: After cr [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/switch",
- "title": "Switch",
- "location": [
- "Introduction to Functions",
- "Task",
- "Switch"
- ]
- },
- {
- "content": "Amazon EMR\nOverview\nAmazon EMR task type, for creating EMR clusters on AWS and running computing tasks. Using aws-java-sdk in the background code, to transfer JSON parameters to RunJobFlowRequest object and submit to AWS.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, click the Create Workflow button to enter the DAG editing page.\nDrag AmazonEMR task from the toolbar to the artboard to complete the creation.\n\nTask Parameters\n [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/emr",
- "title": "Amazon EMR",
- "location": [
- "Introduction to Functions",
- "Task",
- "Amazon EMR"
- ]
- },
- {
- "content": "Apache Zeppelin\nOverview\nUse Zeppelin Task to create a zeppelin-type task and execute zeppelin notebook paragraphs. When the worker executes Zeppelin Task,\nit will call Zeppelin Client API to trigger zeppelin notebook paragraph. Click here for details about Apache Zeppelin Notebook.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the can [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/task/zeppelin",
- "title": "Apache Zeppelin",
- "location": [
- "Introduction to Functions",
- "Task",
- "Apache Zeppelin"
- ]
- },
- {
- "content": "Built-in Parameter\nBasic Built-in Parameter\n\n\n\nVariable\nDeclaration Method\nMeaning\n\n\n\n\nsystem.biz.date\n${system.biz.date}\nThe day before the schedule time of the daily scheduling instance, the format is yyyyMMdd\n\n\nsystem.biz.curdate\n${system.biz.curdate}\nThe schedule time of the daily scheduling instance, the format is yyyyMMdd\n\n\nsystem.datetime\n${system.datetime}\nThe schedule time of the daily scheduling instance, the format is yyyyMMddHHmmss\n\n\ [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/parameter/built-in",
- "title": "Built-in Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Built-in Parameter"
- ]
- },
- {
- "content": "Global Parameter\nScope\nGlobal parameters are parameters that are valid for all task nodes of the entire workflow. It can be configured on the workflow definition page.\nUsage\nThe specific use method can be determined according to the actual production situation. This example uses a shell task to print out the date value of yesterday.\nExample\nCreate a Shell task\nCreate a shell task and enter echo ${dt} in the script content. In this case, dt is the global parameter w [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/parameter/global",
- "title": "Global Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Global Parameter"
- ]
- },
- {
- "content": "Local Parameter\nScope\nParameters configured on the task definition page, the scope of this parameter is inside this task only. But if you configure according to Refer to Parameter Context, it could pass to downstream tasks.\nUsage\n\nIf you want to use parameter in single task, see local parameter with custom parameter.\nIf you want to use set parameter in task and use them in downstream tasks:\n\nFor simple use without using custom parameter, see export parameter via s [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/parameter/local",
- "title": "Local Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Local Parameter"
- ]
- },
- {
- "content": "Parameter Context\nDolphinScheduler provides the ability to refer to each other between parameters, including local parameters refer to global parameters and upstream and downstream parameter transfer. Due to the existence of references, it involves the priority of parameters when the parameter names are the same. see also Parameter Priority\nLocal Task Refers to Global Parameter\nThe premise of local tasks referring global parameters is that you have already defined Glob [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/parameter/context",
- "title": "Parameter Context",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Context"
- ]
- },
- {
- "content": "Parameter Priority\nDolphinScheduler has three parameter types:\n\nGlobal Parameter: parameters defined at the workflow define page.\nParameter Context: parameters passed by upstream task nodes.\nLocal Parameter: parameters belong to its node, which is the parameters defined by the user in [Custom Parameters].\n\nThe user can define part of the parameters when creating workflow definitions.\nAs there are multiple sources of the parameter value, it will raise parameter pri [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/parameter/priority",
- "title": "Parameter Priority",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Priority"
- ]
- },
- {
- "content": "MySQL\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect MYSQL.\n\n\nDatasource name\nEnter the name of the DataSource.\n\n\nDescription\nEnter a description of the DataSource.\n\n\nIP/Host Name\nEnter the MYSQL service IP.\n\n\nPort\nEnter the MYSQL service port.\n\n\nUsername\nSet the username for MYSQL connection.\n\n\nPassword\nSet the password for MYSQL connection.\n\n\nDatabase name\nEnter the database name of the MYSQL connection.\n [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/datasource/mysql",
- "title": "MySQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "MySQL"
- ]
- },
- {
- "content": "PostgreSQL\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect POSTGRESQL.\n\n\nDatasource name\nEnter the name of the DataSource.\n\n\nDescription\nEnter a description of the DataSource.\n\n\nIP/Host Name\nEnter the PostgreSQL service IP.\n\n\nPort\nEnter the PostgreSQL service port.\n\n\nUsername\nSet the username for PostgreSQL connection.\n\n\nPassword\nSet the password for PostgreSQL connection.\n\n\nDatabase name\nEnter the database n [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/datasource/postgresql",
- "title": "PostgreSQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "PostgreSQL"
- ]
- },
- {
- "content": "HIVE\nUse HiveServer2\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect HIVE.\n\n\nDatasource name\nEnter the name of the DataSource.\n\n\nDescription\nEnter a description of the DataSource.\n\n\nIP/Host Name\nEnter the HIVE service IP.\n\n\nPort\nEnter the HIVE service port.\n\n\nUsername\nSet the username for HIVE connection.\n\n\nPassword\nSet the password for HIVE connection.\n\n\nDatabase name\nEnter the database name of the HIVE con [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/datasource/hive",
- "title": "HIVE",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "HIVE"
- ]
- },
- {
- "content": "Spark\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect Spark.\n\n\nDatasource name\nEnter the name of the DataSource.\n\n\nDescription\nEnter a description of the DataSource.\n\n\nIP/Host Name\nEnter the Spark service IP.\n\n\nPort\nEnter the Spark service port.\n\n\nUsername\nSet the username for Spark connection.\n\n\nPassword\nSet the password for Spark connection.\n\n\nDatabase name\nEnter the database name of the Spark connection.\n [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/datasource/spark",
- "title": "Spark",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Spark"
- ]
- },
- {
- "content": "Presto\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect Presto.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the Presto service IP.\n\n\nPort\nEnter the Presto service port.\n\n\nUsername\nSet the username for Presto connection.\n\n\nPassword\nSet the password for Presto connection.\n\n\nDatabase Name\nEnter the database name of the Presto connec [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/datasource/presto",
- "title": "Presto",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Presto"
- ]
- },
- {
- "content": "SQL Server\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect SQLSERVER.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the SQLSERVER service IP.\n\n\nPort\nEnter the SQLSERVER service port.\n\n\nUsername\nSet the username for SQLSERVER connection.\n\n\nPassword\nSet the password for SQLSERVER connection.\n\n\nDatabase Name\nEnter the database name o [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/datasource/sqlserver",
- "title": "SQL Server",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "SQL SERVER"
- ]
- },
- {
- "content": "Amazon Redshift\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect Redshift.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the Redshift service IP.\n\n\nPort\nEnter the Redshift service port.\n\n\nUsername\nSet the username for Redshift connection.\n\n\nPassword\nSet the password for Redshift connection.\n\n\nDatabase Name\nEnter the database name o [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/datasource/redshift",
- "title": "Amazon Redshift",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Amazon Redshift"
- ]
- },
- {
- "content": "ClickHouse\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect CLICKHOUSE.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the CLICKHOUSE service IP.\n\n\nPort\nEnter the CLICKHOUSE service port.\n\n\nUsername\nSet the username for CLICKHOUSE connection.\n\n\nPassword\nSet the password for CLICKHOUSE connection.\n\n\nDatabase Name\nEnter the database n [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/datasource/clickhouse",
- "title": "ClickHouse",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "ClickHouse"
- ]
- },
- {
- "content": "DB2\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect DB2.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the DB2 service IP.\n\n\nPort\nEnter the DB2 service port.\n\n\nUsername\nSet the username for DB2 connection.\n\n\nPassword\nSet the password for DB2 connection.\n\n\nDatabase Name\nEnter the database name of the DB2 connection.\n\n\njdbc conne [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/datasource/db2",
- "title": "DB2",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "IBM DB2"
- ]
- },
- {
- "content": "Oracle\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect Oracle.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the Oracle service IP.\n\n\nPort\nEnter the Oracle service port.\n\n\nUsername\nSet the username for Oracle connection.\n\n\nPassword\nSet the password for Oracle connection.\n\n\nDatabase Name\nEnter the database name of the Oracle connec [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/datasource/oracle",
- "title": "Oracle",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Oracle"
- ]
- },
- {
- "content": "Alert Component User Guide\nAlert Plugins and Alert Groups Creation\nIn version 2.0.0, users need to create alert instances, and needs to choose an alarm policy when defining an alarm instance, there are three options: send if the task succeeds, send on failure, and send on both success and failure. when the workflow or task is executed, if an alarm is triggered, calling the alarm instance send method needs a logical judgment, which matches the alarm instance with the tas [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/alert/alert_plugin_user_guide",
- "title": "Alert Component User Guide",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Alert Component User Guide "
- ]
- },
- {
- "content": "Email\nIf you need to use Email for alerting, create an alert instance in the alert instance management and select the Email plugin.\nThe following shows the Email configuration example:\n\n\n\n",
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/alert/email",
- "title": "Email",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Email"
- ]
- },
- {
- "content": "Telegram\nIf you need Telegram to alert, create an alert instance in the alert instance management, and choose the Telegram plugin.\nThe following shows the Telegram configuration example:\n\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nWebHook\nThe WebHook of Telegram when use robot to send message.\n\n\nbotToken\nThe access token of robot.\n\n\nchatId\nSub Telegram Channel.\n\n\nparseMode\nMessage parse type (support txt, markdown, markdownV2, html).\n [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/alert/telegram",
- "title": "Telegram",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Telegram"
- ]
- },
- {
- "content": "DingTalk\nIf you need to use DingTalk for alerting, create an alert instance in the alert instance management and select the DingTalk plugin.\nThe following shows the DingTalk configuration example:\n\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nWarning Type\nAlert on success or failure or both.\n\n\nWebHook\nThe format is: https://oapi.dingtalk.com/robot/send?access_token=XXXXXX\n\n\nKeyword\nCustom keywords for security settings.\n\n\nSecret\nSignatur [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/alert/dingtalk",
- "title": "DingTalk",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Ding Talk"
- ]
- },
- {
- "content": "Enterprise WeChat\nIf you need to use Enterprise WeChat to alert, create an alert instance in the alert instance management, and choose the WeChat plugin.\nThe following is the WeChat configuration example:\n\nSend Type\nThe parameter send.type corresponds to sending messages to Enterprise WeChat customized APP and group chat created by API respectively.\nAPP\nThe APP sends type means to notify the alert results via Enterprise WeChat customized APPs, supports sending mess [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/alert/enterprise-wechat",
- "title": "Enterprise WeChat",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Enterprise Wechat"
- ]
- },
- {
- "content": "Webex Teams\nIf you need to use Webex Teams to alert, create an alert instance in the alert instance management, and choose the WebexTeams plugin. You can pick private alert or room group chat alert.\nThe following is the WebexTeams configuration example:\n\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nbotAccessToken\nThe access token of robot.\n\n\nroomID\nThe ID of the room that receives message (only support one room ID).\n\n\ntoPersonId\nThe person I [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/alert/enterprise-webexteams",
- "title": "Webex Teams",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Enterprise Webexteams"
- ]
- },
- {
- "content": "Script\nIf you need to use Shell script for alerting, create an alert instance in the alert instance management and select the Script plugin.\nThe following shows the Script configuration example:\n\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nUser Params\nUser defined parameters will pass to the script.\n\n\nScript Path\nThe file location path in the server.\n\n\nType\nSupport Shell script.\n\n\n\nNote\nConsider the script file access privileges with t [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/alert/script",
- "title": "Script",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Script"
- ]
- },
- {
- "content": "HTTP\nIf you need to use Http script for alerting, create an alert instance in the alert instance management and select the Http plugin.\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nURL\nThe Http request URL needs to contain protocol, host, path and parameters if the method is GET.\n\n\nRequest Type\nSelect the request type from POST or GET.\n\n\nHeaders\nThe headers of the Http request in JSON format.\n\n\nBody\nThe request body of the Http request in [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/alert/http",
- "title": "HTTP",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Http"
- ]
- },
- {
- "content": "Feishu\nIf you need to use Feishu for alerting, create an alert instance in the alert instance management and select\nthe Feishu plugin.\nThe following shows the Feishu configuration example:\n\nParameter Configuration\n\n\nWebhook\n\nCopy the robot webhook URL shown below:\n\n\n\n\nFeishu：Use bots in groups\n",
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/alert/feishu",
- "title": "Feishu",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Feishu"
- ]
- },
- {
- "content": "Resource Center Introduction\nThe Resource Center is typically used for uploading files, UDF functions, and task group management. For a stand-alone\nenvironment, you can select the local file directory as the upload folder (this operation does not require Hadoop or HDFS deployment).\nOf course, you can also choose to upload to Hadoop or MinIO cluster. In this case, you need to have Hadoop (2.6+) or MinIOn and other related environments.\n",
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/resource/intro",
- "title": "Resource Center Introduction",
- "location": [
- "Introduction to Functions",
- "Resource",
- "Introduction"
- ]
- },
- {
- "content": "Resource Center Configuration\n\nYou could use Resource Center to upload text files, UDFs and other task-related files.\nYou could configure Resource Center to use distributed file system like Hadoop (2.6+), MinIO cluster or remote storage products like AWS S3, Alibaba Cloud OSS, etc.\nYou could configure Resource Center to use local file system. If you deploy DolphinScheduler in Standalone mode, you could configure it to use local file system for Resouce Center without t [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/resource/configuration",
- "title": "Resource Center Configuration",
- "location": [
- "Introduction to Functions",
- "Resource",
- "Configuration"
- ]
- },
- {
- "content": "File Management\nWhen the third-party jar needs to be used in the scheduling process or the user needs to customize the script, the related operations can be completed on this page. The file types that can be created include: txt, log, sh, conf, py, java etc. And you can edit, rename, download and delete files.\n\n\nNote:\n\nWhen you manage files as admin, remember to set up tenant for admin first.\n\n\nBasic Operations\nCreate File\nThe file format supports the following [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/resource/file-manage",
- "title": "File Management",
- "location": [
- "Introduction to Functions",
- "Resource",
- "File Manage"
- ]
- },
- {
- "content": "UDF Manage\nResource Management\n\nThe resource management and file management functions are similar. The difference is that the resource management is the UDF upload function, and the file management uploads the user programs, scripts and configuration files.\nIt mainly includes the following operations: rename, download, delete, etc.\nUpload UDF resources: Same as uploading files.\n\nFunction Management\nCreate UDF function\nClick Create UDF Function, enter the UDF func [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/resource/udf-manage",
- "title": "UDF Manage",
- "location": [
- "Introduction to Functions",
- "Resource",
- "UDF Manage"
- ]
- },
- {
- "content": "Task Group Settings\nThe task group is mainly used to control the concurrency of task instances, and is designed to control the pressure of other resources (it can also control the pressure of the Hadoop cluster, the cluster will have queue control it). When creating a new task definition, you can configure the corresponding task group and configure the priority of the task running in the task group.\nTask Group Configuration\nCreate Task Group\n\nThe user clicks Resource [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/resource/task-group",
- "title": "Task Group Settings",
- "location": [
- "Introduction to Functions",
- "Resource",
- "Task Group Manage"
- ]
- },
- {
- "content": "Monitor\nService Management\n\nService management is mainly to monitor and display the health status and basic information of each service in the system.\n\nMaster Server\n\nMainly related to master information.\n\n\nWorker Server\n\nMainly related to worker information.\n\n\nDatabase\n\nMainly the health status of the DB.\n\n\nStatistics Management\nStatistics\n\n\n\n\nParameter\nDescription\n\n\n\n\nNumber of commands wait to be executed\nStatistics of the t_ds_command [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/monitor",
- "title": "Monitor",
- "location": [
- "Introduction to Functions",
- "Monitor"
- ]
- },
- {
- "content": "Security (Authorization System)\nOnly the administrator account in the security center has permission to operate, which includes queue management, tenant management, user management, alarm group management, worker group management, token management and other functions. In the user management module, you can authorize resources, data sources, projects, etc.\nAdministrator login, default username/password: admin/dolphinscheduler123\nCreate Queue\n\nThe queue is used when ex [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/security",
- "title": "Security (Authorization System)",
- "location": [
- "Introduction to Functions",
- "Security"
- ]
- },
- {
- "content": "Introduction\nApache DolphinScheduler has export some metrics to monitor the system. We use micrometer for the exporter facade, and\nthe default exporter is prometheus, more exporter is coming soon.\nQuick Start\nYou can add the following config in master/worker/alert/api's yaml file to open the metrics exporter.\nmetrics:\n enabled: true\n\nOnce you open the metrics exporter, you can access the metrics by the url: http://ip:port/actuator/prometheus\nThe exporter port is [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/metrics/metrics",
- "title": "Introduction",
- "location": [
- "Introduction to Functions",
- "Metrics",
- "Metrics"
- ]
- },
- {
- "content": "Data Quality\nIntroduction\nThe data quality task is used to check the data accuracy during the integration and processing of data. Data quality tasks in this release include single-table checking, single-table custom SQL checking, multi-table accuracy, and two-table value comparisons. The running environment of the data quality task is Spark 2.4.0, and other versions have not been verified, and users can verify by themselves.\nThe execution logic of the data quality task [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/data-quality",
- "title": "Data Quality",
- "location": [
- "Introduction to Functions",
- "Data Quality"
- ]
- },
- {
- "content": "DolphinScheduler Upgrade\nPrepare\nBackup Previous Version's Files and Database\nTo prevent data loss by some miss-operation, it is recommended to back up data before upgrading. The backup way according to your environment.\nDownload the Latest Version Installation Package\nDownload the latest binary distribute package from download and then put it in the different\ndirectory where current service running. And all below command is running in this directory.\nUpgrade\nStop [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/upgrade",
- "title": "DolphinScheduler Upgrade",
- "location": [
- "Introduction to Functions",
- "Upgrade"
- ]
- },
- {
- "content": "DolphinScheduler Expansion and Reduction\nExpansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine which locate the expansion master or worker node has already installed the scheduled service, check the [1.4 Modify configuration] and edit the configuration file [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Introduction to Functions",
- "Expansion and Reduction"
- ]
- },
- {
- "content": "Standalone\nStandalone only for quick experience for DolphinScheduler.\nIf you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow Standalone deployment. If you want to experience more complete functions and schedule massive tasks, we recommend you install follow pseudo-cluster deployment. If you want to deploy DolphinScheduler in production, we recommend you follow cluster deployment or Kubernetes deployment.\n\nNote: Standal [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/installation/standalone",
- "title": "Standalone",
- "location": [
- "Installation",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Pseudo-Cluster Deployment\nThe purpose of the pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, API server, are all on the same machine.\nIf you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow Standalone deployment. If you want to experience more complete functions and schedule massive tasks, we recommend you install follow[pseudo-clust [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/installation/pseudo-cluster",
- "title": "Pseudo-Cluster Deployment",
- "location": [
- "Installation",
- "Pseudo Cluster Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\nCluster deployment is to deploy the DolphinScheduler on multiple machines for running massive tasks in production.\nIf you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow Standalone deployment. If you want to experience more complete functions and schedule massive tasks, we recommend you install follow pseudo-cluster deployment. If you want to deploy DolphinScheduler in production, we recommend you foll [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/installation/cluster",
- "title": "Cluster Deployment",
- "location": [
- "Installation",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Kubernetes\nKubernetes deployment is DolphinScheduler deployment in a Kubernetes cluster, which can schedule massive tasks and can be used in production.\nIf you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow Standalone deployment. If you want to experience more complete functions and schedule massive tasks, we recommend you install follow pseudo-cluster deployment. If you want to deploy DolphinScheduler in [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/installation/kubernetes",
- "title": "QuickStart in Kubernetes",
- "location": [
- "Installation",
- "Kubernetes Deployment"
- ]
- },
- {
- "content": "System Architecture Design\nSystem Structure\nSystem Architecture Diagram\n\n \n \n System architecture diagram\n \n\nStart Process Activity Diagram\n\n \n \n Start process activity diagram\n \n\nArchitecture Description\n\n\nMasterServer\nMasterServer adopts a distributed and decentralized design concept. MasterServer is mainly responsible for DAG task segmentation, task submission monitoring, and monitoring the health status of other MasterServer and [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/architecture/design",
- "title": "System Architecture Design",
- "location": [
- "Architecture Guide",
- "Architecture Design"
- ]
- },
- {
- "content": "MetaData\nTable Schema\nsee sql files in dolphinscheduler/dolphinscheduler-dao/src/main/resources/sql\n\nE-R Diagram\nUser Queue DataSource\n\n\nOne tenant can own Multiple users.\nThe queue field in the t_ds_user table stores the queue_name information in the t_ds_queue table, t_ds_tenant stores queue information using queue_id column. During the execution of the process definition, the user queue has the highest priority. If the user queue is null, use the tenant queue. [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/architecture/metadata",
- "title": "MetaData",
- "location": [
- "Architecture Guide",
- "Metadata"
- ]
- },
- {
- "content": "\nConfiguration\nPreface\nThis document explains the DolphinScheduler application configurations.\nDirectory Structure\nThe directory structure of DolphinScheduler is as follows:\n├── LICENSE\n│\n├── NOTICE\n│\n├── licenses directory of licenses\n│\n├── bin directory of DolphinScheduler application commands, configrations scripts \n│ ├── dolphinscheduler-daemon.sh script to start or [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/architecture/configuration",
- "title": "Configuration",
- "location": [
- "Architecture Guide",
- "Configuration File"
- ]
- },
- {
- "content": "Task Structure\nOverall Tasks Storage Structure\nAll tasks in DolphinScheduler are saved in the t_ds_process_definition table.\nThe following shows the t_ds_process_definition table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not released, 1 released\n\n\ [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/architecture/task-structure",
- "title": "Task Structure",
- "location": [
- "Architecture Guide",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker Load Balancing Algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\n\n\nWeighted random (random)\n\n\nSmoothing polling (round-robin)\n\n\nLinear load (lower weight)\n\n\nThe default configurat [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/architecture/load-balance",
- "title": "Load Balance",
- "location": [
- "Architecture Guide",
- "Load Balance"
- ]
- },
- {
- "content": "Cache\nPurpose\nDue to the large database read operations during the master-server scheduling process. Such as read tables like tenant, user, processDefinition, etc. Operations stress read pressure to the DB, and slow down the entire core scheduling process.\nBy considering this part of the business data is a high-read and low-write scenario, a cache module is introduced to reduce the DB read pressure and speed up the core scheduling process.\nCache Settings\nspring:\n c [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/architecture/cache",
- "title": "Cache",
- "location": [
- "Architecture Guide",
- "Cache"
- ]
- },
- {
- "content": "Open API\nBackground\nGenerally, projects and processes are created through pages, but considering the integration with third-party systems requires API calls to manage projects and workflows.\nThe Operation Steps of DolphinScheduler API Calls\nCreate a Token\n\nLog in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.\n\n\n\nSelect the "Expiration time" (To [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/open-api",
- "title": "Open API",
- "location": [
- "API",
- "Open API"
- ]
- },
- {
- "content": "Health Check\nBackground\nHealth check are designed to provide a unique way to check the health of the dolphinscheduler service. It includes the health status of modules, such as DB, cache, network, etc.\nEndpoint\nAPI-Server\ncurl --request GET 'http://localhost:12345/dolphinscheduler/actuator/health'\n\n{"status":"UP","components":{"db":{"status":"UP","details":{"database":"H2 [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/guide/healthcheck",
- "title": "Health Check",
- "location": [
- "API",
- "Health Check"
- ]
- },
- {
- "content": "DSIP\nDolphinScheduler Improvement Proposal(DSIP) introduce major improvements to the Apache DolphinScheduler codebase. It is\nnot for small incremental improvements, and the purpose of DSIP is to notice and inform community the finished or coming\nbig feature for Apache DolphinScheduler.\nWhat is considered as DSIP\n\nAny major new feature, major improvement, introduce or remove components\nAny major change of public interfaces, such as API endpoints, web ui huge change\ [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/DSIP",
- "title": "DSIP",
- "location": [
- "DSIP",
- "DSIP"
- ]
- },
- {
- "content": "Change Log\nBugfix\n\n[Improvement][ui] Unified local parameters UI #11190\n[Fix][ui] Fix bug where icons are displayed in one line. #11320\n[Improvement-11386][ui] Concise the logic available for task action buttons #11419\n[Fix][ui] Fix the preTask options are unavailable when creating a tas… #11501\nS3 Resource center supports bucket customization (#12022)\nfix status check issue (#11580) (#12030)\n[LDAP] Config value should return real null instead of 'null' string (# [...]
- "version": "3.0.1",
- "time": "2022-9-26",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "About DolphinScheduler\nApache DolphinScheduler provides a distributed and easy to expand visual workflow task scheduling open-source platform. It is suitable for enterprise-level scenarios. It provides a solution to visualize operation tasks, workflows, and the entire data processing procedures.\nApache DolphinScheduler aims to solve complex big data task dependencies and to trigger relationships in data OPS orchestration for various big data applications. Solves the int [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/about/introduction",
- "title": "About DolphinScheduler",
- "location": [
- "About",
- "Introduction"
- ]
- },
- {
- "content": "Features\nSimple to Use\n\nVisual DAG: User-friendly drag-and-drop workflow definition and facility for run-time control.\nModular Operation: Modularity facilitates easy customization and maintenance.\n\nRich Scenarios\n\nMultiple Task Type Support: Supports more than 10 task types, like Shell, MR, Spark, SQL, etc., with cross-language support making it easy to extend\nWorkflow Ops: Workflow can be timed, paused, resumed and stopped, enabling easy maintenance and control [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/about/features",
- "title": "Features",
- "location": [
- "About",
- "Features"
- ]
- },
- {
- "content": "Hardware Environment\nThis section briefs about the hardware requirements for DolphinScheduler. DolphinScheduler works as an open-source distributed workflow task scheduling system. It can deploy and run smoothly in Intel architecture server environments and mainstream virtualization environments. It also supports mainstream Linux operating system environments and ARM architecture.\nLinux Operating System Version Requirements\nThe Linux operating systems specified below c [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/about/hardware",
- "title": "Hardware Environment",
- "location": [
- "About",
- "Hardware Environment"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the\nform of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until\nthere are no subsequent nodes. Examples are as follows:\n\nProcess d [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/about/glossary",
- "title": "System Architecture Design",
- "location": [
- "About",
- "Glossary"
- ]
- },
- {
- "content": "Quick Start\nThis is a Quick Start guide to help you get a basic idea of working with Apache DolphinScheduler. Once you've gone through the tutorial video, we encourage you to find out more about Apache DolphinScheduler functions and examples in .\n \n \n\nAdministrator User Login\n\nAddress: http://localhost:12345/dolphinscheduler\nUsername and password: admin/dolphinscheduler123\n\n\nCreate a queue\n\nCreate a tenant\n\nCreate Ordinary Users\n\nCreate an alarm instance [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/start/quick-start",
- "title": "Quick Start",
- "location": [
- "Quick Start",
- "Quick Start"
- ]
- },
- {
- "content": "Docker Quick Start\nThere are three ways to start DolphinScheduler with Docker, Standalone-server is the way you\nfind if you just want to start and try DolphinScheduler as a beginner. docker-compose is for\nsome who want to deploy DolphinScheduler in small or event middle scale workflows in their daily work.\nUsing exists postgresql and zookeeper server is for users who want to reuse the database\nor zookeeper server already exists.\nPrepare\n\nDocker 1.13.1+\nDocker Com [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/start/docker",
- "title": "Docker Quick Start",
- "location": [
- "Quick Start",
- "Docker Deployment"
- ]
- },
- {
- "content": "Home Page\nApache DolphinScheduler home page lets you see task state statistics, workflow state statistics, and project statistics for all projects of users. It is the best way to observe status of your system as a whole as well as diving into individual process to check each status of task and task logs.\n\n",
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/homepage",
- "title": "Home Page",
- "location": [
- "Introduction to Functions",
- "Workflow Overview"
- ]
- },
- {
- "content": "Project\nThis page describes details regarding Project screen in Apache DolphinScheduler. Here, you will see all the functions which can be handled in this screen. The following table explains commonly used terms in Apache DolphinScheduler:\n\n\n\nGlossary\ndescription\n\n\n\n\nDAG\nTasks in a workflow are assembled in form of Directed Acyclic Graph (DAG). A topological traversal is performed from nodes with zero degrees of entry until there are no subsequent nodes.\n\n\n [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/project/project-list",
- "title": "Project",
- "location": [
- "Introduction to Functions",
- "Project",
- "Project List"
- ]
- },
- {
- "content": "Workflow Definition\nOverview\nWork Definition screen shows list of existing workflows and utility to create or import workflows.\n\nCreate workflow definition\nClick Project Management -> Workflow -> Workflow Definition, enter the Workflow Definition page, and click the Create Workflow button to enter the workflow DAG edit page, as shown in the following figure:\n\nCreate Tasks to Workflow\nDrag from the toolbar to the canvas, to add a shell task to the canvas, as [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/project/workflow-definition",
- "title": "Workflow Definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Definition"
- ]
- },
- {
- "content": "Workflow Instance\nView Workflow Instance\nClick Project Management -> Workflow -> Workflow Instance, enter the Workflow Instance page, as shown in the following figure:\n\nClick the workflow name to enter the DAG view page, and check the task execution status, as shown in the following figure:\n\nView Task Log\nEnter the workflow instance page, click the workflow name, enter the DAG view page, double-click the task node, as shown in the following figure:\n\nClick & [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/project/workflow-instance",
- "title": "Workflow Instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Instance"
- ]
- },
- {
- "content": "Task Instance\nCreate Task Instance\nClick Project Management -> Workflow -> Task Instance to enter the task instance page, as shown in the figure below, click the name of the workflow instance to jump to the DAG diagram of the workflow instance to view the task status.\n\nView Log\nClick the View Log button in the operation column to view the log of the task execution\n\n",
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/project/task-instance",
- "title": "Task Instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Instance"
- ]
- },
- {
- "content": "Task Definition\nTask definition allows to modify or operate tasks at the task level rather than modifying them in the workflow definition.\nWe already have workflow level task editor in workflow definition which you can click the specific\nworkflow and then edit its task definition. It is depressing when you want to edit the task definition but do not remember\nwhich workflow it belongs to. So we decide to add Task Definition view under Task menu.\n\nIn this view, you ca [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/project/task-definition",
- "title": "Task Definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Definition"
- ]
- },
- {
- "content": "Shell\nOverview\nShell task type, used to create a shell type task and execute a series of shell scripts. When the worker executes this task, a temporary shell script is generated and executed using the linux user with the same name as the tenant.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameters\n\n\n\nParamet [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/shell",
- "title": "Shell",
- "location": [
- "Introduction to Functions",
- "Task",
- "Shell"
- ]
- },
- {
- "content": "SubProcess Node\nOverview\nThe sub-process node is to execute an external workflow definition as a task node.\nCreate Task\n\nClick Project Management-> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar task node to canvas to create a new SubProcess task.\n\nTask Parameter\n\n\n\nParameter\nDescription\n\n\n\n\nNode name\nUnique name of node in workflow definition.\n\n\nRun flag\nIdentifi [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/sub-process",
- "title": "SubProcess Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "SubProcess"
- ]
- },
- {
- "content": "Dependent\nOverview\nDependent nodes are dependency check nodes. For example, process A depends on the successful execution of process B from yesterday, and the dependent node will check whether process B run successful yesterday.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar task node to canvas.\n\nTask Parameter\n\n\n\nParameter\nDescripti [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/dependent",
- "title": "Dependent",
- "location": [
- "Introduction to Functions",
- "Task",
- "Dependent"
- ]
- },
- {
- "content": "Stored Procedure\n\nExecute the stored procedure according to the selected DataSource.\n\n\nDrag from the PROCEDURE task node into the canvas, as shown in the figure below:\n\n\n \n \nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nNode Name\nSet the name of the task. Node names within a workflow definition are unique.\n\n\nRun flag\nIndicates whether the node can be scheduled normally. If it is not necessary to execute, you can turn on the prohibiting execution [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/stored-procedure",
- "title": "Stored Procedure",
- "location": [
- "Introduction to Functions",
- "Task",
- "Stored Procedure"
- ]
- },
- {
- "content": "SQL\nOverview\nSQL task type used to connect to databases and execute SQL.\nCreate DataSource\nRefer to datasource-setting DataSource Center section\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameter\n\n\n\nParameter\nDescription\n\n\n\n\nData source\nSelect the corresponding DataSource.\n\n\nSQL type\nSupports qu [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/sql",
- "title": "SQL",
- "location": [
- "Introduction to Functions",
- "Task",
- "SQL"
- ]
- },
- {
- "content": "Spark Node\nOverview\nSpark task type for executing Spark application. When executing the Spark task, the worker will submits a job to the Spark cluster by following commands:\n(1) spark submit method to submit tasks. See spark-submit for more details.\n(2) spark sql method to submit tasks. See spark sql for more details.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing pag [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/spark",
- "title": "Spark Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Spark"
- ]
- },
- {
- "content": "MapReduce Node\nOverview\nMapReduce(MR) task type used for executing MapReduce programs. For MapReduce nodes, the worker submits the task by using the Hadoop command hadoop jar. See Hadoop Command Manual for more details.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameters\nGeneral\n\n\n\nParameter\nDescription\n\ [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/map-reduce",
- "title": "MapReduce Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "MapReduce"
- ]
- },
- {
- "content": "Python Node\nOverview\nUse Python Task to create a python-type task and execute python scripts. When the worker executes Python Task,\nit will generate a temporary python script, and executes the script by the Linux user with the same name as the tenant.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameter\n\n\n\nPa [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/python",
- "title": "Python Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Python"
- ]
- },
- {
- "content": "Flink Node\nOverview\nFlink task type, used to execute Flink programs. For Flink nodes:\n\n\nWhen the program type is Java, Scala or Python, the worker submits the task flink run using the Flink command. See flink cli for more details.\n\n\nWhen the program type is SQL, the worker submit tasks using sql-client.sh. See flink sql client for more details.\n\n\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/flink",
- "title": "Flink Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Flink"
- ]
- },
- {
- "content": "HTTP Node\nOverview\nThis node is used to perform http type tasks such as the common POST and GET request types, and also supports http request validation and other functions.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nNode Name\nSet the name of the task. [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/http",
- "title": "HTTP Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "HTTP"
- ]
- },
- {
- "content": "DataX\nOverview\nDataX task type for executing DataX programs. For DataX nodes, the worker will execute ${DATAX_HOME}/bin/datax.py to analyze the input json file.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nNode name\nThe node name in a workflow definition [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/datax",
- "title": "DataX",
- "location": [
- "Introduction to Functions",
- "Task",
- "DataX"
- ]
- },
- {
- "content": "Pigeon\nOverview\nPigeon is a task used to trigger remote tasks, acquire logs or status by calling remote WebSocket service. It is DolphinScheduler uses a remote WebSocket service to call tasks.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas to create a new Pigeon task.\n\nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nN [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/pigeon",
- "title": "Pigeon",
- "location": [
- "Introduction to Functions",
- "Task",
- "Pigeon"
- ]
- },
- {
- "content": "Conditions\nCondition is a conditional node, that determines which downstream task should run based on the condition of the upstream task. Currently, the Conditions support multiple upstream tasks, but only two downstream tasks. When the number of upstream tasks exceeds one, achieve complex upstream dependencies by through and and or operators.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to en [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/conditions",
- "title": "Conditions",
- "location": [
- "Introduction to Functions",
- "Task",
- "Conditions"
- ]
- },
- {
- "content": "Switch\nOverview\nThe switch is a conditional judgment node, decide the branch executes according to the value of global variable and the expression result written by the user.\nNote: Execute expressions using javax.script.ScriptEngine.eval.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar task node to canvas to create a task.\n\nNote: After cr [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/switch",
- "title": "Switch",
- "location": [
- "Introduction to Functions",
- "Task",
- "Switch"
- ]
- },
- {
- "content": "Amazon EMR\nOverview\nAmazon EMR task type, for creating EMR clusters on AWS and running computing tasks. Using aws-java-sdk in the background code, to transfer JSON parameters to RunJobFlowRequest object and submit to AWS.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, click the Create Workflow button to enter the DAG editing page.\nDrag AmazonEMR task from the toolbar to the artboard to complete the creation.\n\nTask Parameters\n [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/emr",
- "title": "Amazon EMR",
- "location": [
- "Introduction to Functions",
- "Task",
- "Amazon EMR"
- ]
- },
- {
- "content": "Apache Zeppelin\nOverview\nUse Zeppelin Task to create a zeppelin-type task and execute zeppelin notebook paragraphs. When the worker executes Zeppelin Task,\nit will call Zeppelin Client API to trigger zeppelin notebook paragraph. Click here for details about Apache Zeppelin Notebook.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the can [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/task/zeppelin",
- "title": "Apache Zeppelin",
- "location": [
- "Introduction to Functions",
- "Task",
- "Apache Zeppelin"
- ]
- },
- {
- "content": "Built-in Parameter\nBasic Built-in Parameter\n\n\n\nVariable\nDeclaration Method\nMeaning\n\n\n\n\nsystem.biz.date\n${system.biz.date}\nThe day before the schedule time of the daily scheduling instance, the format is yyyyMMdd\n\n\nsystem.biz.curdate\n${system.biz.curdate}\nThe schedule time of the daily scheduling instance, the format is yyyyMMdd\n\n\nsystem.datetime\n${system.datetime}\nThe schedule time of the daily scheduling instance, the format is yyyyMMddHHmmss\n\n\ [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/parameter/built-in",
- "title": "Built-in Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Built-in Parameter"
- ]
- },
- {
- "content": "Global Parameter\nScope\nGlobal parameters are parameters that are valid for all task nodes of the entire workflow. It can be configured on the workflow definition page.\nUsage\nThe specific use method can be determined according to the actual production situation. This example uses a shell task to print out the date value of yesterday.\nExample\nCreate a Shell task\nCreate a shell task and enter echo ${dt} in the script content. In this case, dt is the global parameter w [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/parameter/global",
- "title": "Global Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Global Parameter"
- ]
- },
- {
- "content": "Local Parameter\nScope\nParameters configured on the task definition page, the scope of this parameter is inside this task only. But if you configure according to Refer to Parameter Context, it could pass to downstream tasks.\nUsage\n\nIf you want to use parameter in single task, see local parameter with custom parameter.\nIf you want to use set parameter in task and use them in downstream tasks:\n\nFor simple use without using custom parameter, see export parameter via s [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/parameter/local",
- "title": "Local Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Local Parameter"
- ]
- },
- {
- "content": "Parameter Context\nDolphinScheduler provides the ability to refer to each other between parameters, including local parameters refer to global parameters and upstream and downstream parameter transfer. Due to the existence of references, it involves the priority of parameters when the parameter names are the same. see also Parameter Priority\nLocal Task Refers to Global Parameter\nThe premise of local tasks referring global parameters is that you have already defined Glob [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/parameter/context",
- "title": "Parameter Context",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Context"
- ]
- },
- {
- "content": "Parameter Priority\nDolphinScheduler has three parameter types:\n\nGlobal Parameter: parameters defined at the workflow define page.\nParameter Context: parameters passed by upstream task nodes.\nLocal Parameter: parameters belong to its node, which is the parameters defined by the user in [Custom Parameters].\n\nThe user can define part of the parameters when creating workflow definitions.\nAs there are multiple sources of the parameter value, it will raise parameter pri [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/parameter/priority",
- "title": "Parameter Priority",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Priority"
- ]
- },
- {
- "content": "MySQL\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect MYSQL.\n\n\nDatasource name\nEnter the name of the DataSource.\n\n\nDescription\nEnter a description of the DataSource.\n\n\nIP/Host Name\nEnter the MYSQL service IP.\n\n\nPort\nEnter the MYSQL service port.\n\n\nUsername\nSet the username for MYSQL connection.\n\n\nPassword\nSet the password for MYSQL connection.\n\n\nDatabase name\nEnter the database name of the MYSQL connection.\n [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/datasource/mysql",
- "title": "MySQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "MySQL"
- ]
- },
- {
- "content": "PostgreSQL\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect POSTGRESQL.\n\n\nDatasource name\nEnter the name of the DataSource.\n\n\nDescription\nEnter a description of the DataSource.\n\n\nIP/Host Name\nEnter the PostgreSQL service IP.\n\n\nPort\nEnter the PostgreSQL service port.\n\n\nUsername\nSet the username for PostgreSQL connection.\n\n\nPassword\nSet the password for PostgreSQL connection.\n\n\nDatabase name\nEnter the database n [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/datasource/postgresql",
- "title": "PostgreSQL",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "PostgreSQL"
- ]
- },
- {
- "content": "HIVE\nUse HiveServer2\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect HIVE.\n\n\nDatasource name\nEnter the name of the DataSource.\n\n\nDescription\nEnter a description of the DataSource.\n\n\nIP/Host Name\nEnter the HIVE service IP.\n\n\nPort\nEnter the HIVE service port.\n\n\nUsername\nSet the username for HIVE connection.\n\n\nPassword\nSet the password for HIVE connection.\n\n\nDatabase name\nEnter the database name of the HIVE con [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/datasource/hive",
- "title": "HIVE",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "HIVE"
- ]
- },
- {
- "content": "Spark\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect Spark.\n\n\nDatasource name\nEnter the name of the DataSource.\n\n\nDescription\nEnter a description of the DataSource.\n\n\nIP/Host Name\nEnter the Spark service IP.\n\n\nPort\nEnter the Spark service port.\n\n\nUsername\nSet the username for Spark connection.\n\n\nPassword\nSet the password for Spark connection.\n\n\nDatabase name\nEnter the database name of the Spark connection.\n [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/datasource/spark",
- "title": "Spark",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Spark"
- ]
- },
- {
- "content": "Presto\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect Presto.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the Presto service IP.\n\n\nPort\nEnter the Presto service port.\n\n\nUsername\nSet the username for Presto connection.\n\n\nPassword\nSet the password for Presto connection.\n\n\nDatabase Name\nEnter the database name of the Presto connec [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/datasource/presto",
- "title": "Presto",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Presto"
- ]
- },
- {
- "content": "SQL Server\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect SQLSERVER.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the SQLSERVER service IP.\n\n\nPort\nEnter the SQLSERVER service port.\n\n\nUsername\nSet the username for SQLSERVER connection.\n\n\nPassword\nSet the password for SQLSERVER connection.\n\n\nDatabase Name\nEnter the database name o [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/datasource/sqlserver",
- "title": "SQL Server",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "SQL SERVER"
- ]
- },
- {
- "content": "Amazon Redshift\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect Redshift.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the Redshift service IP.\n\n\nPort\nEnter the Redshift service port.\n\n\nUsername\nSet the username for Redshift connection.\n\n\nPassword\nSet the password for Redshift connection.\n\n\nDatabase Name\nEnter the database name o [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/datasource/redshift",
- "title": "Amazon Redshift",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Amazon Redshift"
- ]
- },
- {
- "content": "ClickHouse\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect CLICKHOUSE.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the CLICKHOUSE service IP.\n\n\nPort\nEnter the CLICKHOUSE service port.\n\n\nUsername\nSet the username for CLICKHOUSE connection.\n\n\nPassword\nSet the password for CLICKHOUSE connection.\n\n\nDatabase Name\nEnter the database n [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/datasource/clickhouse",
- "title": "ClickHouse",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "ClickHouse"
- ]
- },
- {
- "content": "DB2\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect DB2.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the DB2 service IP.\n\n\nPort\nEnter the DB2 service port.\n\n\nUsername\nSet the username for DB2 connection.\n\n\nPassword\nSet the password for DB2 connection.\n\n\nDatabase Name\nEnter the database name of the DB2 connection.\n\n\njdbc conne [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/datasource/db2",
- "title": "DB2",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "IBM DB2"
- ]
- },
- {
- "content": "Oracle\n\nDatasource Parameters\n\n\n\nDatasource\nDescription\n\n\n\n\nDatasource\nSelect Oracle.\n\n\nDatasource Name\nEnter the name of the datasource.\n\n\nDescription\nEnter a description of the datasource.\n\n\nIP/Host Name\nEnter the Oracle service IP.\n\n\nPort\nEnter the Oracle service port.\n\n\nUsername\nSet the username for Oracle connection.\n\n\nPassword\nSet the password for Oracle connection.\n\n\nDatabase Name\nEnter the database name of the Oracle connec [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/datasource/oracle",
- "title": "Oracle",
- "location": [
- "Introduction to Functions",
- "Data Source",
- "Oracle"
- ]
- },
- {
- "content": "Alert Component User Guide\nAlert Plugins and Alert Groups Creation\nIn version 2.0.0, users need to create alert instances, and needs to choose an alarm policy when defining an alarm instance, there are three options: send if the task succeeds, send on failure, and send on both success and failure. when the workflow or task is executed, if an alarm is triggered, calling the alarm instance send method needs a logical judgment, which matches the alarm instance with the tas [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/alert/alert_plugin_user_guide",
- "title": "Alert Component User Guide",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Alert Component User Guide "
- ]
- },
- {
- "content": "Email\nIf you need to use Email for alerting, create an alert instance in the alert instance management and select the Email plugin.\nThe following shows the Email configuration example:\n\n\n\n",
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/alert/email",
- "title": "Email",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Email"
- ]
- },
- {
- "content": "Telegram\nIf you need Telegram to alert, create an alert instance in the alert instance management, and choose the Telegram plugin.\nThe following shows the Telegram configuration example:\n\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nWebHook\nThe WebHook of Telegram when use robot to send message.\n\n\nbotToken\nThe access token of robot.\n\n\nchatId\nSub Telegram Channel.\n\n\nparseMode\nMessage parse type (support txt, markdown, markdownV2, html).\n [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/alert/telegram",
- "title": "Telegram",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Telegram"
- ]
- },
- {
- "content": "DingTalk\nIf you need to use DingTalk for alerting, create an alert instance in the alert instance management and select the DingTalk plugin.\nThe following shows the DingTalk configuration example:\n\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nWarning Type\nAlert on success or failure or both.\n\n\nWebHook\nThe format is: https://oapi.dingtalk.com/robot/send?access_token=XXXXXX\n\n\nKeyword\nCustom keywords for security settings.\n\n\nSecret\nSignatur [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/alert/dingtalk",
- "title": "DingTalk",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Ding Talk"
- ]
- },
- {
- "content": "Enterprise WeChat\nIf you need to use Enterprise WeChat to alert, create an alert instance in the alert instance management, and choose the WeChat plugin.\nThe following is the WeChat configuration example:\n\nSend Type\nThe parameter send.type corresponds to sending messages to Enterprise WeChat customized APP and group chat created by API respectively.\nAPP\nThe APP sends type means to notify the alert results via Enterprise WeChat customized APPs, supports sending mess [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/alert/enterprise-wechat",
- "title": "Enterprise WeChat",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Enterprise Wechat"
- ]
- },
- {
- "content": "Webex Teams\nIf you need to use Webex Teams to alert, create an alert instance in the alert instance management, and choose the WebexTeams plugin. You can pick private alert or room group chat alert.\nThe following is the WebexTeams configuration example:\n\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nbotAccessToken\nThe access token of robot.\n\n\nroomID\nThe ID of the room that receives message (only support one room ID).\n\n\ntoPersonId\nThe person I [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/alert/enterprise-webexteams",
- "title": "Webex Teams",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Enterprise Webexteams"
- ]
- },
- {
- "content": "Script\nIf you need to use Shell script for alerting, create an alert instance in the alert instance management and select the Script plugin.\nThe following shows the Script configuration example:\n\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nUser Params\nUser defined parameters will pass to the script.\n\n\nScript Path\nThe file location path in the server.\n\n\nType\nSupport Shell script.\n\n\n\nNote\nConsider the script file access privileges with t [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/alert/script",
- "title": "Script",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Script"
- ]
- },
- {
- "content": "HTTP\nIf you need to use Http script for alerting, create an alert instance in the alert instance management and select the Http plugin.\nParameter Configuration\n\n\n\nParameter\nDescription\n\n\n\n\nURL\nThe Http request URL needs to contain protocol, host, path and parameters if the method is GET.\n\n\nRequest Type\nSelect the request type from POST or GET.\n\n\nHeaders\nThe headers of the Http request in JSON format.\n\n\nBody\nThe request body of the Http request in [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/alert/http",
- "title": "HTTP",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Http"
- ]
- },
- {
- "content": "Feishu\nIf you need to use Feishu for alerting, create an alert instance in the alert instance management and select\nthe Feishu plugin.\nThe following shows the Feishu configuration example:\n\nParameter Configuration\n\n\nWebhook\n\nCopy the robot webhook URL shown below:\n\n\n\n\nFeishu：Use bots in groups\n",
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/alert/feishu",
- "title": "Feishu",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Feishu"
- ]
- },
- {
- "content": "Slack\nIf you need to use Slack for alerting, create an alert instance in the alert instance management and select\nthe Slack plugin.\nThe following shows the Slack configuration example:\n\nParameter Configuration\n\n\nWebhook\n\nCopy the Incoming Webhooks address from the APP, check the image below:\n\n\n\n\nUserName\n\n(Deprecated) The sender name. Currently, Slack use APP as the sender name due to the Slack updates.\n\n\n\nHow to Get Webhook\nPlease refer to the offic [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/alert/slack",
- "title": "Slack",
- "location": [
- "Introduction to Functions",
- "Alert",
- "Slack"
- ]
- },
- {
- "content": "Resource Center Introduction\nThe Resource Center is typically used for uploading files, UDF functions, and task group management. For a stand-alone\nenvironment, you can select the local file directory as the upload folder (this operation does not require Hadoop or HDFS deployment).\nOf course, you can also choose to upload to Hadoop or MinIO cluster. In this case, you need to have Hadoop (2.6+) or MinIOn and other related environments.\n",
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/resource/intro",
- "title": "Resource Center Introduction",
- "location": [
- "Introduction to Functions",
- "Resource",
- "Introduction"
- ]
- },
- {
- "content": "Resource Center Configuration\n\nYou could use Resource Center to upload text files, UDFs and other task-related files.\nYou could configure Resource Center to use distributed file system like Hadoop (2.6+), MinIO cluster or remote storage products like AWS S3, Alibaba Cloud OSS, etc.\nYou could configure Resource Center to use local file system. If you deploy DolphinScheduler in Standalone mode, you could configure it to use local file system for Resouce Center without t [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/resource/configuration",
- "title": "Resource Center Configuration",
- "location": [
- "Introduction to Functions",
- "Resource",
- "Configuration"
- ]
- },
- {
- "content": "File Management\nWhen the third-party jar needs to be used in the scheduling process or the user needs to customize the script, the related operations can be completed on this page. The file types that can be created include: txt, log, sh, conf, py, java etc. And you can edit, rename, download and delete files.\n\n\nNote:\n\nWhen you manage files as admin, remember to set up tenant for admin first.\n\n\nBasic Operations\nCreate File\nThe file format supports the following [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/resource/file-manage",
- "title": "File Management",
- "location": [
- "Introduction to Functions",
- "Resource",
- "File Manage"
- ]
- },
- {
- "content": "UDF Manage\nResource Management\n\nThe resource management and file management functions are similar. The difference is that the resource management is the UDF upload function, and the file management uploads the user programs, scripts and configuration files.\nIt mainly includes the following operations: rename, download, delete, etc.\nUpload UDF resources: Same as uploading files.\n\nFunction Management\nCreate UDF function\nClick Create UDF Function, enter the UDF func [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/resource/udf-manage",
- "title": "UDF Manage",
- "location": [
- "Introduction to Functions",
- "Resource",
- "UDF Manage"
- ]
- },
- {
- "content": "Task Group Settings\nThe task group is mainly used to control the concurrency of task instances, and is designed to control the pressure of other resources (it can also control the pressure of the Hadoop cluster, the cluster will have queue control it). When creating a new task definition, you can configure the corresponding task group and configure the priority of the task running in the task group.\nTask Group Configuration\nCreate Task Group\n\nThe user clicks Resource [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/resource/task-group",
- "title": "Task Group Settings",
- "location": [
- "Introduction to Functions",
- "Resource",
- "Task Group Manage"
- ]
- },
- {
- "content": "Monitor\nService Management\n\nService management is mainly to monitor and display the health status and basic information of each service in the system.\n\nMaster Server\n\nMainly related to master information.\n\n\nWorker Server\n\nMainly related to worker information.\n\n\nDatabase\n\nMainly the health status of the DB.\n\n\nStatistics Management\nStatistics\n\n\n\n\nParameter\nDescription\n\n\n\n\nNumber of commands wait to be executed\nStatistics of the t_ds_command [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/monitor",
- "title": "Monitor",
- "location": [
- "Introduction to Functions",
- "Monitor"
- ]
- },
- {
- "content": "Security (Authorization System)\nOnly the administrator account in the security center has permission to operate, which includes queue management, tenant management, user management, alarm group management, worker group management, token management and other functions. In the user management module, you can authorize resources, data sources, projects, etc.\nAdministrator login, default username/password: admin/dolphinscheduler123\nCreate Queue\n\nThe queue is used when ex [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/security",
- "title": "Security (Authorization System)",
- "location": [
- "Introduction to Functions",
- "Security"
- ]
- },
- {
- "content": "Introduction\nApache DolphinScheduler has export some metrics to monitor the system. We use micrometer for the exporter facade, and\nthe default exporter is prometheus, more exporter is coming soon.\nQuick Start\nYou can add the following config in master/worker/alert/api's yaml file to open the metrics exporter.\nmetrics:\n enabled: true\n\nOnce you open the metrics exporter, you can access the metrics by the url: http://ip:port/actuator/prometheus\nThe exporter port is [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/metrics/metrics",
- "title": "Introduction",
- "location": [
- "Introduction to Functions",
- "Metrics",
- "Metrics"
- ]
- },
- {
- "content": "Data Quality\nIntroduction\nThe data quality task is used to check the data accuracy during the integration and processing of data. Data quality tasks in this release include single-table checking, single-table custom SQL checking, multi-table accuracy, and two-table value comparisons. The running environment of the data quality task is Spark 2.4.0, and other versions have not been verified, and users can verify by themselves.\nThe execution logic of the data quality task [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/data-quality",
- "title": "Data Quality",
- "location": [
- "Introduction to Functions",
- "Data Quality"
- ]
- },
- {
- "content": "DolphinScheduler Upgrade\nPrepare\nBackup Previous Version's Files and Database\nTo prevent data loss by some miss-operation, it is recommended to back up data before upgrading. The backup way according to your environment.\nDownload the Latest Version Installation Package\nDownload the latest binary distribute package from download and then put it in the different\ndirectory where current service running. And all below command is running in this directory.\nUpgrade\nStop [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/upgrade",
- "title": "DolphinScheduler Upgrade",
- "location": [
- "Introduction to Functions",
- "Upgrade"
- ]
- },
- {
- "content": "DolphinScheduler Expansion and Reduction\nExpansion\nThis article describes how to add a new master service or worker service to an existing DolphinScheduler cluster.\n Attention: There cannot be more than one master service process or worker service process on a physical machine.\n If the physical machine which locate the expansion master or worker node has already installed the scheduled service, check the [1.4 Modify configuration] and edit the configuration file [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/expansion-reduction",
- "title": "DolphinScheduler Expansion and Reduction",
- "location": [
- "Introduction to Functions",
- "Expansion and Reduction"
- ]
- },
- {
- "content": "Standalone\nStandalone only for quick experience for DolphinScheduler.\nIf you are a newbie and want to experience DolphinScheduler functions, we recommend you install follow Standalone deployment.\nIf you want to experience more complete functions and schedule massive tasks, we recommend you install follow pseudo-cluster deployment.\nIf you want to deploy DolphinScheduler in production, we recommend you follow cluster deployment or Kubernetes deployment.\n\nNote: Standal [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/installation/standalone",
- "title": "Standalone",
- "location": [
- "Installation",
- "Standalone Deployment"
- ]
- },
- {
- "content": "Pseudo-Cluster Deployment\nThe purpose of the pseudo-cluster deployment is to deploy the DolphinScheduler service on a single machine. In this mode, DolphinScheduler's master, worker, API server, are all on the same machine.\nIf you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow Standalone deployment. If you want to experience more complete functions and schedule massive tasks, we recommend you install follow[pseudo-clust [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/installation/pseudo-cluster",
- "title": "Pseudo-Cluster Deployment",
- "location": [
- "Installation",
- "Pseudo Cluster Deployment"
- ]
- },
- {
- "content": "Cluster Deployment\nCluster deployment is to deploy the DolphinScheduler on multiple machines for running massive tasks in production.\nIf you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow Standalone deployment. If you want to experience more complete functions and schedule massive tasks, we recommend you install follow pseudo-cluster deployment. If you want to deploy DolphinScheduler in production, we recommend you foll [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/installation/cluster",
- "title": "Cluster Deployment",
- "location": [
- "Installation",
- "Cluster Deployment"
- ]
- },
- {
- "content": "QuickStart in Kubernetes\nKubernetes deployment is DolphinScheduler deployment in a Kubernetes cluster, which can schedule massive tasks and can be used in production.\nIf you are a new hand and want to experience DolphinScheduler functions, we recommend you install follow Standalone deployment. If you want to experience more complete functions and schedule massive tasks, we recommend you install follow pseudo-cluster deployment. If you want to deploy DolphinScheduler in [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/installation/kubernetes",
- "title": "QuickStart in Kubernetes",
- "location": [
- "Installation",
- "Kubernetes Deployment"
- ]
- },
- {
- "content": "System Architecture Design\nSystem Structure\nSystem Architecture Diagram\n\n \n \n System architecture diagram\n \n\nStart Process Activity Diagram\n\n \n \n Start process activity diagram\n \n\nArchitecture Description\n\n\nMasterServer\nMasterServer adopts a distributed and decentralized design concept. MasterServer is mainly responsible for DAG task segmentation, task submission monitoring, and monitoring the health status of other MasterServer and [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/architecture/design",
- "title": "System Architecture Design",
- "location": [
- "Architecture Guide",
- "Architecture Design"
- ]
- },
- {
- "content": "MetaData\nTable Schema\nsee sql files in dolphinscheduler/dolphinscheduler-dao/src/main/resources/sql\n\nE-R Diagram\nUser Queue DataSource\n\n\nOne tenant can own Multiple users.\nThe queue field in the t_ds_user table stores the queue_name information in the t_ds_queue table, t_ds_tenant stores queue information using queue_id column. During the execution of the process definition, the user queue has the highest priority. If the user queue is null, use the tenant queue. [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/architecture/metadata",
- "title": "MetaData",
- "location": [
- "Architecture Guide",
- "Metadata"
- ]
- },
- {
- "content": "\nConfiguration\nPreface\nThis document explains the DolphinScheduler application configurations.\nDirectory Structure\nThe directory structure of DolphinScheduler is as follows:\n├── LICENSE\n│\n├── NOTICE\n│\n├── licenses directory of licenses\n│\n├── bin directory of DolphinScheduler application commands, configrations scripts \n│ ├── dolphinscheduler-daemon.sh script to start or [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/architecture/configuration",
- "title": "Configuration",
- "location": [
- "Architecture Guide",
- "Configuration File"
- ]
- },
- {
- "content": "Task Structure\nOverall Tasks Storage Structure\nAll tasks in DolphinScheduler are saved in the t_ds_process_definition table.\nThe following shows the t_ds_process_definition table structure:\n\n\n\nNo.\nfield\ntype\ndescription\n\n\n\n\n1\nid\nint(11)\nprimary key\n\n\n2\nname\nvarchar(255)\nprocess definition name\n\n\n3\nversion\nint(11)\nprocess definition version\n\n\n4\nrelease_state\ntinyint(4)\nrelease status of process definition: 0 not released, 1 released\n\n\ [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/architecture/task-structure",
- "title": "Task Structure",
- "location": [
- "Architecture Guide",
- "Task Structure"
- ]
- },
- {
- "content": "Load Balance\nLoad balancing refers to the reasonable allocation of server pressure through routing algorithms (usually in cluster environments) to achieve the maximum optimization of server performance.\nDolphinScheduler-Worker Load Balancing Algorithms\nDolphinScheduler-Master allocates tasks to workers, and by default provides three algorithms:\n\n\nWeighted random (random)\n\n\nSmoothing polling (round-robin)\n\n\nLinear load (lower weight)\n\n\nThe default configurat [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/architecture/load-balance",
- "title": "Load Balance",
- "location": [
- "Architecture Guide",
- "Load Balance"
- ]
- },
- {
- "content": "Cache\nPurpose\nDue to the large database read operations during the master-server scheduling process. Such as read tables like tenant, user, processDefinition, etc. Operations stress read pressure to the DB, and slow down the entire core scheduling process.\nBy considering this part of the business data is a high-read and low-write scenario, a cache module is introduced to reduce the DB read pressure and speed up the core scheduling process.\nCache Settings\nspring:\n c [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/architecture/cache",
- "title": "Cache",
- "location": [
- "Architecture Guide",
- "Cache"
- ]
- },
- {
- "content": "Open API\nBackground\nGenerally, projects and processes are created through pages, but considering the integration with third-party systems requires API calls to manage projects and workflows.\nThe Operation Steps of DolphinScheduler API Calls\nCreate a Token\n\nLog in to the scheduling system, click "Security", then click "Token manage" on the left, and click "Create token" to create a token.\n\n\n\nSelect the "Expiration time" (To [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/open-api",
- "title": "Open API",
- "location": [
- "API",
- "Open API"
- ]
- },
- {
- "content": "Health Check\nBackground\nHealth check are designed to provide a unique way to check the health of the dolphinscheduler service. It includes the health status of modules, such as DB, cache, network, etc.\nEndpoint\nAPI-Server\ncurl --request GET 'http://localhost:12345/dolphinscheduler/actuator/health'\n\n{"status":"UP","components":{"db":{"status":"UP","details":{"database":"H2 [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/guide/healthcheck",
- "title": "Health Check",
- "location": [
- "API",
- "Health Check"
- ]
- },
- {
- "content": "DSIP\nDolphinScheduler Improvement Proposal(DSIP) introduce major improvements to the Apache DolphinScheduler codebase. It is\nnot for small incremental improvements, and the purpose of DSIP is to notice and inform community the finished or coming\nbig feature for Apache DolphinScheduler.\nWhat is considered as DSIP\n\nAny major new feature, major improvement, introduce or remove components\nAny major change of public interfaces, such as API endpoints, web ui huge change\ [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/DSIP",
- "title": "DSIP",
- "location": [
- "DSIP",
- "DSIP"
- ]
- },
- {
- "content": "Change Log\nBugfix\n\nFix default value for resource center in k8s mode (#12906)\nFix send script alert NPE (#12495) (#12895)\nFix procedure task output param (#12894)\nFix memory leak in worker due to message retry map (#12878) (#12887)\nFix flink sql cannot run due to missing main jar (#12705) (#12800)\nFix sql error (#12717) (#12799)\nIf the worker's clock early than master node, will ignore dispatch time (#12219) (#12798)\nFix dependent task can not predicate the stat [...]
- "version": "3.0.2",
- "time": "2022-11-21",
- "link": "/changelog",
- "title": "Change Log",
- "location": [
- "Change Log",
- "Change Log"
- ]
- },
- {
- "content": "About DolphinScheduler\nApache DolphinScheduler provides a distributed and easy to expand visual workflow task scheduling open-source platform. It is suitable for enterprise-level scenarios. It provides a solution to visualize operation tasks, workflows, and the entire data processing procedures.\nApache DolphinScheduler aims to solve complex big data task dependencies and to trigger relationships in data OPS orchestration for various big data applications. Solves the int [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/about/introduction",
- "title": "About DolphinScheduler",
- "location": [
- "About",
- "Introduction"
- ]
- },
- {
- "content": "Features\nSimple to Use\n\nVisual DAG: User-friendly drag-and-drop workflow definition and facility for run-time control.\nModular Operation: Modularity facilitates easy customization and maintenance.\n\nRich Scenarios\n\nMultiple Task Type Support: Supports more than 10 task types, like Shell, MR, Spark, SQL, etc., with cross-language support making it easy to extend\nWorkflow Ops: Workflow can be timed, paused, resumed and stopped, enabling easy maintenance and control [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/about/features",
- "title": "Features",
- "location": [
- "About",
- "Features"
- ]
- },
- {
- "content": "Hardware Environment\nThis section briefs about the hardware requirements for DolphinScheduler. DolphinScheduler works as an open-source distributed workflow task scheduling system. It can deploy and run smoothly in Intel architecture server environments and mainstream virtualization environments. It also supports mainstream Linux operating system environments and ARM architecture.\nLinux Operating System Version Requirements\nThe Linux operating systems specified below c [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/about/hardware",
- "title": "Hardware Environment",
- "location": [
- "About",
- "Hardware Environment"
- ]
- },
- {
- "content": "System Architecture Design\nBefore explaining the architecture of the scheduling system, let's first understand the commonly used terms of the\nscheduling system\n1.Glossary\nDAG： The full name is Directed Acyclic Graph, referred to as DAG. Task tasks in the workflow are assembled in the\nform of a directed acyclic graph, and topological traversal is performed from nodes with zero degrees of entry until\nthere are no subsequent nodes. Examples are as follows:\n\nProcess d [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/about/glossary",
- "title": "System Architecture Design",
- "location": [
- "About",
- "Glossary"
- ]
- },
- {
- "content": "Quick Start\nThis is a Quick Start guide to help you get a basic idea of working with Apache DolphinScheduler. Once you've gone through the tutorial video, we encourage you to find out more about Apache DolphinScheduler functions and examples in .\n \n \n\nAdministrator User Login\n\nAddress: http://localhost:12345/dolphinscheduler\nUsername and password: admin/dolphinscheduler123\n\n\nCreate a queue\n\nCreate a tenant\n\nCreate Ordinary Users\n\nCreate an alarm instance [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/start/quick-start",
- "title": "Quick Start",
- "location": [
- "Quick Start",
- "Quick Start"
- ]
- },
- {
- "content": "Docker Quick Start\nThere are three ways to start DolphinScheduler with Docker, Standalone-server is the way you\nfind if you just want to start and try DolphinScheduler as a beginner. docker-compose is for\nsome who want to deploy DolphinScheduler in small or event middle scale workflows in their daily work.\nUsing exists postgresql and zookeeper server is for users who want to reuse the database\nor zookeeper server already exists.\nPrepare\n\nDocker 1.13.1+\nDocker Com [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/start/docker",
- "title": "Docker Quick Start",
- "location": [
- "Quick Start",
- "Docker Deployment"
- ]
- },
- {
- "content": "Home Page\nApache DolphinScheduler home page lets you see task state statistics, workflow state statistics, and project statistics for all projects of users. It is the best way to observe status of your system as a whole as well as diving into individual process to check each status of task and task logs.\n\n",
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/homepage",
- "title": "Home Page",
- "location": [
- "Introduction to Functions",
- "Workflow Overview"
- ]
- },
- {
- "content": "Project\nThis page describes details regarding Project screen in Apache DolphinScheduler. Here, you will see all the functions which can be handled in this screen. The following table explains commonly used terms in Apache DolphinScheduler:\n\n\n\nGlossary\ndescription\n\n\n\n\nDAG\nTasks in a workflow are assembled in form of Directed Acyclic Graph (DAG). A topological traversal is performed from nodes with zero degrees of entry until there are no subsequent nodes.\n\n\n [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/project/project-list",
- "title": "Project",
- "location": [
- "Introduction to Functions",
- "Project",
- "Project List"
- ]
- },
- {
- "content": "Workflow Definition\nOverview\nWork Definition screen shows list of existing workflows and utility to create or import workflows.\n\nCreate workflow definition\nClick Project Management -> Workflow -> Workflow Definition, enter the Workflow Definition page, and click the Create Workflow button to enter the workflow DAG edit page, as shown in the following figure:\n\nCreate Tasks to Workflow\nDrag from the toolbar to the canvas, to add a shell task to the canvas, as [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/project/workflow-definition",
- "title": "Workflow Definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Definition"
- ]
- },
- {
- "content": "Workflow Instance\nView Workflow Instance\nClick Project Management -> Workflow -> Workflow Instance, enter the Workflow Instance page, as shown in the following figure:\n\nClick the workflow name to enter the DAG view page, and check the task execution status, as shown in the following figure:\n\nView Task Log\nEnter the workflow instance page, click the workflow name, enter the DAG view page, double-click the task node, as shown in the following figure:\n\nClick & [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/project/workflow-instance",
- "title": "Workflow Instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Workflow Instance"
- ]
- },
- {
- "content": "Task Instance\nCreate Task Instance\nClick Project Management -> Workflow -> Task Instance to enter the task instance page, as shown in the figure below, click the name of the workflow instance to jump to the DAG diagram of the workflow instance to view the task status.\n\nView Log\nClick the View Log button in the operation column to view the log of the task execution\n\n",
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/project/task-instance",
- "title": "Task Instance",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Instance"
- ]
- },
- {
- "content": "Task Definition\nTask definition allows to modify or operate tasks at the task level rather than modifying them in the workflow definition.\nWe already have workflow level task editor in workflow definition which you can click the specific\nworkflow and then edit its task definition. It is depressing when you want to edit the task definition but do not remember\nwhich workflow it belongs to. So we decide to add Task Definition view under Task menu.\n\nIn this view, you ca [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/project/task-definition",
- "title": "Task Definition",
- "location": [
- "Introduction to Functions",
- "Project",
- "Task Definition"
- ]
- },
- {
- "content": "Shell\nOverview\nShell task type, used to create a shell type task and execute a series of shell scripts. When the worker executes this task, a temporary shell script is generated and executed using the linux user with the same name as the tenant.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameters\n\n\n\nParamet [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/shell",
- "title": "Shell",
- "location": [
- "Introduction to Functions",
- "Task",
- "Shell"
- ]
- },
- {
- "content": "SubProcess Node\nOverview\nThe sub-process node is to execute an external workflow definition as a task node.\nCreate Task\n\nClick Project Management-> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar task node to canvas to create a new SubProcess task.\n\nTask Parameter\n\n\n\nParameter\nDescription\n\n\n\n\nNode name\nUnique name of node in workflow definition.\n\n\nRun flag\nIdentifi [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/sub-process",
- "title": "SubProcess Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "SubProcess"
- ]
- },
- {
- "content": "Dependent\nOverview\nDependent nodes are dependency check nodes. For example, process A depends on the successful execution of process B from yesterday, and the dependent node will check whether process B run successful yesterday.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar task node to canvas.\n\nTask Parameter\n\n\n\nParameter\nDescripti [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/dependent",
- "title": "Dependent",
- "location": [
- "Introduction to Functions",
- "Task",
- "Dependent"
- ]
- },
- {
- "content": "Stored Procedure\n\nExecute the stored procedure according to the selected DataSource.\n\n\nDrag from the PROCEDURE task node into the canvas, as shown in the figure below:\n\n\n \n \nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nNode Name\nSet the name of the task. Node names within a workflow definition are unique.\n\n\nRun flag\nIndicates whether the node can be scheduled normally. If it is not necessary to execute, you can turn on the prohibiting execution [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/stored-procedure",
- "title": "Stored Procedure",
- "location": [
- "Introduction to Functions",
- "Task",
- "Stored Procedure"
- ]
- },
- {
- "content": "SQL\nOverview\nSQL task type used to connect to databases and execute SQL.\nCreate DataSource\nRefer to datasource-setting DataSource Center section\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameter\n\n\n\nParameter\nDescription\n\n\n\n\nData source\nSelect the corresponding DataSource.\n\n\nSQL type\nSupports qu [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/sql",
- "title": "SQL",
- "location": [
- "Introduction to Functions",
- "Task",
- "SQL"
- ]
- },
- {
- "content": "Spark Node\nOverview\nSpark task type for executing Spark application. When executing the Spark task, the worker will submits a job to the Spark cluster by following commands:\n(1) spark submit method to submit tasks. See spark-submit for more details.\n(2) spark sql method to submit tasks. See spark sql for more details.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing pag [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/spark",
- "title": "Spark Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Spark"
- ]
- },
- {
- "content": "MapReduce Node\nOverview\nMapReduce(MR) task type used for executing MapReduce programs. For MapReduce nodes, the worker submits the task by using the Hadoop command hadoop jar. See Hadoop Command Manual for more details.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameters\nGeneral\n\n\n\nParameter\nDescription\n\ [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/map-reduce",
- "title": "MapReduce Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "MapReduce"
- ]
- },
- {
- "content": "Python Node\nOverview\nUse Python Task to create a python-type task and execute python scripts. When the worker executes Python Task,\nit will generate a temporary python script, and executes the script by the Linux user with the same name as the tenant.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas.\n\nTask Parameter\n\n\n\nPa [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/python",
- "title": "Python Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Python"
- ]
- },
- {
- "content": "Flink Node\nOverview\nFlink task type, used to execute Flink programs. For Flink nodes:\n\n\nWhen the program type is Java, Scala or Python, the worker submits the task flink run using the Flink command. See flink cli for more details.\n\n\nWhen the program type is SQL, the worker submit tasks using sql-client.sh. See flink sql client for more details.\n\n\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/flink",
- "title": "Flink Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "Flink"
- ]
- },
- {
- "content": "HTTP Node\nOverview\nThis node is used to perform http type tasks such as the common POST and GET request types, and also supports http request validation and other functions.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nNode Name\nSet the name of the task. [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/http",
- "title": "HTTP Node",
- "location": [
- "Introduction to Functions",
- "Task",
- "HTTP"
- ]
- },
- {
- "content": "DataX\nOverview\nDataX task type for executing DataX programs. For DataX nodes, the worker will execute ${DATAX_HOME}/bin/datax.py to analyze the input json file.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag the from the toolbar to the drawing board.\n\nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nNode name\nThe node name in a workflow definition [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/datax",
- "title": "DataX",
- "location": [
- "Introduction to Functions",
- "Task",
- "DataX"
- ]
- },
- {
- "content": "Pigeon\nOverview\nPigeon is a task used to trigger remote tasks, acquire logs or status by calling remote WebSocket service. It is DolphinScheduler uses a remote WebSocket service to call tasks.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the canvas to create a new Pigeon task.\n\nTask Parameters\n\n\n\nParameter\nDescription\n\n\n\n\nN [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/pigeon",
- "title": "Pigeon",
- "location": [
- "Introduction to Functions",
- "Task",
- "Pigeon"
- ]
- },
- {
- "content": "Conditions\nCondition is a conditional node, that determines which downstream task should run based on the condition of the upstream task. Currently, the Conditions support multiple upstream tasks, but only two downstream tasks. When the number of upstream tasks exceeds one, achieve complex upstream dependencies by through and and or operators.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to en [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/conditions",
- "title": "Conditions",
- "location": [
- "Introduction to Functions",
- "Task",
- "Conditions"
- ]
- },
- {
- "content": "Switch\nOverview\nThe switch is a conditional judgment node, decide the branch executes according to the value of global variable and the expression result written by the user.\nNote: Execute expressions using javax.script.ScriptEngine.eval.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar task node to canvas to create a task.\n\nNote: After cr [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/switch",
- "title": "Switch",
- "location": [
- "Introduction to Functions",
- "Task",
- "Switch"
- ]
- },
- {
- "content": "Amazon EMR\nOverview\nAmazon EMR task type, for creating EMR clusters on AWS and running computing tasks. Using aws-java-sdk in the background code, to transfer JSON parameters to RunJobFlowRequest object and submit to AWS.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, click the Create Workflow button to enter the DAG editing page.\nDrag AmazonEMR task from the toolbar to the artboard to complete the creation.\n\nTask Parameters\n [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/emr",
- "title": "Amazon EMR",
- "location": [
- "Introduction to Functions",
- "Task",
- "Amazon EMR"
- ]
- },
- {
- "content": "Apache Zeppelin\nOverview\nUse Zeppelin Task to create a zeppelin-type task and execute zeppelin notebook paragraphs. When the worker executes Zeppelin Task,\nit will call Zeppelin Client API to trigger zeppelin notebook paragraph. Click here for details about Apache Zeppelin Notebook.\nCreate Task\n\nClick Project Management -> Project Name -> Workflow Definition, and click the Create Workflow button to enter the DAG editing page.\nDrag from the toolbar to the can [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/task/zeppelin",
- "title": "Apache Zeppelin",
- "location": [
- "Introduction to Functions",
- "Task",
- "Apache Zeppelin"
- ]
- },
- {
- "content": "Built-in Parameter\nBasic Built-in Parameter\n\n\n\nVariable\nDeclaration Method\nMeaning\n\n\n\n\nsystem.biz.date\n${system.biz.date}\nThe day before the schedule time of the daily scheduling instance, the format is yyyyMMdd\n\n\nsystem.biz.curdate\n${system.biz.curdate}\nThe schedule time of the daily scheduling instance, the format is yyyyMMdd\n\n\nsystem.datetime\n${system.datetime}\nThe schedule time of the daily scheduling instance, the format is yyyyMMddHHmmss\n\n\ [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/parameter/built-in",
- "title": "Built-in Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Built-in Parameter"
- ]
- },
- {
- "content": "Global Parameter\nScope\nGlobal parameters are parameters that are valid for all task nodes of the entire workflow. It can be configured on the workflow definition page.\nUsage\nThe specific use method can be determined according to the actual production situation. This example uses a shell task to print out the date value of yesterday.\nExample\nCreate a Shell task\nCreate a shell task and enter echo ${dt} in the script content. In this case, dt is the global parameter w [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/parameter/global",
- "title": "Global Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Global Parameter"
- ]
- },
- {
- "content": "Local Parameter\nScope\nParameters configured on the task definition page, the scope of this parameter is inside this task only. But if you configure according to Refer to Parameter Context, it could pass to downstream tasks.\nUsage\n\nIf you want to use parameter in single task, see local parameter with custom parameter.\nIf you want to use set parameter in task and use them in downstream tasks:\n\nFor simple use without using custom parameter, see export parameter via s [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/parameter/local",
- "title": "Local Parameter",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Local Parameter"
- ]
- },
- {
- "content": "Parameter Context\nDolphinScheduler provides the ability to refer to each other between parameters, including local parameters refer to global parameters and upstream and downstream parameter transfer. Due to the existence of references, it involves the priority of parameters when the parameter names are the same. see also Parameter Priority\nLocal Task Refers to Global Parameter\nThe premise of local tasks referring global parameters is that you have already defined Glob [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/parameter/context",
- "title": "Parameter Context",
- "location": [
- "Introduction to Functions",
- "Parameter",
- "Parameter Context"
- ]
- },
- {
- "content": "Parameter Priority\nDolphinScheduler has three parameter types:\n\nGlobal Parameter: parameters defined at the workflow define page.\nParameter Context: parameters passed by upstream task nodes.\nLocal Parameter: parameters belong to its node, which is the parameters defined by the user in [Custom Parameters].\n\nThe user can define part of the parameters when creating workflow definitions.\nAs there are multiple sources of the parameter value, it will raise parameter pri [...]
- "version": "3.0.3",
- "time": "2022-12-08",
- "link": "/guide/parameter/priority",
- "title": "Parameter Priority",
... 11951 lines suppressed ...