You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by le...@apache.org on 2021/11/26 04:26:22 UTC
[dolphinscheduler] branch 2.0.1-prepare updated: [2.0.1-prepare#6712]Refactor registry plugin and simplify its usage (#6999)
This is an automated email from the ASF dual-hosted git repository.
leonbao pushed a commit to branch 2.0.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/2.0.1-prepare by this push:
new 6821c7a [2.0.1-prepare#6712]Refactor registry plugin and simplify its usage (#6999)
6821c7a is described below
commit 6821c7a9ac31467324ef541e0f9dc9353656e58a
Author: Kirs <ac...@163.com>
AuthorDate: Fri Nov 26 12:26:15 2021 +0800
[2.0.1-prepare#6712]Refactor registry plugin and simplify its usage (#6999)
* [2.0.1-prepare#6712]Refactor registry plugin and simplify its usage (#6712)
* upgrade version and fix dep err
* Clean up the utility codes (#6732)
* remove code style check
* upgrade docker version
Co-authored-by: kezhenxu94 <ke...@apache.org>
---
.../registry.properties => .github/CODEOWNERS | 8 +-
.github/actions/sanity-check/action.yml | 13 -
.../conf/dolphinscheduler/registry.properties.tpl | 8 -
docker/build/startup-init-conf.sh | 1 -
docker/docker-swarm/config.env.sh | 1 -
docker/docker-swarm/docker-compose.yml | 8 +-
docker/docker-swarm/docker-stack.yml | 8 +-
docker/kubernetes/dolphinscheduler/Chart.yaml | 4 +-
.../dolphinscheduler/templates/_helpers.tpl | 8 +-
docker/kubernetes/dolphinscheduler/values.yaml | 2 +-
.../dolphinscheduler-alert-dingtalk/pom.xml | 4 +-
.../dolphinscheduler-alert-email/pom.xml | 2 +-
.../dolphinscheduler-alert-feishu/pom.xml | 4 +-
.../dolphinscheduler-alert-http/pom.xml | 4 +-
.../dolphinscheduler-alert-script/pom.xml | 4 +-
.../dolphinscheduler-alert-slack/pom.xml | 4 +-
.../dolphinscheduler-alert-wechat/pom.xml | 4 +-
dolphinscheduler-alert-plugin/pom.xml | 4 +-
dolphinscheduler-alert/pom.xml | 2 +-
.../dolphinscheduler/alert/runner/AlertSender.java | 7 +-
dolphinscheduler-api/pom.xml | 2 +-
.../impl/AlertPluginInstanceServiceImpl.java | 3 +-
.../api/service/impl/EnvironmentServiceImpl.java | 3 +-
.../api/service/impl/ExecutorServiceImpl.java | 43 +-
.../api/service/impl/LoggerServiceImpl.java | 19 +-
.../api/service/impl/MonitorServiceImpl.java | 69 +--
.../service/impl/ProcessDefinitionServiceImpl.java | 1 +
.../service/impl/ProcessInstanceServiceImpl.java | 15 +-
.../api/service/impl/ResourcesServiceImpl.java | 77 ++--
.../api/service/impl/SessionServiceImpl.java | 19 +-
.../api/service/impl/TenantServiceImpl.java | 17 +-
.../api/service/impl/UiPluginServiceImpl.java | 7 +-
.../api/service/impl/UsersServiceImpl.java | 2 +-
.../api/service/impl/WorkerGroupServiceImpl.java | 31 +-
.../dolphinscheduler/api/utils/FileUtils.java | 29 +-
.../api/utils/FourLetterWordMain.java | 95 -----
.../dolphinscheduler/api/utils/RegexUtils.java | 18 -
.../api/utils/RegistryCenterUtils.java | 82 ----
.../api/controller/AbstractControllerTest.java | 17 +-
.../api/controller/SchedulerControllerTest.java | 3 -
.../api/controller/TaskInstanceControllerTest.java | 3 -
.../api/controller/TenantControllerTest.java | 3 -
.../api/controller/WorkerGroupControllerTest.java | 7 +-
.../api/service/AlertGroupServiceTest.java | 20 +-
.../api/service/EnvironmentServiceTest.java | 3 +-
.../api/service/MonitorServiceTest.java | 3 +-
.../api/service/ProjectServiceTest.java | 3 +-
.../api/service/QueueServiceTest.java | 3 +-
.../api/service/ResourcesServiceTest.java | 50 +--
.../api/service/TenantServiceTest.java | 14 +-
.../api/service/UdfFuncServiceTest.java | 5 +-
.../api/service/UsersServiceTest.java | 7 +-
.../api/service/WorkerGroupServiceTest.java | 96 +----
.../dolphinscheduler/api/utils/FileUtilsTest.java | 7 +-
.../api/utils/FourLetterWordMainTest.java | 217 ----------
.../dolphinscheduler/api/utils/RegexUtilsTest.java | 13 +-
.../api/utils/RegistryCenterUtilsTest.java | 43 --
dolphinscheduler-common/pom.xml | 2 +-
.../apache/dolphinscheduler/common/Constants.java | 8 +-
.../dolphinscheduler/common/enums/NodeType.java | 11 +-
.../apache/dolphinscheduler/common/graph/DAG.java | 19 +-
.../common/task/AbstractParameters.java | 2 +-
.../common/task/sql/SqlParameters.java | 2 +-
.../dolphinscheduler/common/utils/ArrayUtils.java | 38 --
.../common/utils/CollectionUtils.java | 223 +---------
.../dolphinscheduler/common/utils/EnumUtils.java | 48 ---
.../dolphinscheduler/common/utils/FileUtils.java | 73 +---
.../dolphinscheduler/common/utils/LoggerUtils.java | 29 +-
.../dolphinscheduler/common/utils/OSUtils.java | 130 ++----
.../common/utils/ParameterUtils.java | 94 -----
.../common/utils/RetryerUtils.java | 41 +-
.../dolphinscheduler/common/utils/SchemaUtils.java | 46 +-
.../common/utils/SensitiveLogUtils.java | 45 --
.../placeholder/PropertyPlaceholderHelper.java | 84 +---
.../utils/placeholder/TimePlaceholderUtils.java | 6 +-
.../dolphinscheduler/common/ConstantsTest.java | 5 +-
.../dolphinscheduler/common/os/OSUtilsTest.java | 10 -
.../common/task/FlinkParametersTest.java | 4 +-
.../common/task/SparkParametersTest.java | 3 +-
.../common/task/SqlParametersTest.java | 3 +-
.../common/utils/CollectionUtilsTest.java | 161 -------
.../common/utils/FileUtilsTest.java | 8 -
.../common/utils/LoggerUtilsTest.java | 85 ----
.../dolphinscheduler/common/utils/OSUtilsTest.java | 8 +-
.../common/utils/RetryerUtilsTest.java | 109 -----
.../common/utils/SchemaUtilsTest.java | 3 +-
.../common/utils/SensitiveLogUtilsTest.java | 37 --
dolphinscheduler-dao/pom.xml | 2 +-
.../org/apache/dolphinscheduler/dao/PluginDao.java | 3 +-
.../dolphinscheduler/dao/upgrade/UpgradeDao.java | 2 +-
.../dolphinscheduler/dao/utils/DagHelper.java | 6 +-
.../dao/utils/ResourceProcessDefinitionUtils.java | 2 +-
.../dao/mapper/ResourceMapperTest.java | 2 +-
dolphinscheduler-dist/pom.xml | 2 +-
.../src/main/provisio/dolphinscheduler.xml | 5 -
.../ZookeeperConnectionStateListener.java | 56 ---
.../zookeeper/ZookeeperRegistryPlugin.java | 34 --
.../dolphinscheduler-registry-api/pom.xml | 32 ++
.../registry/api/ConnectionListener.java | 25 ++
.../registry/api/ConnectionState.java | 27 ++
.../dolphinscheduler/registry/api/Event.java | 48 +++
.../dolphinscheduler/registry/api/Registry.java | 48 +++
.../registry/api/RegistryException.java | 31 ++
.../registry/api/RegistryFactory.java | 26 ++
.../registry/api/RegistryFactoryLoader.java | 35 ++
.../registry/api/SubscribeListener.java | 24 ++
.../dolphinscheduler-registry-zookeeper/pom.xml | 20 +-
.../registry/zookeeper/ZookeeperConfiguration.java | 0
.../ZookeeperConnectionStateListener.java | 54 +++
.../registry/zookeeper/ZookeeperRegistry.java | 229 ++++------
.../zookeeper/ZookeeperRegistryFactory.java | 14 +-
.../registry/zookeeper/ZookeeperRegistryTest.java | 23 +-
.../dolphinscheduler-registry-plugins/pom.xml | 36 ++
.../pom.xml | 27 +-
dolphinscheduler-remote/pom.xml | 2 +-
dolphinscheduler-server/pom.xml | 2 +-
.../server/log/SensitiveDataConverter.java | 11 +-
.../master/dispatch/host/CommonHostManager.java | 2 +-
.../dispatch/host/LowerWeightHostManager.java | 25 +-
.../dispatch/host/assign/AbstractSelector.java | 2 +-
.../master/registry/MasterRegistryClient.java | 73 ++--
.../registry/MasterRegistryDataListener.java | 40 +-
.../server/master/registry/ServerNodeManager.java | 58 +--
.../master/runner/WorkflowExecuteThread.java | 56 +--
.../master/runner/task/BaseTaskProcessor.java | 12 +-
.../server/monitor/AbstractMonitor.java | 3 +-
.../server/monitor/RegistryMonitorImpl.java | 22 +-
.../server/registry/HeartBeatTask.java | 10 +-
.../dolphinscheduler/server/utils/ArgsUtils.java | 30 --
.../server/utils/ProcessUtils.java | 5 +-
.../worker/processor/TaskCallbackService.java | 6 -
.../worker/processor/TaskExecuteProcessor.java | 4 +-
.../worker/registry/WorkerRegistryClient.java | 7 +-
.../src/main/resources/config/install_config.conf | 5 -
.../server/log/SensitiveDataConverterTest.java | 114 +----
.../master/dispatch/ExecutorDispatcherTest.java | 2 +-
.../master/registry/MasterRegistryClientTest.java | 16 +-
.../server/utils/ProcessUtilsTest.java | 10 +-
dolphinscheduler-service/pom.xml | 10 +-
.../service/log/LogClientService.java | 7 +-
.../service/permission/PermissionCheck.java | 3 +-
.../service/process/ProcessService.java | 104 +----
.../service/quartz/cron/CronUtils.java | 3 +-
.../service/registry/RegistryCenter.java | 243 -----------
.../service/registry/RegistryClient.java | 467 ++++++++-------------
.../src/main/resources/registry.properties | 13 -
.../service/registry/RegistryClientTest.java | 74 ----
.../service/registry/RegistryPluginTest.java | 45 --
dolphinscheduler-spi/pom.xml | 4 +-
.../spi/DolphinSchedulerPlugin.java | 9 -
.../spi/register/ConnectStateListener.java | 23 -
.../spi/register/DataChangeEvent.java | 37 --
.../spi/register/ListenerManager.java | 66 ---
.../dolphinscheduler/spi/register/Registry.java | 102 -----
.../spi/register/RegistryConnectListener.java | 23 -
.../spi/register/RegistryConnectState.java | 37 --
.../spi/register/RegistryException.java | 32 --
.../spi/register/RegistryFactory.java | 34 --
.../spi/register/RegistryPluginManager.java | 82 ----
.../spi/register/SubscribeListener.java | 30 --
dolphinscheduler-standalone-server/pom.xml | 2 +-
.../dolphinscheduler/server/StandaloneServer.java | 9 -
.../dolphinscheduler-task-api/pom.xml | 4 +-
.../dolphinscheduler-task-datax/pom.xml | 2 +-
.../dolphinscheduler-task-flink/pom.xml | 4 +-
.../dolphinscheduler-task-http/pom.xml | 2 +-
.../dolphinscheduler-task-mr/pom.xml | 4 +-
.../dolphinscheduler-task-pigeon/pom.xml | 4 +-
.../dolphinscheduler-task-procedure/pom.xml | 2 +-
.../dolphinscheduler-task-python/pom.xml | 4 +-
.../dolphinscheduler-task-shell/pom.xml | 4 +-
.../dolphinscheduler-task-spark/pom.xml | 2 +-
.../dolphinscheduler-task-sql/pom.xml | 4 +-
.../dolphinscheduler-task-sqoop/pom.xml | 4 +-
dolphinscheduler-task-plugin/pom.xml | 4 +-
dolphinscheduler-ui/pom.xml | 2 +-
pom.xml | 30 +-
script/env/dolphinscheduler_env.sh | 2 +-
178 files changed, 1291 insertions(+), 4062 deletions(-)
diff --git a/dolphinscheduler-standalone-server/src/main/resources/registry.properties b/.github/CODEOWNERS
similarity index 74%
rename from dolphinscheduler-standalone-server/src/main/resources/registry.properties
rename to .github/CODEOWNERS
index 3c33799..9def4be 100644
--- a/dolphinscheduler-standalone-server/src/main/resources/registry.properties
+++ b/.github/CODEOWNERS
@@ -15,8 +15,6 @@
# limitations under the License.
#
-# This file is only to override the production configurations in standalone server.
-
-registry.plugin.dir=./dolphinscheduler-dist/target/dolphinscheduler-dist-2.0.0-SNAPSHOT/lib/plugin/registry/zookeeper
-registry.plugin.name=zookeeper
-registry.servers=127.0.0.1:2181
+dolphinscheduler/dolphinscheduler-e2e @kezhenxu94
+dolphinscheduler/dolphinscheduler-registry @kezhenxu94
+dolphinscheduler/dolphinscheduler-standalone-server @kezhenxu94
diff --git a/.github/actions/sanity-check/action.yml b/.github/actions/sanity-check/action.yml
index a1d03a3..419b499 100644
--- a/.github/actions/sanity-check/action.yml
+++ b/.github/actions/sanity-check/action.yml
@@ -38,16 +38,3 @@ runs:
with:
reviewdog_version: v0.10.2
- - shell: bash
- run: ./mvnw -B -q checkstyle:checkstyle-aggregate
-
- - shell: bash
- env:
- REVIEWDOG_GITHUB_API_TOKEN: ${{ inputs.token }}
- run: |
- if [[ -n "${{ inputs.token }}" ]]; then
- reviewdog -f=checkstyle \
- -reporter="github-pr-check" \
- -filter-mode="added" \
- -fail-on-error="true" < target/checkstyle-result.xml
- fi
diff --git a/docker/build/conf/dolphinscheduler/registry.properties.tpl b/docker/build/conf/dolphinscheduler/registry.properties.tpl
index 40836f5..9ee8add 100644
--- a/docker/build/conf/dolphinscheduler/registry.properties.tpl
+++ b/docker/build/conf/dolphinscheduler/registry.properties.tpl
@@ -15,13 +15,5 @@
# limitations under the License.
#
-#registry.plugin.dir config the Registry Plugin dir.
-registry.plugin.dir=${REGISTRY_PLUGIN_DIR}
-
registry.plugin.name=${REGISTRY_PLUGIN_NAME}
registry.servers=${REGISTRY_SERVERS}
-
-#maven.local.repository=/usr/local/localRepository
-
-#registry.plugin.binding config the Registry Plugin need be load when development and run in IDE
-#registry.plugin.binding=./dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml
diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh
index 35f70bb..4717834 100755
--- a/docker/build/startup-init-conf.sh
+++ b/docker/build/startup-init-conf.sh
@@ -37,7 +37,6 @@ export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
#============================================================================
# Registry
#============================================================================
-export REGISTRY_PLUGIN_DIR=${REGISTRY_PLUGIN_DIR:-"lib/plugin/registry"}
export REGISTRY_PLUGIN_NAME=${REGISTRY_PLUGIN_NAME:-"zookeeper"}
export REGISTRY_SERVERS=${REGISTRY_SERVERS:-"127.0.0.1:2181"}
diff --git a/docker/docker-swarm/config.env.sh b/docker/docker-swarm/config.env.sh
index afc09b0..0c79a04 100755
--- a/docker/docker-swarm/config.env.sh
+++ b/docker/docker-swarm/config.env.sh
@@ -39,7 +39,6 @@ DATABASE_PARAMS=characterEncoding=utf8
#============================================================================
# Registry
#============================================================================
-REGISTRY_PLUGIN_DIR=lib/plugin/registry
REGISTRY_PLUGIN_NAME=zookeeper
REGISTRY_SERVERS=dolphinscheduler-zookeeper:2181
diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml
index 5e42c9c..56fc66e 100644
--- a/docker/docker-swarm/docker-compose.yml
+++ b/docker/docker-swarm/docker-compose.yml
@@ -44,7 +44,7 @@ services:
- dolphinscheduler
dolphinscheduler-api:
- image: apache/dolphinscheduler:2.0.0
+ image: apache/dolphinscheduler:2.0.1-SNAPSHOT
command: api-server
ports:
- 12345:12345
@@ -68,7 +68,7 @@ services:
- dolphinscheduler
dolphinscheduler-alert:
- image: apache/dolphinscheduler:2.0.0
+ image: apache/dolphinscheduler:2.0.1-SNAPSHOT
command: alert-server
environment:
TZ: Asia/Shanghai
@@ -87,7 +87,7 @@ services:
- dolphinscheduler
dolphinscheduler-master:
- image: apache/dolphinscheduler:2.0.0
+ image: apache/dolphinscheduler:2.0.1-SNAPSHOT
command: master-server
environment:
TZ: Asia/Shanghai
@@ -108,7 +108,7 @@ services:
- dolphinscheduler
dolphinscheduler-worker:
- image: apache/dolphinscheduler:2.0.0
+ image: apache/dolphinscheduler:2.0.1-SNAPSHOT
command: worker-server
environment:
TZ: Asia/Shanghai
diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml
index 03de21d..9bd192f 100644
--- a/docker/docker-swarm/docker-stack.yml
+++ b/docker/docker-swarm/docker-stack.yml
@@ -48,7 +48,7 @@ services:
replicas: 1
dolphinscheduler-api:
- image: apache/dolphinscheduler:2.0.0
+ image: apache/dolphinscheduler:2.0.1-SNAPSHOT
command: api-server
ports:
- 12345:12345
@@ -71,7 +71,7 @@ services:
replicas: 1
dolphinscheduler-alert:
- image: apache/dolphinscheduler:2.0.0
+ image: apache/dolphinscheduler:2.0.1-SNAPSHOT
command: alert-server
environment:
TZ: Asia/Shanghai
@@ -90,7 +90,7 @@ services:
replicas: 1
dolphinscheduler-master:
- image: apache/dolphinscheduler:2.0.0
+ image: apache/dolphinscheduler:2.0.1-SNAPSHOT
command: master-server
environment:
TZ: Asia/Shanghai
@@ -110,7 +110,7 @@ services:
replicas: 1
dolphinscheduler-worker:
- image: apache/dolphinscheduler:2.0.0
+ image: apache/dolphinscheduler:2.0.1-SNAPSHOT
command: worker-server
environment:
TZ: Asia/Shanghai
diff --git a/docker/kubernetes/dolphinscheduler/Chart.yaml b/docker/kubernetes/dolphinscheduler/Chart.yaml
index 0ec4edf..bf337e5 100644
--- a/docker/kubernetes/dolphinscheduler/Chart.yaml
+++ b/docker/kubernetes/dolphinscheduler/Chart.yaml
@@ -35,11 +35,11 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
-version: 2.0.0
+version: 2.0.1-SNAPSHOT
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application.
-appVersion: 2.0.0
+appVersion: 2.0.1-SNAPSHOT
dependencies:
- name: postgresql
diff --git a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl
index 2b0786f..5ef83a7 100644
--- a/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl
+++ b/docker/kubernetes/dolphinscheduler/templates/_helpers.tpl
@@ -166,12 +166,6 @@ Create a database environment variables.
Create a registry environment variables.
*/}}
{{- define "dolphinscheduler.registry.env_vars" -}}
-- name: REGISTRY_PLUGIN_DIR
- {{- if .Values.zookeeper.enabled }}
- value: "lib/plugin/registry"
- {{- else }}
- value: {{ .Values.externalRegistry.registryPluginDir }}
- {{- end }}
- name: REGISTRY_PLUGIN_NAME
{{- if .Values.zookeeper.enabled }}
value: "zookeeper"
@@ -239,4 +233,4 @@ Create a fsFileResourcePersistence volumeMount.
- mountPath: {{ default "/dolphinscheduler" .Values.common.configmap.RESOURCE_UPLOAD_PATH | quote }}
name: {{ include "dolphinscheduler.fullname" . }}-fs-file
{{- end -}}
-{{- end -}}
\ No newline at end of file
+{{- end -}}
diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml
index 86fe553..4c90cc2 100644
--- a/docker/kubernetes/dolphinscheduler/values.yaml
+++ b/docker/kubernetes/dolphinscheduler/values.yaml
@@ -23,7 +23,7 @@ timezone: "Asia/Shanghai"
image:
repository: "apache/dolphinscheduler"
- tag: "2.0.0"
+ tag: "2.0.1-SNAPSHOT"
pullPolicy: "IfNotPresent"
pullSecret: ""
diff --git a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/pom.xml b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/pom.xml
index 4810e55..96257e5 100644
--- a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/pom.xml
+++ b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-dingtalk/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-alert-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -79,4 +79,4 @@
<finalName>dolphinscheduler-alert-dingtalk-${project.version}</finalName>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/pom.xml b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/pom.xml
index 0e97b92..9d008e5 100644
--- a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/pom.xml
+++ b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-email/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-alert-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/pom.xml b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/pom.xml
index 8fe5cb7..d6238ab 100644
--- a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/pom.xml
+++ b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-feishu/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-alert-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -79,4 +79,4 @@
<finalName>dolphinscheduler-alert-feishu-${project.version}</finalName>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-http/pom.xml b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-http/pom.xml
index 8bfdc2e..99a0918 100644
--- a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-http/pom.xml
+++ b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-http/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-alert-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -73,4 +73,4 @@
<finalName>dolphinscheduler-alert-http-${project.version}</finalName>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/pom.xml b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/pom.xml
index 77ee663..34ff065 100644
--- a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/pom.xml
+++ b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-script/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-alert-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -75,4 +75,4 @@
<finalName>dolphinscheduler-alert-script-${project.version}</finalName>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-slack/pom.xml b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-slack/pom.xml
index 77e2cfe..5e85b32 100644
--- a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-slack/pom.xml
+++ b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-slack/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-alert-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -80,4 +80,4 @@
<finalName>dolphinscheduler-alert-slack-${project.version}</finalName>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/pom.xml b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/pom.xml
index 3097cd7..8d8ad43 100644
--- a/dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/pom.xml
+++ b/dolphinscheduler-alert-plugin/dolphinscheduler-alert-wechat/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-alert-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -74,4 +74,4 @@
<build>
<finalName>dolphinscheduler-alert-wechat-${project.version}</finalName>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-alert-plugin/pom.xml b/dolphinscheduler-alert-plugin/pom.xml
index ad4e343..0c980aa 100644
--- a/dolphinscheduler-alert-plugin/pom.xml
+++ b/dolphinscheduler-alert-plugin/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -46,4 +46,4 @@
</dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml
index 456c076..3bcccb1 100644
--- a/dolphinscheduler-alert/pom.xml
+++ b/dolphinscheduler-alert/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-alert</artifactId>
<name>${project.artifactId}</name>
diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java
index d7bcc2c..726dcef 100644
--- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java
+++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/runner/AlertSender.java
@@ -17,9 +17,9 @@
package org.apache.dolphinscheduler.alert.runner;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.dolphinscheduler.alert.plugin.AlertPluginManager;
import org.apache.dolphinscheduler.common.enums.AlertStatus;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.entity.Alert;
@@ -30,14 +30,13 @@ import org.apache.dolphinscheduler.spi.alert.AlertChannel;
import org.apache.dolphinscheduler.spi.alert.AlertData;
import org.apache.dolphinscheduler.spi.alert.AlertInfo;
import org.apache.dolphinscheduler.spi.alert.AlertResult;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
public class AlertSender {
private static final Logger logger = LoggerFactory.getLogger(AlertSender.class);
diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml
index a25256d..3cdaf24 100644
--- a/dolphinscheduler-api/pom.xml
+++ b/dolphinscheduler-api/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-api</artifactId>
<name>${project.artifactId}</name>
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java
index f271960..5592e2c 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/AlertPluginInstanceServiceImpl.java
@@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.api.vo.AlertPluginInstanceVO;
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.AlertPluginInstance;
import org.apache.dolphinscheduler.dao.entity.PluginDefine;
@@ -33,6 +32,8 @@ import org.apache.dolphinscheduler.dao.mapper.AlertPluginInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.PluginDefineMapper;
import org.apache.dolphinscheduler.spi.params.PluginParamsTransfer;
+import org.apache.commons.collections.CollectionUtils;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java
index 89cd2af..ff0eea0 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/EnvironmentServiceImpl.java
@@ -25,7 +25,6 @@ import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.Environment;
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
@@ -34,7 +33,7 @@ import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper;
import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
-
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections4.SetUtils;
import org.apache.commons.lang.StringUtils;
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
index 02e5b57..6a4f179 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ExecutorServiceImpl.java
@@ -17,39 +17,21 @@
package org.apache.dolphinscheduler.api.service.impl;
-import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
-import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
-import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING;
-import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODES;
-import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_PARAMS;
-import static org.apache.dolphinscheduler.common.Constants.MAX_TASK_TIMEOUT;
-
import com.fasterxml.jackson.core.type.TypeReference;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.api.enums.ExecuteType;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.ExecutorService;
import org.apache.dolphinscheduler.api.service.MonitorService;
import org.apache.dolphinscheduler.api.service.ProjectService;
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.enums.CommandType;
-import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.common.enums.FailureStrategy;
-import org.apache.dolphinscheduler.common.enums.Priority;
-import org.apache.dolphinscheduler.common.enums.ReleaseState;
-import org.apache.dolphinscheduler.common.enums.RunMode;
-import org.apache.dolphinscheduler.common.enums.TaskDependType;
-import org.apache.dolphinscheduler.common.enums.WarningType;
+import org.apache.dolphinscheduler.common.enums.*;
import org.apache.dolphinscheduler.common.model.Server;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.dao.entity.Command;
-import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
-import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
-import org.apache.dolphinscheduler.dao.entity.Project;
-import org.apache.dolphinscheduler.dao.entity.Schedule;
-import org.apache.dolphinscheduler.dao.entity.Tenant;
-import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.entity.*;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
@@ -57,22 +39,15 @@ import org.apache.dolphinscheduler.remote.command.StateEventChangeCommand;
import org.apache.dolphinscheduler.remote.processor.StateEventCallbackService;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
-
-import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
+import java.util.*;
+
+import static org.apache.dolphinscheduler.common.Constants.*;
+
/**
* executor service impl
*/
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java
index df79464..c9965fe 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java
@@ -17,30 +17,27 @@
package org.apache.dolphinscheduler.api.service.impl;
+import com.google.common.primitives.Bytes;
+import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.service.LoggerService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.utils.ArrayUtils;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.dolphinscheduler.service.process.ProcessService;
-
-import org.apache.commons.lang.StringUtils;
-
-import java.nio.charset.StandardCharsets;
-import java.util.Objects;
-
-import javax.annotation.PostConstruct;
-import javax.annotation.PreDestroy;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+
/**
* logger service impl
*/
@@ -129,7 +126,7 @@ public class LoggerServiceImpl implements LoggerService {
taskInstance.getLogPath(),
host,
Constants.SYSTEM_LINE_SEPARATOR).getBytes(StandardCharsets.UTF_8);
- return ArrayUtils.addAll(head,
+ return Bytes.concat(head,
logClient.getLogBytes(host, Constants.RPC_PORT, taskInstance.getLogPath()));
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java
index 329f625..934aaf4 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/MonitorServiceImpl.java
@@ -19,14 +19,14 @@ package org.apache.dolphinscheduler.api.service.impl;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.MonitorService;
-import org.apache.dolphinscheduler.api.utils.RegistryCenterUtils;
import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.NodeType;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.model.WorkerServerModel;
import org.apache.dolphinscheduler.dao.MonitorDBDao;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
import org.apache.dolphinscheduler.dao.entity.User;
-import org.apache.dolphinscheduler.dao.entity.ZookeeperRecord;
+import org.apache.dolphinscheduler.service.registry.RegistryClient;
import java.util.HashMap;
import java.util.List;
@@ -48,6 +48,9 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
@Autowired
private MonitorDBDao monitorDBDao;
+ @Autowired
+ private RegistryClient registryClient;
+
/**
* query database state
*
@@ -55,7 +58,7 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
* @return data base state
*/
@Override
- public Map<String,Object> queryDatabaseState(User loginUser) {
+ public Map<String, Object> queryDatabaseState(User loginUser) {
Map<String, Object> result = new HashMap<>();
List<MonitorRecord> monitorRecordList = monitorDBDao.queryDatabaseState();
@@ -74,13 +77,13 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
* @return master information list
*/
@Override
- public Map<String,Object> queryMaster(User loginUser) {
+ public Map<String, Object> queryMaster(User loginUser) {
Map<String, Object> result = new HashMap<>();
List<Server> masterServers = getServerListFromRegistry(true);
result.put(Constants.DATA_LIST, masterServers);
- putMsg(result,Status.SUCCESS);
+ putMsg(result, Status.SUCCESS);
return result;
}
@@ -92,46 +95,48 @@ public class MonitorServiceImpl extends BaseServiceImpl implements MonitorServic
* @return worker information list
*/
@Override
- public Map<String,Object> queryWorker(User loginUser) {
+ public Map<String, Object> queryWorker(User loginUser) {
Map<String, Object> result = new HashMap<>();
List<WorkerServerModel> workerServers = getServerListFromRegistry(false)
- .stream()
- .map((Server server) -> {
- WorkerServerModel model = new WorkerServerModel();
- model.setId(server.getId());
- model.setHost(server.getHost());
- model.setPort(server.getPort());
- model.setZkDirectories(Sets.newHashSet(server.getZkDirectory()));
- model.setResInfo(server.getResInfo());
- model.setCreateTime(server.getCreateTime());
- model.setLastHeartbeatTime(server.getLastHeartbeatTime());
- return model;
- })
- .collect(Collectors.toList());
+ .stream()
+ .map((Server server) -> {
+ WorkerServerModel model = new WorkerServerModel();
+ model.setId(server.getId());
+ model.setHost(server.getHost());
+ model.setPort(server.getPort());
+ model.setZkDirectories(Sets.newHashSet(server.getZkDirectory()));
+ model.setResInfo(server.getResInfo());
+ model.setCreateTime(server.getCreateTime());
+ model.setLastHeartbeatTime(server.getLastHeartbeatTime());
+ return model;
+ })
+ .collect(Collectors.toList());
Map<String, WorkerServerModel> workerHostPortServerMapping = workerServers
- .stream()
- .collect(Collectors.toMap(
- (WorkerServerModel worker) -> {
- String[] s = worker.getZkDirectories().iterator().next().split("/");
- return s[s.length - 1];
- }
- , Function.identity()
- , (WorkerServerModel oldOne, WorkerServerModel newOne) -> {
- oldOne.getZkDirectories().addAll(newOne.getZkDirectories());
- return oldOne;
- }));
+ .stream()
+ .collect(Collectors.toMap(
+ (WorkerServerModel worker) -> {
+ String[] s = worker.getZkDirectories().iterator().next().split("/");
+ return s[s.length - 1];
+ }
+ , Function.identity()
+ , (WorkerServerModel oldOne, WorkerServerModel newOne) -> {
+ oldOne.getZkDirectories().addAll(newOne.getZkDirectories());
+ return oldOne;
+ }));
result.put(Constants.DATA_LIST, workerHostPortServerMapping.values());
- putMsg(result,Status.SUCCESS);
+ putMsg(result, Status.SUCCESS);
return result;
}
@Override
public List<Server> getServerListFromRegistry(boolean isMaster) {
- return isMaster ? RegistryCenterUtils.getMasterServers() : RegistryCenterUtils.getWorkerServers();
+ return isMaster
+ ? registryClient.getServerList(NodeType.MASTER)
+ : registryClient.getServerList(NodeType.WORKER);
}
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
index 96318bc..6bdc8dc 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
@@ -68,6 +68,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.commons.collections4.CollectionUtils;
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
index 3df2060..22cfcd0 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
@@ -46,7 +46,6 @@ import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
import org.apache.dolphinscheduler.common.process.Property;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
@@ -68,6 +67,7 @@ import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import java.io.BufferedReader;
@@ -76,11 +76,13 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.function.Function;
import java.util.stream.Collectors;
import org.springframework.beans.factory.annotation.Autowired;
@@ -265,8 +267,15 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
project.getCode(), processDefineCode, searchVal, executorId, statusArray, host, start, end);
List<ProcessInstance> processInstances = processInstanceList.getRecords();
- List<Integer> userIds = CollectionUtils.transformToList(processInstances, ProcessInstance::getExecutorId);
- Map<Integer, User> idToUserMap = CollectionUtils.collectionToMap(usersService.queryUser(userIds), User::getId);
+ List<Integer> userIds = Collections.emptyList();
+ if (CollectionUtils.isNotEmpty(processInstances)) {
+ userIds = processInstances.stream().map(ProcessInstance::getExecutorId).collect(Collectors.toList());
+ }
+ List<User> users = usersService.queryUser(userIds);
+ Map<Integer, User> idToUserMap = Collections.emptyMap();
+ if (CollectionUtils.isNotEmpty(users)) {
+ idToUserMap = users.stream().collect(Collectors.toMap(User::getId, Function.identity()));
+ }
for (ProcessInstance processInstance : processInstances) {
processInstance.setDuration(DateUtils.format2Duration(processInstance.getStartTime(), processInstance.getEndTime()));
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
index 7835716..d9e250c 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
@@ -17,10 +17,13 @@
package org.apache.dolphinscheduler.api.service.impl;
-import static org.apache.dolphinscheduler.common.Constants.ALIAS;
-import static org.apache.dolphinscheduler.common.Constants.CONTENT;
-import static org.apache.dolphinscheduler.common.Constants.JAR;
-
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.google.common.io.Files;
+import org.apache.commons.beanutils.BeanMap;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
import org.apache.dolphinscheduler.api.dto.resources.filter.ResourceFilter;
import org.apache.dolphinscheduler.api.dto.resources.visitor.ResourceTreeVisitor;
@@ -33,42 +36,14 @@ import org.apache.dolphinscheduler.api.utils.RegexUtils;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ProgramType;
-import org.apache.dolphinscheduler.spi.enums.ResourceType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
-import org.apache.dolphinscheduler.dao.entity.Resource;
-import org.apache.dolphinscheduler.dao.entity.ResourcesUser;
-import org.apache.dolphinscheduler.dao.entity.Tenant;
-import org.apache.dolphinscheduler.dao.entity.UdfFunc;
-import org.apache.dolphinscheduler.dao.entity.User;
-import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
-import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
-import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
-import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
-import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
-import org.apache.dolphinscheduler.dao.mapper.UserMapper;
+import org.apache.dolphinscheduler.dao.entity.*;
+import org.apache.dolphinscheduler.dao.mapper.*;
import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils;
-
-import org.apache.commons.beanutils.BeanMap;
-import org.apache.commons.lang.StringUtils;
-
-import java.io.IOException;
-import java.text.MessageFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.regex.Matcher;
-import java.util.stream.Collectors;
-
+import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -77,9 +52,13 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
-import com.fasterxml.jackson.databind.SerializationFeature;
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.stream.Collectors;
+
+import static org.apache.dolphinscheduler.common.Constants.*;
/**
* resources service impl
@@ -330,8 +309,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
if (!resource.isDirectory()) {
//get the origin file suffix
- String originSuffix = FileUtils.suffix(originFullName);
- String suffix = FileUtils.suffix(fullName);
+ String originSuffix = Files.getFileExtension(originFullName);
+ String suffix = Files.getFileExtension(fullName);
boolean suffixIsChanged = false;
if (StringUtils.isBlank(suffix) && StringUtils.isNotBlank(originSuffix)) {
suffixIsChanged = true;
@@ -474,8 +453,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
// file suffix
- String fileSuffix = FileUtils.suffix(file.getOriginalFilename());
- String nameSuffix = FileUtils.suffix(name);
+ String fileSuffix = Files.getFileExtension(file.getOriginalFilename());
+ String nameSuffix = Files.getFileExtension(name);
// determine file suffix
if (!(StringUtils.isNotEmpty(fileSuffix) && fileSuffix.equalsIgnoreCase(nameSuffix))) {
@@ -576,8 +555,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
*/
private boolean upload(User loginUser, String fullName, MultipartFile file, ResourceType type) {
// save to local
- String fileSuffix = FileUtils.suffix(file.getOriginalFilename());
- String nameSuffix = FileUtils.suffix(fullName);
+ String fileSuffix = Files.getFileExtension(file.getOriginalFilename());
+ String nameSuffix = Files.getFileExtension(fullName);
// determine file suffix
if (!(StringUtils.isNotEmpty(fileSuffix) && fileSuffix.equalsIgnoreCase(nameSuffix))) {
@@ -599,11 +578,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
org.apache.dolphinscheduler.api.utils.FileUtils.copyFile(file, localFilename);
HadoopUtils.getInstance().copyLocalToHdfs(localFilename, hdfsFilename, true, true);
} catch (Exception e) {
- try {
- FileUtils.deleteFile(localFilename);
- } catch (IOException ex) {
- logger.error("delete local tmp file:{} error", localFilename, ex);
- }
+ FileUtils.deleteFile(localFilename);
logger.error(e.getMessage(), e);
return false;
}
@@ -840,7 +815,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
return result;
}
//check preview or not by file suffix
- String nameSuffix = FileUtils.suffix(resource.getAlias());
+ String nameSuffix = Files.getFileExtension(resource.getAlias());
String resourceViewSuffixs = FileUtils.getResourceViewSuffixs();
if (StringUtils.isNotEmpty(resourceViewSuffixs)) {
List<String> strList = Arrays.asList(resourceViewSuffixs.split(","));
@@ -1005,7 +980,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
return result;
}
//check can edit by file suffix
- String nameSuffix = FileUtils.suffix(resource.getAlias());
+ String nameSuffix = Files.getFileExtension(resource.getAlias());
String resourceViewSuffixs = FileUtils.getResourceViewSuffixs();
if (StringUtils.isNotEmpty(resourceViewSuffixs)) {
List<String> strList = Arrays.asList(resourceViewSuffixs.split(","));
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java
index de1c417..c7f55fc 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/SessionServiceImpl.java
@@ -17,23 +17,14 @@
package org.apache.dolphinscheduler.api.service.impl;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.api.controller.BaseController;
import org.apache.dolphinscheduler.api.service.SessionService;
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.Session;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.SessionMapper;
-
-import org.apache.commons.lang.StringUtils;
-
-import java.util.Date;
-import java.util.List;
-import java.util.UUID;
-
-import javax.servlet.http.Cookie;
-import javax.servlet.http.HttpServletRequest;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -41,6 +32,12 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.util.WebUtils;
+import javax.servlet.http.Cookie;
+import javax.servlet.http.HttpServletRequest;
+import java.util.Date;
+import java.util.List;
+import java.util.UUID;
+
/**
* session service implement
*/
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
index fc01bd7..d0ccd75 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java
@@ -17,13 +17,16 @@
package org.apache.dolphinscheduler.api.service.impl;
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.TenantService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.RegexUtils;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
@@ -34,21 +37,15 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
-
-import org.apache.commons.lang.StringUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
-
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
-
/**
* tenant service impl
*/
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UiPluginServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UiPluginServiceImpl.java
index 0f14839..0fe6e27 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UiPluginServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UiPluginServiceImpl.java
@@ -17,21 +17,20 @@
package org.apache.dolphinscheduler.api.service.impl;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.UiPluginService;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.PluginType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.PluginDefine;
import org.apache.dolphinscheduler.dao.mapper.PluginDefineMapper;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
/**
* ui plugin service impl
*/
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
index a45482e..a98d45f 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
@@ -29,7 +29,6 @@ import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UserType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.EncryptionUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
@@ -55,6 +54,7 @@ import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import java.io.IOException;
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java
index 379af64..46a9241 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java
@@ -20,25 +20,27 @@ package org.apache.dolphinscheduler.api.service.impl;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.WorkerGroupService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
-import org.apache.dolphinscheduler.api.utils.RegistryCenterUtils;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.NodeType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
+import org.apache.dolphinscheduler.service.registry.RegistryClient;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.stream.Collectors;
import org.slf4j.Logger;
@@ -58,10 +60,13 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
private static final Logger logger = LoggerFactory.getLogger(WorkerGroupServiceImpl.class);
@Autowired
- WorkerGroupMapper workerGroupMapper;
+ private WorkerGroupMapper workerGroupMapper;
@Autowired
- ProcessInstanceMapper processInstanceMapper;
+ private ProcessInstanceMapper processInstanceMapper;
+
+ @Autowired
+ private RegistryClient registryClient;
/**
* create or update a worker group
@@ -139,7 +144,7 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
}
// check zookeeper
String workerGroupPath = Constants.REGISTRY_DOLPHINSCHEDULER_WORKERS + Constants.SINGLE_SLASH + workerGroup.getName();
- return RegistryCenterUtils.isNodeExisted(workerGroupPath);
+ return registryClient.exists(workerGroupPath);
}
/**
@@ -149,7 +154,7 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
* @return boolean
*/
private String checkWorkerGroupAddrList(WorkerGroup workerGroup) {
- Map<String, String> serverMaps = RegistryCenterUtils.getServerMaps(NodeType.WORKER, true);
+ Map<String, String> serverMaps = registryClient.getServerMaps(NodeType.WORKER, true);
if (Strings.isNullOrEmpty(workerGroup.getAddrList())) {
return null;
}
@@ -250,11 +255,11 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
List<WorkerGroup> workerGroups = workerGroupMapper.queryAllWorkerGroup();
// worker groups from zookeeper
String workerPath = Constants.REGISTRY_DOLPHINSCHEDULER_WORKERS;
- List<String> workerGroupList = null;
+ Collection<String> workerGroupList = null;
try {
- workerGroupList = RegistryCenterUtils.getChildrenNodes(workerPath);
+ workerGroupList = registryClient.getChildrenKeys(workerPath);
} catch (Exception e) {
- logger.error("getWorkerGroups exception: {}, workerPath: {}, isPaging: {}", e.getMessage(), workerPath, isPaging);
+ logger.error("getWorkerGroups exception, workerPath: {}, isPaging: {}", workerPath, isPaging, e);
}
if (CollectionUtils.isEmpty(workerGroupList)) {
@@ -268,9 +273,9 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
for (String workerGroup : workerGroupList) {
String workerGroupPath = workerPath + Constants.SINGLE_SLASH + workerGroup;
- List<String> childrenNodes = null;
+ Collection<String> childrenNodes = null;
try {
- childrenNodes = RegistryCenterUtils.getChildrenNodes(workerGroupPath);
+ childrenNodes = registryClient.getChildrenKeys(workerGroupPath);
} catch (Exception e) {
logger.error("getChildrenNodes exception: {}, workerGroupPath: {}", e.getMessage(), workerGroupPath);
}
@@ -281,7 +286,7 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
wg.setName(workerGroup);
if (isPaging) {
wg.setAddrList(String.join(Constants.COMMA, childrenNodes));
- String registeredValue = RegistryCenterUtils.getNodeData(workerGroupPath + Constants.SINGLE_SLASH + childrenNodes.get(0));
+ String registeredValue = registryClient.get(workerGroupPath + Constants.SINGLE_SLASH + childrenNodes.iterator().next());
wg.setCreateTime(DateUtils.stringToDate(registeredValue.split(Constants.COMMA)[6]));
wg.setUpdateTime(DateUtils.stringToDate(registeredValue.split(Constants.COMMA)[7]));
wg.setSystemDefault(true);
@@ -328,7 +333,7 @@ public class WorkerGroupServiceImpl extends BaseServiceImpl implements WorkerGro
@Override
public Map<String, Object> getWorkerAddressList() {
Map<String, Object> result = new HashMap<>();
- List<String> serverNodeList = RegistryCenterUtils.getServerNodeList(NodeType.WORKER, true);
+ Set<String> serverNodeList = registryClient.getServerNodeSet(NodeType.WORKER, true);
result.put(Constants.DATA_LIST, serverNodeList);
putMsg(result, Status.SUCCESS);
return result;
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FileUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FileUtils.java
index 355f6c9..6046a34 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FileUtils.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FileUtils.java
@@ -16,19 +16,17 @@
*/
package org.apache.dolphinscheduler.api.utils;
+import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.io.Resource;
import org.springframework.core.io.UrlResource;
import org.springframework.web.multipart.MultipartFile;
-import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
-import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -44,18 +42,9 @@ public class FileUtils {
* @param file file
* @param destFilename destination file name
*/
-
public static void copyFile(MultipartFile file, String destFilename) {
try {
-
- File destFile = new File(destFilename);
- File destParentDir = new File(destFile.getParent());
-
- if (!destParentDir.exists()) {
- org.apache.commons.io.FileUtils.forceMkdir(destParentDir);
- }
-
- Files.copy(file.getInputStream(), Paths.get(destFilename));
+ org.apache.commons.io.FileUtils.copyFile(file.getResource().getFile(), new File(destFilename));
} catch (IOException e) {
logger.error("failed to copy file , {} is empty file", file.getOriginalFilename(), e);
}
@@ -87,20 +76,12 @@ public class FileUtils {
* @return file content string
*/
public static String file2String(MultipartFile file) {
- StringBuilder strBuilder = new StringBuilder();
-
- try (InputStreamReader inputStreamReader = new InputStreamReader(file.getInputStream(), StandardCharsets.UTF_8)) {
- BufferedReader streamReader = new BufferedReader(inputStreamReader);
- String inputStr;
-
- while ((inputStr = streamReader.readLine()) != null) {
- strBuilder.append(inputStr);
- }
-
+ try {
+ return IOUtils.toString(file.getInputStream(), StandardCharsets.UTF_8);
} catch (IOException e) {
logger.error("file convert to string failed: {}", file.getName());
}
- return strBuilder.toString();
+ return "";
}
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java
deleted file mode 100644
index 340a389..0000000
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMain.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.api.utils;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.SocketTimeoutException;
-import java.util.Objects;
-
-public class FourLetterWordMain {
-
- private static final int DEFAULT_SOCKET_TIMEOUT = 5000;
- protected static final Logger LOG = LoggerFactory.getLogger(FourLetterWordMain.class);
-
- private FourLetterWordMain() {
- throw new IllegalStateException("FourLetterWordMain class");
- }
-
- /**
- * Send the 4letterword
- * @param host the destination host
- * @param port the destination port
- * @param cmd the 4letterword
- * @return server response
- * @throws java.io.IOException io exceptions
- */
- public static String send4LetterWord(String host, int port, String cmd)
- throws IOException {
- return send4LetterWord(host, port, cmd, DEFAULT_SOCKET_TIMEOUT);
- }
-
- /**
- * Send the 4letterword
- * @param host the destination host
- * @param port the destination port
- * @param cmd the 4letterword
- * @param timeout in milliseconds, maximum time to wait while connecting/reading data
- * @return server response
- * @throws java.io.IOException io exceptions
- */
- public static String send4LetterWord(String host, int port, String cmd, int timeout)
- throws IOException {
- Objects.requireNonNull(cmd, "cmd must not be null");
- LOG.info("connecting to {} {}", host, port);
- InetSocketAddress hostaddress= host != null ? new InetSocketAddress(host, port) :
- new InetSocketAddress(InetAddress.getByName(null), port);
-
- try (Socket sock = new Socket()) {
- sock.setSoTimeout(timeout);
- sock.connect(hostaddress, timeout);
- OutputStream outstream = sock.getOutputStream();
- outstream.write(cmd.getBytes());
- outstream.flush();
- // this replicates NC - close the output stream before reading
- sock.shutdownOutput();
-
- try (BufferedReader reader =
- new BufferedReader(
- new InputStreamReader(sock.getInputStream()))) {
- StringBuilder sb = new StringBuilder();
- String line;
- while ((line = reader.readLine()) != null) {
- sb.append(line + "\n");
- }
- return sb.toString();
- }
- } catch (SocketTimeoutException e) {
- throw new IOException("Exception while executing four letter word: " + cmd, e);
- }
- }
-}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java
index 2e43d2b..4ddf073 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegexUtils.java
@@ -17,7 +17,6 @@
package org.apache.dolphinscheduler.api.utils;
-import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
@@ -25,29 +24,12 @@ import java.util.regex.Pattern;
*/
public class RegexUtils {
- /**
- * check number regex expression
- */
- private static final String CHECK_NUMBER = "^-?\\d+(\\.\\d+)?$";
-
private static final String LINUX_USERNAME_PATTERN = "[a-z_][a-z\\d_]{0,30}";
private RegexUtils() {
}
/**
- * check if the input is number
- *
- * @param str input
- * @return
- */
- public static boolean isNumeric(String str) {
- Pattern pattern = Pattern.compile(CHECK_NUMBER);
- Matcher isNum = pattern.matcher(str);
- return isNum.matches();
- }
-
- /**
* check if the input is a valid linux username
* @param str input
* @return boolean
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegistryCenterUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegistryCenterUtils.java
deleted file mode 100644
index 71e0456..0000000
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/RegistryCenterUtils.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.api.utils;
-
-import org.apache.dolphinscheduler.common.enums.NodeType;
-import org.apache.dolphinscheduler.common.model.Server;
-import org.apache.dolphinscheduler.dao.entity.ZookeeperRecord;
-import org.apache.dolphinscheduler.service.registry.RegistryClient;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * monitor zookeeper info todo registry-spi
- * fixme Some of the information obtained in the api belongs to the unique information of zk.
- * I am not sure whether there is a good abstraction method. This is related to whether the specific plug-in is provided.
- */
-public class RegistryCenterUtils {
-
- private static RegistryClient registryClient = RegistryClient.getInstance();
-
- /**
- * @return zookeeper info list
- */
- public static List<ZookeeperRecord> zookeeperInfoList() {
- return null;
- }
-
- /**
- * get master servers
- *
- * @return master server information
- */
- public static List<Server> getMasterServers() {
- return registryClient.getServerList(NodeType.MASTER);
- }
-
- /**
- * master construct is the same with worker, use the master instead
- *
- * @return worker server informations
- */
- public static List<Server> getWorkerServers() {
- return registryClient.getServerList(NodeType.WORKER);
- }
-
- public static Map<String, String> getServerMaps(NodeType nodeType, boolean hostOnly) {
- return registryClient.getServerMaps(nodeType, hostOnly);
- }
-
- public static List<String> getServerNodeList(NodeType nodeType, boolean hostOnly) {
- return registryClient.getServerNodeList(nodeType, hostOnly);
- }
-
- public static boolean isNodeExisted(String key) {
- return registryClient.isExisted(key);
- }
-
- public static List<String> getChildrenNodes(final String key) {
- return registryClient.getChildrenKeys(key);
- }
-
- public static String getNodeData(String key) {
- return registryClient.get(key);
- }
-}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java
index 41a1cc3..dcc9da3 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/AbstractControllerTest.java
@@ -20,11 +20,9 @@ package org.apache.dolphinscheduler.api.controller;
import org.apache.dolphinscheduler.api.ApiApplicationServer;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.SessionService;
-import org.apache.dolphinscheduler.api.utils.RegistryCenterUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.dao.entity.User;
-import org.apache.dolphinscheduler.service.registry.RegistryClient;
import org.apache.commons.lang.StringUtils;
@@ -36,11 +34,6 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.runner.RunWith;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-import org.powermock.modules.junit4.PowerMockRunnerDelegate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
@@ -51,11 +44,8 @@ import org.springframework.web.context.WebApplicationContext;
/**
* abstract controller test
*/
-@RunWith(PowerMockRunner.class)
-@PowerMockRunnerDelegate(SpringRunner.class)
+@RunWith(SpringRunner.class)
@SpringBootTest(classes = ApiApplicationServer.class)
-@PrepareForTest({ RegistryCenterUtils.class, RegistryClient.class })
-@PowerMockIgnore({"javax.management.*"})
public class AbstractControllerTest {
public static final String SESSION_ID = "sessionId";
@@ -74,9 +64,6 @@ public class AbstractControllerTest {
@Before
public void setUp() {
- PowerMockito.suppress(PowerMockito.constructor(RegistryClient.class));
- PowerMockito.mockStatic(RegistryCenterUtils.class);
-
mockMvc = MockMvcBuilders.webAppContextSetup(webApplicationContext).build();
createSession();
@@ -98,7 +85,7 @@ public class AbstractControllerTest {
String session = sessionService.createSession(loginUser, "127.0.0.1");
sessionId = session;
- Assert.assertTrue(!StringUtils.isEmpty(session));
+ Assert.assertFalse(StringUtils.isEmpty(session));
}
public Map<String, Object> success() {
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java
index 62022d8..a0a1772 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java
@@ -48,9 +48,6 @@ import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
-/**
- * scheduler controller test
- */
public class SchedulerControllerTest extends AbstractControllerTest {
private static Logger logger = LoggerFactory.getLogger(SchedulerControllerTest.class);
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java
index 486707c..a61e7d3 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TaskInstanceControllerTest.java
@@ -50,9 +50,6 @@ import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
-/**
- * task instance controller test
- */
public class TaskInstanceControllerTest extends AbstractControllerTest {
@InjectMocks
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java
index 14cd52f..7f675a5 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java
@@ -37,9 +37,6 @@ import org.springframework.test.web.servlet.MvcResult;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
-/**
- * tenant controller test
- */
public class TenantControllerTest extends AbstractControllerTest {
private static Logger logger = LoggerFactory.getLogger(TenantControllerTest.class);
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java
index f6e79bf..873236f 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/WorkerGroupControllerTest.java
@@ -22,7 +22,6 @@ import static org.springframework.test.web.servlet.request.MockMvcRequestBuilder
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
-import org.apache.dolphinscheduler.api.utils.RegistryCenterUtils;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.NodeType;
@@ -30,6 +29,7 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
+import org.apache.dolphinscheduler.service.registry.RegistryClient;
import java.util.HashMap;
import java.util.Map;
@@ -59,12 +59,15 @@ public class WorkerGroupControllerTest extends AbstractControllerTest {
@MockBean
private ProcessInstanceMapper processInstanceMapper;
+ @MockBean
+ private RegistryClient registryClient;
+
@Test
public void testSaveWorkerGroup() throws Exception {
Map<String, String> serverMaps = new HashMap<>();
serverMaps.put("192.168.0.1", "192.168.0.1");
serverMaps.put("192.168.0.2", "192.168.0.2");
- PowerMockito.when(RegistryCenterUtils.getServerMaps(NodeType.WORKER, true)).thenReturn(serverMaps);
+ PowerMockito.when(registryClient.getServerMaps(NodeType.WORKER, true)).thenReturn(serverMaps);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("name","cxc_work_group");
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java
index eea323e..cbbeec6 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java
@@ -17,25 +17,19 @@
package org.apache.dolphinscheduler.api.service;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.impl.AlertGroupServiceImpl;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.AlertGroup;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper;
import org.apache.dolphinscheduler.dao.vo.AlertGroupVo;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -47,8 +41,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.dao.DuplicateKeyException;
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
/**
* alert group service test
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/EnvironmentServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/EnvironmentServiceTest.java
index b9b95ec..81ba834 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/EnvironmentServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/EnvironmentServiceTest.java
@@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.Environment;
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
import org.apache.dolphinscheduler.dao.entity.User;
@@ -31,6 +30,8 @@ import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper;
import org.apache.dolphinscheduler.dao.mapper.EnvironmentWorkerGroupRelationMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
+import org.apache.commons.collections.CollectionUtils;
+
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java
index 2401f2b..50e0849 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java
@@ -22,10 +22,11 @@ import org.apache.dolphinscheduler.api.service.impl.MonitorServiceImpl;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.model.Server;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.MonitorDBDao;
import org.apache.dolphinscheduler.dao.entity.MonitorRecord;
+import org.apache.commons.collections.CollectionUtils;
+
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java
index 61c8a1d..6f516b7 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java
@@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
@@ -33,6 +32,8 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
+import org.apache.commons.collections.CollectionUtils;
+
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java
index 7d69d65..02e3ea1 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java
@@ -23,12 +23,13 @@ import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.Queue;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.QueueMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
+import org.apache.commons.collections.CollectionUtils;
+
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
index cca867c..141f9de 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
@@ -17,14 +17,16 @@
package org.apache.dolphinscheduler.api.service;
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import com.google.common.io.Files;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.impl.ResourcesServiceImpl;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UserType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
@@ -32,19 +34,8 @@ import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
-import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
-import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
-import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
-import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
-import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
-import org.apache.dolphinscheduler.dao.mapper.UserMapper;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
+import org.apache.dolphinscheduler.dao.mapper.*;
+import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -60,15 +51,20 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.mock.web.MockMultipartFile;
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
/**
* resources service test
*/
@RunWith(PowerMockRunner.class)
@PowerMockIgnore({"sun.security.*", "javax.net.*"})
-@PrepareForTest({HadoopUtils.class, PropertyUtils.class, FileUtils.class, org.apache.dolphinscheduler.api.utils.FileUtils.class})
+@PrepareForTest({HadoopUtils.class, PropertyUtils.class,
+ FileUtils.class, org.apache.dolphinscheduler.api.utils.FileUtils.class,
+ Files.class})
public class ResourcesServiceTest {
private static final Logger logger = LoggerFactory.getLogger(ResourcesServiceTest.class);
@@ -99,9 +95,9 @@ public class ResourcesServiceTest {
@Before
public void setUp() {
-
PowerMockito.mockStatic(HadoopUtils.class);
PowerMockito.mockStatic(FileUtils.class);
+ PowerMockito.mockStatic(Files.class);
PowerMockito.mockStatic(org.apache.dolphinscheduler.api.utils.FileUtils.class);
try {
// new HadoopUtils
@@ -132,15 +128,15 @@ public class ResourcesServiceTest {
//RESOURCE_SUFFIX_FORBID_CHANGE
mockMultipartFile = new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes());
- PowerMockito.when(FileUtils.suffix("test.pdf")).thenReturn("pdf");
- PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar");
+ PowerMockito.when(Files.getFileExtension("test.pdf")).thenReturn("pdf");
+ PowerMockito.when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
result = resourcesService.createResource(user, "ResourcesServiceTest.jar", "ResourcesServiceTest", ResourceType.FILE, mockMultipartFile, -1, "/");
logger.info(result.toString());
Assert.assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(), result.getMsg());
//UDF_RESOURCE_SUFFIX_NOT_JAR
mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf", "ResourcesServiceTest.pdf", "pdf", "test".getBytes());
- PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.pdf")).thenReturn("pdf");
+ PowerMockito.when(Files.getFileExtension("ResourcesServiceTest.pdf")).thenReturn("pdf");
result = resourcesService.createResource(user, "ResourcesServiceTest.pdf", "ResourcesServiceTest", ResourceType.UDF, mockMultipartFile, -1, "/");
logger.info(result.toString());
Assert.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(), result.getMsg());
@@ -268,10 +264,10 @@ public class ResourcesServiceTest {
resourcePage.setRecords(getResourceList());
Mockito.when(resourcesMapper.queryResourcePaging(Mockito.any(Page.class),
- Mockito.eq(0), Mockito.eq(-1), Mockito.eq(0), Mockito.eq("test"), Mockito.any())).thenReturn(resourcePage);
+ Mockito.eq(0), Mockito.eq(-1), Mockito.eq(0), Mockito.eq("test"), Mockito.any())).thenReturn(resourcePage);
Result result = resourcesService.queryResourceListPaging(loginUser, -1, ResourceType.FILE, "test", 1, 10);
logger.info(result.toString());
- Assert.assertEquals(Status.SUCCESS.getCode(), (int)result.getCode());
+ Assert.assertEquals(Status.SUCCESS.getCode(), (int) result.getCode());
PageInfo pageInfo = (PageInfo) result.getData();
Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getTotalList()));
@@ -407,7 +403,7 @@ public class ResourcesServiceTest {
//USER_NOT_EXIST
PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar");
- PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar");
+ PowerMockito.when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
result = resourcesService.readResource(1, 1, 10);
logger.info(result.toString());
Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
@@ -508,7 +504,7 @@ public class ResourcesServiceTest {
//USER_NOT_EXIST
PowerMockito.when(FileUtils.getResourceViewSuffixs()).thenReturn("jar");
- PowerMockito.when(FileUtils.suffix("ResourcesServiceTest.jar")).thenReturn("jar");
+ PowerMockito.when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
result = resourcesService.updateResourceContent(1, "content");
logger.info(result.toString());
Assert.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
@@ -717,4 +713,4 @@ public class ResourcesServiceTest {
resources.add(resource);
return resources;
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
index 7f662dc..f51c7a8 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
@@ -17,13 +17,15 @@
package org.apache.dolphinscheduler.api.service;
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.impl.TenantServiceImpl;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Tenant;
@@ -32,11 +34,6 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -47,8 +44,9 @@ import org.mockito.junit.MockitoJUnitRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
/**
* tenant service test
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java
index 23ac7b0..632c7a7 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java
@@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.enums.UserType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
@@ -33,6 +32,8 @@ import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
+import org.apache.commons.collections.CollectionUtils;
+
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -244,4 +245,4 @@ public class UdfFuncServiceTest {
udfFunc.setType(UdfType.HIVE);
return udfFunc;
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
index 429e849..e586db8 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
@@ -26,9 +26,7 @@ import org.apache.dolphinscheduler.api.service.impl.UsersServiceImpl;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UserType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.EncryptionUtils;
import org.apache.dolphinscheduler.dao.entity.AlertGroup;
import org.apache.dolphinscheduler.dao.entity.Project;
@@ -45,6 +43,9 @@ import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
+import org.apache.dolphinscheduler.spi.enums.ResourceType;
+
+import org.apache.commons.collections.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
@@ -680,4 +681,4 @@ public class UsersServiceTest {
return alertGroups;
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java
index 0c76835..1dfcc64 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java
@@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.api.service;
+import org.apache.dolphinscheduler.api.ApiApplicationServer;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.service.impl.WorkerGroupServiceImpl;
import org.apache.dolphinscheduler.common.Constants;
@@ -33,105 +34,32 @@ import java.util.List;
import java.util.Map;
import org.junit.Assert;
-import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
import org.mockito.Mockito;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PowerMockIgnore;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
-/**
- * worker group service test
- */
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({ RegistryClient.class })
-@PowerMockIgnore({"javax.management.*"})
+@RunWith(SpringJUnit4ClassRunner.class)
+@SpringBootTest(classes = ApiApplicationServer.class)
public class WorkerGroupServiceTest {
+ @MockBean
+ private RegistryClient registryClient;
- @InjectMocks
+ @Autowired
private WorkerGroupServiceImpl workerGroupService;
- @Mock
+ @MockBean
private WorkerGroupMapper workerGroupMapper;
- @Mock
+ @MockBean
private ProcessInstanceMapper processInstanceMapper;
-
private String groupName = "groupName000001";
- /* @Before
- public void init() {
- ZookeeperConfig zookeeperConfig = new ZookeeperConfig();
- zookeeperConfig.setDsRoot("/dolphinscheduler_qzw");
- Mockito.when(zookeeperCachedOperator.getZookeeperConfig()).thenReturn(zookeeperConfig);
-
- String workerPath = zookeeperCachedOperator.getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_WORKERS;
-
- List<String> workerGroupStrList = new ArrayList<>();
- workerGroupStrList.add("default");
- workerGroupStrList.add("test");
- Mockito.when(zookeeperCachedOperator.getChildrenNodes(workerPath)).thenReturn(workerGroupStrList);
-
- List<String> defaultAddressList = new ArrayList<>();
- defaultAddressList.add("192.168.220.188:1234");
- defaultAddressList.add("192.168.220.189:1234");
-
- Mockito.when(zookeeperCachedOperator.getChildrenNodes(workerPath + "/default")).thenReturn(defaultAddressList);
-
- Mockito.when(zookeeperCachedOperator.get(workerPath + "/default" + "/" + defaultAddressList.get(0))).thenReturn("0.01,0.17,0.03,25.83,8.0,1.0,2020-07-21 11:17:59,2020-07-21 14:39:20,0,13238");
- }
-
-*//**
- * create or update a worker group
- *//*
- @Test
- public void testSaveWorkerGroup() {
- // worker server maps
- Map<String, String> serverMaps = new HashMap<>();
- serverMaps.put("127.0.0.1:1234", "0.3,0.07,4.4,7.42,16.0,0.3,2021-03-19 20:17:58,2021-03-19 20:25:29,0,79214");
- Mockito.when(zookeeperMonitor.getServerMaps(ZKNodeType.WORKER, true)).thenReturn(serverMaps);
-
- User user = new User();
- // general user add
- user.setUserType(UserType.GENERAL_USER);
- Map<String, Object> result = workerGroupService.saveWorkerGroup(user, 0, groupName, "127.0.0.1:1234");
- Assert.assertEquals(Status.USER_NO_OPERATION_PERM.getMsg(), result.get(Constants.MSG));
-
- // success
- user.setUserType(UserType.ADMIN_USER);
- result = workerGroupService.saveWorkerGroup(user, 0, groupName, "127.0.0.1:1234");
- Assert.assertEquals(Status.SUCCESS.getMsg(), result.get(Constants.MSG));
- // group name exist
- Mockito.when(workerGroupMapper.selectById(2)).thenReturn(getWorkerGroup(2));
- Mockito.when(workerGroupMapper.queryWorkerGroupByName(groupName)).thenReturn(getList());
- result = workerGroupService.saveWorkerGroup(user, 2, groupName, "127.0.0.1:1234");
- Assert.assertEquals(Status.NAME_EXIST, result.get(Constants.STATUS));
- }*/
-
- /**
- * query worker group paging
- */
- /* @Test
- public void testQueryAllGroupPaging() {
- User user = new User();
- // general user add
- user.setUserType(UserType.ADMIN_USER);
- Map<String, Object> result = workerGroupService.queryAllGroupPaging(user, 1, 10, null);
- PageInfo<WorkerGroup> pageInfo = (PageInfo) result.get(Constants.DATA_LIST);
- Assert.assertEquals(pageInfo.getLists().size(), 1);
- }*/
-
- @Before
- public void before() {
- PowerMockito.suppress(PowerMockito.constructor(RegistryClient.class));
- }
-
@Test
public void testQueryAllGroup() {
Map<String, Object> result = workerGroupService.queryAllGroup();
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java
index 228eaf9..2276e34 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FileUtilsTest.java
@@ -63,16 +63,17 @@ public class FileUtilsTest {
public void testCopyFile() throws IOException {
//Define dest file path
+ String src = rootPath + System.getProperty("file.separator") + "src.txt";
String destFilename = rootPath + System.getProperty("file.separator") + "data.txt";
logger.info("destFilename: "+destFilename);
//Define InputStream for MultipartFile
String data = "data text";
- InputStream targetStream = new ByteArrayInputStream(data.getBytes());
+ org.apache.commons.io.FileUtils.writeStringToFile(new File(src), data);
//Use Mockito to mock MultipartFile
- MultipartFile file = Mockito.mock(MultipartFile.class);
- Mockito.when(file.getInputStream()).thenReturn(targetStream);
+ MultipartFile file = Mockito.mock(MultipartFile.class, Mockito.RETURNS_DEEP_STUBS);
+ Mockito.when(file.getResource().getFile()).thenReturn(new File(src));
//Invoke copyFile
FileUtils.copyFile(file,destFilename);
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java
deleted file mode 100644
index 69d1f21..0000000
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/FourLetterWordMainTest.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.api.utils;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.InjectMocks;
-import org.mockito.Mock;
-import org.mockito.Mockito;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.InputStream;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.net.SocketTimeoutException;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.when;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({FourLetterWordMain.class, Socket.class})
-public class FourLetterWordMainTest {
-
- private static final Logger logger =
- LoggerFactory.getLogger(FourLetterWordMainTest.class);
- private static final String NEW_LINE = "\n";
-
- @InjectMocks
- private FourLetterWordMain fourLetterWord;
- @Mock
- private Socket socket;
- @Mock
- private InetSocketAddress socketAddress;
-
- private final String localHost = "127.0.0.1";
- private final int zkPort = 2181;
- private ByteArrayOutputStream byteArrayOutputStream;
- private InputStream inputStream;
-
- private String cmd;
- private String testResult;
- private String expectedStr;
-
- @Before
- public void setUp() {
- // mock socket class
- PowerMockito.mockStatic(Socket.class);
- try {
- PowerMockito.whenNew(Socket.class).withNoArguments()
- .thenReturn(socket);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- /**
- * None mock test method, just to check zookeeper status.
- * Comment @Before notation to run this test.
- * Zookeeper status will be as:
- * Zookeeper version: 3.4.11 ...
- * Received: 6739707
- * Sent: 6739773
- * Connections: 20
- * Outstanding: 0
- * Zxid: 0x9ba
- * Mode: standalone
- * Node count: 263
- */
- public void testCmd() {
- // "192.168.64.11"
- // final String zkHost = localHost;
- final String zkHost = "192.168.64.11";
- cmd = "srvr";
- try {
- // Change localhost to right zk host ip.
- final String result = FourLetterWordMain
- .send4LetterWord(zkHost, zkPort, cmd);
- logger.info(cmd + ": " + result + "<<<");
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- @Test
- public void testEmptyCmd() {
- cmd = "";
- expectedStr = "";
- testSend4LetterWord(cmd, expectedStr);
- }
-
- @Test
- public void testNullCmd() {
- cmd = null;
-
- try {
- testResult = FourLetterWordMain
- .send4LetterWord(localHost, zkPort, cmd);
- } catch (Exception e) {
- testResult = e.getMessage();
- }
-
- logger.info("testNullCmd result: " + testResult);
- assertEquals("cmd must not be null", testResult);
- }
-
- @Test
- public void testNullSocketOutput() {
- cmd = "test null socket output";
- expectedStr = null;
- testSend4LetterWord(cmd, expectedStr);
- }
-
- @Test
- public void testOneLineOutput() {
- cmd = "line 1";
-
- // line end without \n
- expectedStr = "line 1" + NEW_LINE;
- testSend4LetterWord(cmd, expectedStr);
-
- // line end with \n
- expectedStr = "line 1\n" + NEW_LINE;
- testSend4LetterWord(cmd, expectedStr);
- }
-
- @Test
- public void testMultiline() {
- cmd = "line 1 " + NEW_LINE +
- "line 2 " + NEW_LINE +
- "line 3 " + NEW_LINE;
-
- expectedStr = cmd + NEW_LINE;
- testSend4LetterWord(cmd, expectedStr);
-
- expectedStr = NEW_LINE + NEW_LINE + NEW_LINE;
- testSend4LetterWord(cmd, expectedStr);
- }
-
- @Test
- public void testSocketTimeOut() {
- cmd = "test socket time out";
-
- try {
- doThrow(new SocketTimeoutException())
- .when(socket)
- .connect(any(InetSocketAddress.class), Mockito.anyInt());
- testResult = FourLetterWordMain
- .send4LetterWord(localHost, zkPort, cmd);
- } catch (Exception e) {
- testResult = e.getMessage();
- }
-
- logger.info("testSocketTimeOut result: " + testResult);
- assertEquals(
- "Exception while executing four letter word: " + cmd,
- testResult
- );
- }
-
- /**
- * Test FourLetterWordMain.send4LetterWord() with input cmd and output
- * string.
- * @param cmd
- * @param expectedStr
- */
- public void testSend4LetterWord(String cmd, String expectedStr) {
- try {
- final byte[] strBytes = cmd.getBytes();
- byteArrayOutputStream = new ByteArrayOutputStream(strBytes.length);
- byteArrayOutputStream.write(strBytes, 0, strBytes.length);
-
- inputStream = new ByteArrayInputStream(expectedStr.getBytes());
-
- when(socket.getOutputStream())
- .thenReturn(byteArrayOutputStream);
- when(socket.getInputStream()).thenReturn(inputStream);
-
- final String result = FourLetterWordMain
- .send4LetterWord(localHost, zkPort, cmd);
- logger.info(
- "testSend4LetterWord: " +
- "cmd: " + cmd +
- ", expectedStr: " + expectedStr +
- ", result: " + result + "."
- );
- Assert.assertEquals(expectedStr, result);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
-}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java
index 067dfeb..2a339fd 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegexUtilsTest.java
@@ -26,17 +26,6 @@ import org.junit.Test;
public class RegexUtilsTest {
@Test
- public void testIsNumeric() {
- String num1 = "123467854678";
- boolean numeric = RegexUtils.isNumeric(num1);
- Assert.assertTrue(numeric);
-
- String num2 = "0.0.01";
- boolean numeric2 = RegexUtils.isNumeric(num2);
- Assert.assertFalse(numeric2);
- }
-
- @Test
public void testIsValidLinuxUserName() {
String name1 = "10000";
Assert.assertFalse(RegexUtils.isValidLinuxUserName(name1));
@@ -72,4 +61,4 @@ public class RegexUtilsTest {
Assert.assertNull(result4);
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegistryCenterUtilsTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegistryCenterUtilsTest.java
deleted file mode 100644
index b8b4945..0000000
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/utils/RegistryCenterUtilsTest.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.api.utils;
-
-import org.apache.dolphinscheduler.common.model.Server;
-
-import java.util.List;
-
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * zookeeper monitor utils test
- */
-@Ignore
-public class RegistryCenterUtilsTest {
-
- @Test
- public void testGetMasterList(){
- List<Server> masterServerList = RegistryCenterUtils.getMasterServers();
- List<Server> workerServerList = RegistryCenterUtils.getWorkerServers();
-
- Assert.assertTrue(masterServerList.size() >= 0);
- Assert.assertTrue(workerServerList.size() >= 0);
- }
-
-}
diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml
index 481e00c..ec114ed 100644
--- a/dolphinscheduler-common/pom.xml
+++ b/dolphinscheduler-common/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-common</artifactId>
<name>dolphinscheduler-common</name>
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
index 91ae62a..bebfa29 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@@ -18,9 +18,9 @@
package org.apache.dolphinscheduler.common;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.SystemUtils;
import java.util.regex.Pattern;
@@ -90,8 +90,6 @@ public final class Constants {
public static final String REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS = "/lock/failover/masters";
public static final String REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_WORKERS = "/lock/failover/workers";
public static final String REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS = "/lock/failover/startup-masters";
- public static final String REGISTRY_PLUGIN_BINDING = "registry.plugin.binding";
- public static final String REGISTRY_PLUGIN_DIR = "registry.plugin.dir";
public static final String REGISTRY_SERVERS = "registry.servers";
/**
@@ -720,7 +718,7 @@ public final class Constants {
* application regex
*/
public static final String APPLICATION_REGEX = "application_\\d+_\\d+";
- public static final String PID = OSUtils.isWindows() ? "handle" : "pid";
+ public static final String PID = SystemUtils.IS_OS_WINDOWS ? "handle" : "pid";
/**
* month_begin
*/
@@ -752,7 +750,7 @@ public final class Constants {
public static final char LEFT_BRACE_CHAR = '(';
public static final char RIGHT_BRACE_CHAR = ')';
public static final String ADD_STRING = "+";
- public static final String MULTIPLY_STRING = "*";
+ public static final String STAR = "*";
public static final String DIVISION_STRING = "/";
public static final String LEFT_BRACE_STRING = "(";
public static final char P = 'P';
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/NodeType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/NodeType.java
index acc3c02..cb247cf 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/NodeType.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/NodeType.java
@@ -16,15 +16,6 @@
*/
package org.apache.dolphinscheduler.common.enums;
-/**
- * zk node type
- */
public enum NodeType {
-
- /**
- * 0 master node;
- * 1 worker node;
- * 2 dead_server node;
- */
- MASTER, WORKER, DEAD_SERVER;
+ MASTER, WORKER, DEAD_SERVER
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java
index 397f32e..c9217d0 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/graph/DAG.java
@@ -16,14 +16,23 @@
*/
package org.apache.dolphinscheduler.common.graph;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.*;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.commons.collections.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* analysis of DAG
* Node: node
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/AbstractParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/AbstractParameters.java
index 26f3a65..5714c8a 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/AbstractParameters.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/AbstractParameters.java
@@ -19,9 +19,9 @@ package org.apache.dolphinscheduler.common.task;
import org.apache.dolphinscheduler.common.enums.Direct;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java
index c7a341d..bd3825d 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/task/sql/SqlParameters.java
@@ -21,9 +21,9 @@ import org.apache.dolphinscheduler.common.enums.DataType;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ArrayUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ArrayUtils.java
deleted file mode 100644
index 60fcbce..0000000
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ArrayUtils.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.common.utils;
-
-public class ArrayUtils {
-
- public static byte[] clone(byte[] array) {
- return array == null ? null : (byte[])((byte[])array.clone());
- }
-
- public static byte[] addAll(byte[] array1, byte[] array2) {
- if (array1 == null) {
- return clone(array2);
- } else if (array2 == null) {
- return clone(array1);
- } else {
- byte[] joinedArray = new byte[array1.length + array2.length];
- System.arraycopy(array1, 0, joinedArray, 0, array1.length);
- System.arraycopy(array2, 0, joinedArray, array1.length, array2.length);
- return joinedArray;
- }
- }
-}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java
index 9801beb..0de217f 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CollectionUtils.java
@@ -18,18 +18,14 @@
package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.beanutils.BeanMap;
-import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.function.Function;
-import java.util.stream.Collectors;
/**
* Provides utility methods and decorators for {@link Collection} instances.
@@ -50,175 +46,6 @@ public class CollectionUtils {
}
/**
- * The load factor used when none specified in constructor.
- */
- static final float DEFAULT_LOAD_FACTOR = 0.75f;
-
- /**
- * Returns a new {@link Collection} containing <i>a</i> minus a subset of
- * <i>b</i>. Only the elements of <i>b</i> that satisfy the predicate
- * condition, <i>p</i> are subtracted from <i>a</i>.
- *
- * <p>The cardinality of each element <i>e</i> in the returned {@link Collection}
- * that satisfies the predicate condition will be the cardinality of <i>e</i> in <i>a</i>
- * minus the cardinality of <i>e</i> in <i>b</i>, or zero, whichever is greater.</p>
- * <p>The cardinality of each element <i>e</i> in the returned {@link Collection} that does <b>not</b>
- * satisfy the predicate condition will be equal to the cardinality of <i>e</i> in <i>a</i>.</p>
- *
- * @param a the collection to subtract from, must not be null
- * @param b the collection to subtract, must not be null
- * @param <T> T
- * @return a new collection with the results
- * @see Collection#removeAll
- */
- public static <T> Collection<T> subtract(Set<T> a, Set<T> b) {
- return org.apache.commons.collections4.CollectionUtils.subtract(a, b);
- }
-
- public static boolean isNotEmpty(Collection coll) {
- return !isEmpty(coll);
- }
-
- public static boolean isEmpty(Collection coll) {
- return coll == null || coll.isEmpty();
- }
-
- /**
- * String to map
- *
- * @param str string
- * @param separator separator
- * @return string to map
- */
- public static Map<String, String> stringToMap(String str, String separator) {
- return stringToMap(str, separator, "");
- }
-
- /**
- * String to map
- *
- * @param str string
- * @param separator separator
- * @param keyPrefix prefix
- * @return string to map
- */
- public static Map<String, String> stringToMap(String str, String separator, String keyPrefix) {
-
- Map<String, String> emptyMap = new HashMap<>(0);
- if (StringUtils.isEmpty(str)) {
- return emptyMap;
- }
- if (StringUtils.isEmpty(separator)) {
- return emptyMap;
- }
- String[] strings = str.split(separator);
- int initialCapacity = (int)(strings.length / DEFAULT_LOAD_FACTOR) + 1;
- Map<String, String> map = new HashMap<>(initialCapacity);
- for (int i = 0; i < strings.length; i++) {
- String[] strArray = strings[i].split("=");
- if (strArray.length != 2) {
- return emptyMap;
- }
- //strArray[0] KEY strArray[1] VALUE
- if (StringUtils.isEmpty(keyPrefix)) {
- map.put(strArray[0], strArray[1]);
- } else {
- map.put(keyPrefix + strArray[0], strArray[1]);
- }
- }
- return map;
- }
-
- /**
- * Transform item in collection
- *
- * @param collection origin collection
- * @param transformFunc transform function
- * @param <R> origin item type
- * @param <T> target type
- * @return transform list
- */
- public static <R, T> List<T> transformToList(Collection<R> collection, Function<R, T> transformFunc) {
- if (isEmpty(collection)) {
- return new ArrayList<>();
- }
- return collection.stream().map(transformFunc).collect(Collectors.toList());
- }
-
- /**
- * Collect collection to map
- *
- * @param collection origin collection
- * @param keyTransformFunction key transform function
- * @param <K> target k type
- * @param <V> value
- * @return map
- */
- public static <K, V> Map<K, V> collectionToMap(Collection<V> collection, Function<V, K> keyTransformFunction) {
- if (isEmpty(collection)) {
- return new HashMap<>();
- }
- return collection.stream().collect(Collectors.toMap(keyTransformFunction, Function.identity()));
- }
-
- /**
- * Helper class to easily access cardinality properties of two collections.
- *
- * @param <O> the element type
- */
- private static class CardinalityHelper<O> {
-
- /**
- * Contains the cardinality for each object in collection A.
- */
- final Map<O, Integer> cardinalityA;
-
- /**
- * Contains the cardinality for each object in collection B.
- */
- final Map<O, Integer> cardinalityB;
-
- /**
- * Create a new CardinalityHelper for two collections.
- *
- * @param a the first collection
- * @param b the second collection
- */
- public CardinalityHelper(final Iterable<? extends O> a, final Iterable<? extends O> b) {
- cardinalityA = CollectionUtils.getCardinalityMap(a);
- cardinalityB = CollectionUtils.getCardinalityMap(b);
- }
-
- /**
- * Returns the frequency of this object in collection A.
- *
- * @param obj the object
- * @return the frequency of the object in collection A
- */
- public int freqA(final Object obj) {
- return getFreq(obj, cardinalityA);
- }
-
- /**
- * Returns the frequency of this object in collection B.
- *
- * @param obj the object
- * @return the frequency of the object in collection B
- */
- public int freqB(final Object obj) {
- return getFreq(obj, cardinalityB);
- }
-
- private int getFreq(final Object obj, final Map<?, Integer> freqMap) {
- final Integer count = freqMap.get(obj);
- if (count != null) {
- return count;
- }
- return 0;
- }
- }
-
- /**
* returns {@code true} iff the given {@link Collection}s contain
* exactly the same elements with exactly the same cardinalities.
*
@@ -236,55 +63,7 @@ public class CollectionUtils {
return false;
}
- return isEqualCollection(a, b);
- }
-
- /**
- * Returns {@code true} iff the given {@link Collection}s contain
- * exactly the same elements with exactly the same cardinalities.
- * <p>
- * That is, iff the cardinality of <i>e</i> in <i>a</i> is
- * equal to the cardinality of <i>e</i> in <i>b</i>,
- * for each element <i>e</i> in <i>a</i> or <i>b</i>.
- *
- * @param a the first collection, must not be null
- * @param b the second collection, must not be null
- * @return <code>true</code> iff the collections contain the same elements with the same cardinalities.
- */
- public static boolean isEqualCollection(final Collection<?> a, final Collection<?> b) {
- if (a.size() != b.size()) {
- return false;
- }
- final CardinalityHelper<Object> helper = new CardinalityHelper<>(a, b);
- if (helper.cardinalityA.size() != helper.cardinalityB.size()) {
- return false;
- }
- for (final Object obj : helper.cardinalityA.keySet()) {
- if (helper.freqA(obj) != helper.freqB(obj)) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * Returns a {@link Map} mapping each unique element in the given
- * {@link Collection} to an {@link Integer} representing the number
- * of occurrences of that element in the {@link Collection}.
- * <p>
- * Only those elements present in the collection will appear as
- * keys in the map.
- *
- * @param <O> the type of object in the returned {@link Map}. This is a super type of O
- * @param coll the collection to get the cardinality map for, must not be null
- * @return the populated cardinality map
- */
- public static <O> Map<O, Integer> getCardinalityMap(final Iterable<? extends O> coll) {
- final Map<O, Integer> count = new HashMap<>();
- for (final O obj : coll) {
- count.put(obj, count.getOrDefault(obj, 0) + 1);
- }
- return count;
+ return org.apache.commons.collections.CollectionUtils.isEqualCollection(a, b);
}
/**
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java
deleted file mode 100644
index 10963b4..0000000
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.common.utils;
-
-public class EnumUtils {
-
- private EnumUtils() {
- throw new UnsupportedOperationException("Construct EnumUtils");
- }
-
- public static <E extends Enum<E>> E getEnum(final Class<E> enumClass, final String enumName) {
- if (enumName == null) {
- return null;
- }
- try {
- return Enum.valueOf(enumClass, enumName);
- } catch (final IllegalArgumentException ex) {
- return null;
- }
- }
-
- public static <E extends Enum<E>> boolean isValidEnum(final Class<E> enumClass, final String enumName) {
- if (enumName == null) {
- return false;
- }
- try {
- Enum.valueOf(enumClass, enumName);
- return true;
- } catch (final IllegalArgumentException ex) {
- return false;
- }
- }
-}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
index 9fa9b5f..284f0a9 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
@@ -17,28 +17,14 @@
package org.apache.dolphinscheduler.common.utils;
-import static org.apache.dolphinscheduler.common.Constants.DATA_BASEDIR_PATH;
-import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS;
-import static org.apache.dolphinscheduler.common.Constants.RESOURCE_VIEW_SUFFIXS_DEFAULT_VALUE;
-import static org.apache.dolphinscheduler.common.Constants.YYYYMMDDHHMMSS;
-
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.StringReader;
+import java.io.*;
import java.nio.charset.StandardCharsets;
-import java.util.Optional;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import static org.apache.dolphinscheduler.common.Constants.*;
/**
* file utils
@@ -49,31 +35,11 @@ public class FileUtils {
public static final String DATA_BASEDIR = PropertyUtils.getString(DATA_BASEDIR_PATH, "/tmp/dolphinscheduler");
- public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>();
-
private FileUtils() {
throw new UnsupportedOperationException("Construct FileUtils");
}
/**
- * get file suffix
- *
- * @param filename file name
- * @return file suffix
- */
- public static String suffix(String filename) {
-
- String fileSuffix = "";
- if (!StringUtils.isEmpty(filename)) {
- int lastIndex = filename.lastIndexOf('.');
- if (lastIndex > 0) {
- fileSuffix = filename.substring(lastIndex + 1);
- }
- }
- return fileSuffix;
- }
-
- /**
* get download file absolute path and name
*
* @param filename file name
@@ -152,8 +118,7 @@ public class FileUtils {
//create work dir
org.apache.commons.io.FileUtils.forceMkdir(execLocalPathFile);
String mkdirLog = "create dir success " + execLocalPath;
- LoggerUtils.logInfo(Optional.ofNullable(logger), mkdirLog);
- LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), mkdirLog);
+ logger.info(mkdirLog);
}
/**
@@ -164,30 +129,16 @@ public class FileUtils {
* @return true if write success
*/
public static boolean writeContent2File(String content, String filePath) {
- BufferedReader bufferedReader = null;
- BufferedWriter bufferedWriter = null;
try {
File distFile = new File(filePath);
if (!distFile.getParentFile().exists() && !distFile.getParentFile().mkdirs()) {
- FileUtils.logger.error("mkdir parent failed");
+ logger.error("mkdir parent failed");
return false;
}
- bufferedReader = new BufferedReader(new StringReader(content));
- bufferedWriter = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(distFile), StandardCharsets.UTF_8));
- char[] buf = new char[1024];
- int len;
- while ((len = bufferedReader.read(buf)) != -1) {
- bufferedWriter.write(buf, 0, len);
- }
- bufferedWriter.flush();
- bufferedReader.close();
- bufferedWriter.close();
+ IOUtils.write(content, new FileOutputStream(filePath), StandardCharsets.UTF_8);
} catch (IOException e) {
- FileUtils.logger.error(e.getMessage(), e);
+ logger.error(e.getMessage(), e);
return false;
- } finally {
- IOUtils.closeQuietly(bufferedWriter);
- IOUtils.closeQuietly(bufferedReader);
}
return true;
}
@@ -203,13 +154,9 @@ public class FileUtils {
* </ul>
*
* @param filename file name
- * @throws IOException in case deletion is unsuccessful
*/
- public static void deleteFile(String filename) throws IOException {
- File file = new File(filename);
- if (file.exists()) {
- org.apache.commons.io.FileUtils.forceDelete(file);
- }
+ public static void deleteFile(String filename) {
+ org.apache.commons.io.FileUtils.deleteQuietly(new File(filename));
}
/**
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
index 545bbb2..1b25298 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/LoggerUtils.java
@@ -18,6 +18,8 @@
package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.FileInputStream;
@@ -25,13 +27,9 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
-import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
* logger utils
*/
@@ -124,25 +122,4 @@ public class LoggerUtils {
}
return "";
}
-
- public static void logError(Optional<Logger> optionalLogger
- , String error) {
- optionalLogger.ifPresent((Logger logger) -> logger.error(error));
- }
-
- public static void logError(Optional<Logger> optionalLogger
- , Throwable e) {
- optionalLogger.ifPresent((Logger logger) -> logger.error(e.getMessage(), e));
- }
-
- public static void logError(Optional<Logger> optionalLogger
- , String error, Throwable e) {
- optionalLogger.ifPresent((Logger logger) -> logger.error(error, e));
- }
-
- public static void logInfo(Optional<Logger> optionalLogger
- , String info) {
- optionalLogger.ifPresent((Logger logger) -> logger.info(info));
- }
-
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
index 07053fa..f21d3d2 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java
@@ -17,9 +17,15 @@
package org.apache.dolphinscheduler.common.utils;
-import org.apache.dolphinscheduler.common.shell.ShellExecutor;
-
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.SystemUtils;
+import org.apache.dolphinscheduler.common.shell.ShellExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import oshi.SystemInfo;
+import oshi.hardware.CentralProcessor;
+import oshi.hardware.GlobalMemory;
+import oshi.hardware.HardwareAbstractionLayer;
import java.io.BufferedReader;
import java.io.FileInputStream;
@@ -30,22 +36,9 @@ import java.lang.management.OperatingSystemMXBean;
import java.lang.management.RuntimeMXBean;
import java.math.RoundingMode;
import java.text.DecimalFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
-import java.util.StringTokenizer;
+import java.util.*;
import java.util.regex.Pattern;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import oshi.SystemInfo;
-import oshi.hardware.CentralProcessor;
-import oshi.hardware.GlobalMemory;
-import oshi.hardware.HardwareAbstractionLayer;
-
/**
* os utils
*/
@@ -53,8 +46,6 @@ public class OSUtils {
private static final Logger logger = LoggerFactory.getLogger(OSUtils.class);
- public static final ThreadLocal<Logger> taskLoggerThreadLocal = new ThreadLocal<>();
-
private static final SystemInfo SI = new SystemInfo();
public static final String TWO_DECIMAL = "0.00";
@@ -64,7 +55,7 @@ public class OSUtils {
*/
public static final double NEGATIVE_ONE = -1;
- private static HardwareAbstractionLayer hal = SI.getHardware();
+ private static final HardwareAbstractionLayer hal = SI.getHardware();
private OSUtils() {
throw new UnsupportedOperationException("Construct OSUtils");
@@ -108,22 +99,6 @@ public class OSUtils {
}
/**
- * get total physical memory size
- * <p>
- * Keep 2 decimal
- *
- * @return available Physical Memory Size, unit: G
- */
- public static double totalPhysicalMemorySize() {
- GlobalMemory memory = hal.getMemory();
- double totalPhysicalMemorySize = memory.getTotal() / 1024.0 / 1024 / 1024;
-
- DecimalFormat df = new DecimalFormat(TWO_DECIMAL);
- df.setRoundingMode(RoundingMode.HALF_UP);
- return Double.parseDouble(df.format(totalPhysicalMemorySize));
- }
-
- /**
* load average
*
* @return load average
@@ -164,9 +139,9 @@ public class OSUtils {
public static List<String> getUserList() {
try {
- if (isMacOS()) {
+ if (SystemUtils.IS_OS_MAC) {
return getUserListFromMac();
- } else if (isWindows()) {
+ } else if (SystemUtils.IS_OS_WINDOWS) {
return getUserListFromWindows();
} else {
return getUserListFromLinux();
@@ -262,14 +237,10 @@ public class OSUtils {
*/
public static void createUserIfAbsent(String userName) {
// if not exists this user, then create
- taskLoggerThreadLocal.set(taskLoggerThreadLocal.get());
if (!getUserList().contains(userName)) {
boolean isSuccess = createUser(userName);
- String infoLog = String.format("create user %s %s", userName, isSuccess ? "success" : "fail");
- LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog);
- LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog);
+ logger.info("create user {} {}", userName, isSuccess ? "success" : "fail");
}
- taskLoggerThreadLocal.remove();
}
/**
@@ -283,21 +254,19 @@ public class OSUtils {
String userGroup = getGroup();
if (StringUtils.isEmpty(userGroup)) {
String errorLog = String.format("%s group does not exist for this operating system.", userGroup);
- LoggerUtils.logError(Optional.ofNullable(logger), errorLog);
- LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), errorLog);
+ logger.error(errorLog);
return false;
}
- if (isMacOS()) {
+ if (SystemUtils.IS_OS_MAC) {
createMacUser(userName, userGroup);
- } else if (isWindows()) {
+ } else if (SystemUtils.IS_OS_WINDOWS) {
createWindowsUser(userName, userGroup);
} else {
createLinuxUser(userName, userGroup);
}
return true;
} catch (Exception e) {
- LoggerUtils.logError(Optional.ofNullable(logger), e);
- LoggerUtils.logError(Optional.ofNullable(taskLoggerThreadLocal.get()), e);
+ logger.error(e.getMessage(), e);
}
return false;
@@ -311,14 +280,9 @@ public class OSUtils {
* @throws IOException in case of an I/O error
*/
private static void createLinuxUser(String userName, String userGroup) throws IOException {
- String infoLog1 = String.format("create linux os user : %s", userName);
- LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1);
- LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1);
-
+ logger.info("create linux os user: {}", userName);
String cmd = String.format("sudo useradd -g %s %s", userGroup, userName);
- String infoLog2 = String.format("execute cmd : %s", cmd);
- LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2);
- LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2);
+ logger.info("execute cmd: {}", cmd);
exeCmd(cmd);
}
@@ -330,23 +294,14 @@ public class OSUtils {
* @throws IOException in case of an I/O error
*/
private static void createMacUser(String userName, String userGroup) throws IOException {
- Optional<Logger> optionalLogger = Optional.ofNullable(logger);
- Optional<Logger> optionalTaskLogger = Optional.ofNullable(taskLoggerThreadLocal.get());
-
- String infoLog1 = String.format("create mac os user : %s", userName);
- LoggerUtils.logInfo(optionalLogger, infoLog1);
- LoggerUtils.logInfo(optionalTaskLogger, infoLog1);
+ logger.info("create mac os user: {}", userName);
String createUserCmd = String.format("sudo sysadminctl -addUser %s -password %s", userName, userName);
- String infoLog2 = String.format("create user command : %s", createUserCmd);
- LoggerUtils.logInfo(optionalLogger, infoLog2);
- LoggerUtils.logInfo(optionalTaskLogger, infoLog2);
+ logger.info("create user command: {}", createUserCmd);
exeCmd(createUserCmd);
String appendGroupCmd = String.format("sudo dseditgroup -o edit -a %s -t user %s", userName, userGroup);
- String infoLog3 = String.format("append user to group : %s", appendGroupCmd);
- LoggerUtils.logInfo(optionalLogger, infoLog3);
- LoggerUtils.logInfo(optionalTaskLogger, infoLog3);
+ logger.info("append user to group: {}", appendGroupCmd);
exeCmd(appendGroupCmd);
}
@@ -358,20 +313,14 @@ public class OSUtils {
* @throws IOException in case of an I/O error
*/
private static void createWindowsUser(String userName, String userGroup) throws IOException {
- String infoLog1 = String.format("create windows os user : %s", userName);
- LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog1);
- LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog1);
+ logger.info("create windows os user: {}", userName);
String userCreateCmd = String.format("net user \"%s\" /add", userName);
- String infoLog2 = String.format("execute create user command : %s", userCreateCmd);
- LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog2);
- LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog2);
+ logger.info("execute create user command: {}", userCreateCmd);
exeCmd(userCreateCmd);
String appendGroupCmd = String.format("net localgroup \"%s\" \"%s\" /add", userGroup, userName);
- String infoLog3 = String.format("execute append user to group : %s", appendGroupCmd);
- LoggerUtils.logInfo(Optional.ofNullable(logger), infoLog3);
- LoggerUtils.logInfo(Optional.ofNullable(taskLoggerThreadLocal.get()), infoLog3);
+ logger.info("execute append user to group: {}", appendGroupCmd);
exeCmd(appendGroupCmd);
}
@@ -382,7 +331,7 @@ public class OSUtils {
* @throws IOException errors
*/
public static String getGroup() throws IOException {
- if (isWindows()) {
+ if (SystemUtils.IS_OS_WINDOWS) {
String currentProcUserName = System.getProperty("user.name");
String result = exeCmd(String.format("net user \"%s\"", currentProcUserName));
String line = result.split("\n")[22];
@@ -455,33 +404,6 @@ public class OSUtils {
}
/**
- * whether is macOS
- *
- * @return true if mac
- */
- public static boolean isMacOS() {
- return getOSName().startsWith("Mac");
- }
-
- /**
- * whether is windows
- *
- * @return true if windows
- */
- public static boolean isWindows() {
- return getOSName().startsWith("Windows");
- }
-
- /**
- * get current OS name
- *
- * @return current OS name
- */
- public static String getOSName() {
- return System.getProperty("os.name");
- }
-
- /**
* check memory and cpu usage
*
* @param maxCpuloadAvg maxCpuloadAvg
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
index c7e6e14..f247aa0 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ParameterUtils.java
@@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CommandType;
-import org.apache.dolphinscheduler.common.enums.DataType;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.utils.placeholder.BusinessTimeUtils;
import org.apache.dolphinscheduler.common.utils.placeholder.PlaceholderUtils;
@@ -27,7 +26,6 @@ import org.apache.dolphinscheduler.common.utils.placeholder.TimePlaceholderUtils
import org.apache.commons.lang.StringUtils;
-import java.sql.PreparedStatement;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
@@ -37,16 +35,10 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
* parameter parse utils
*/
public class ParameterUtils {
-
- private static final Logger logger = LoggerFactory.getLogger(ParameterUtils.class);
-
private static final String DATE_PARSE_PATTERN = "\\$\\[([^\\$\\]]+)]";
private static final String DATE_START_PATTERN = "^[0-9]";
@@ -86,72 +78,6 @@ public class ParameterUtils {
}
/**
- * new
- * convert parameters place holders
- *
- * @param parameterString parameter
- * @param parameterMap parameter map
- * @return convert parameters place holders
- */
- public static String convertParameterPlaceholders2(String parameterString, Map<String, String> parameterMap) {
- if (StringUtils.isEmpty(parameterString)) {
- return parameterString;
- }
- //Get current time, schedule execute time
- String cronTimeStr = parameterMap.get(Constants.PARAMETER_SHECDULE_TIME);
- Date cronTime = null;
-
- if (!StringUtils.isEmpty(cronTimeStr)) {
- cronTime = DateUtils.parse(cronTimeStr, Constants.PARAMETER_FORMAT_TIME);
-
- } else {
- cronTime = new Date();
- }
-
- // replace variable ${} form,refers to the replacement of system variables and custom variables
- if (!parameterMap.isEmpty()) {
- parameterString = PlaceholderUtils.replacePlaceholders(parameterString, parameterMap, true);
- }
-
- // replace time $[...] form, eg. $[yyyyMMdd]
- if (cronTime != null) {
- return dateTemplateParse(parameterString, cronTime);
- }
- return parameterString;
- }
-
- /**
- * set in parameter
- *
- * @param index index
- * @param stmt preparedstatement
- * @param dataType data type
- * @param value value
- * @throws Exception errors
- */
- public static void setInParameter(int index, PreparedStatement stmt, DataType dataType, String value) throws Exception {
- if (dataType.equals(DataType.VARCHAR)) {
- stmt.setString(index, value);
- } else if (dataType.equals(DataType.INTEGER)) {
- stmt.setInt(index, Integer.parseInt(value));
- } else if (dataType.equals(DataType.LONG)) {
- stmt.setLong(index, Long.parseLong(value));
- } else if (dataType.equals(DataType.FLOAT)) {
- stmt.setFloat(index, Float.parseFloat(value));
- } else if (dataType.equals(DataType.DOUBLE)) {
- stmt.setDouble(index, Double.parseDouble(value));
- } else if (dataType.equals(DataType.DATE)) {
- stmt.setDate(index, java.sql.Date.valueOf(value));
- } else if (dataType.equals(DataType.TIME)) {
- stmt.setString(index, value);
- } else if (dataType.equals(DataType.TIMESTAMP)) {
- stmt.setTimestamp(index, java.sql.Timestamp.valueOf(value));
- } else if (dataType.equals(DataType.BOOLEAN)) {
- stmt.setBoolean(index, Boolean.parseBoolean(value));
- }
- }
-
- /**
* curing user define parameters
*
* @param globalParamMap global param map
@@ -218,26 +144,6 @@ public class ParameterUtils {
}
/**
- * $[yyyyMMdd] replace schedule time
- */
- public static String replaceScheduleTime(String text, Date scheduleTime) {
- Map<String, Property> paramsMap = new HashMap<>();
- //if getScheduleTime null ,is current date
- if (null == scheduleTime) {
- scheduleTime = new Date();
- }
-
- String dateTime = org.apache.dolphinscheduler.common.utils.DateUtils.format(scheduleTime, Constants.PARAMETER_FORMAT_TIME);
- Property p = new Property();
- p.setValue(dateTime);
- p.setProp(Constants.PARAMETER_SHECDULE_TIME);
- paramsMap.put(Constants.PARAMETER_SHECDULE_TIME, p);
- text = ParameterUtils.convertParameterPlaceholders2(text, convert(paramsMap));
-
- return text;
- }
-
- /**
* format convert
*
* @param paramsMap params map
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java
index 23861c7..da04bc0 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/RetryerUtils.java
@@ -17,26 +17,14 @@
package org.apache.dolphinscheduler.common.utils;
+import com.github.rholder.retry.*;
import org.apache.dolphinscheduler.common.Constants;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.github.rholder.retry.RetryException;
-import com.github.rholder.retry.Retryer;
-import com.github.rholder.retry.RetryerBuilder;
-import com.github.rholder.retry.StopStrategies;
-import com.github.rholder.retry.WaitStrategies;
-
-/**
- * The Retryer util.
- */
public class RetryerUtils {
- private static final Logger logger = LoggerFactory.getLogger(RetryerUtils.class);
private static Retryer<Boolean> defaultRetryerResultCheck;
private static Retryer<Boolean> defaultRetryerResultNoCheck;
@@ -114,31 +102,4 @@ public class RetryerUtils {
public static Boolean retryCall(final Callable<Boolean> callable) throws ExecutionException, RetryException {
return retryCall(callable, true);
}
-
- /**
- * Retry call silent without exceptions thrown
- *
- * @param callable the callable
- * @param checkResult whether check result
- * @return if no exceptions ,it's result returned by callable ,else always false
- */
- public static boolean retryCallSilent(final Callable<Boolean> callable, boolean checkResult) {
- boolean result = false;
- try {
- result = getDefaultRetryer(checkResult).call(callable);
- } catch (ExecutionException | RetryException e) {
- logger.warn("Retry call {} failed {}", callable, e.getMessage(), e);
- }
- return result;
- }
-
- /**
- * Retry call silent without exceptions thrown
- *
- * @param callable the callable
- * @return if no exceptions ,it's result returned by callable ,else always false
- */
- public static boolean retryCallSilent(final Callable<Boolean> callable) {
- return retryCallSilent(callable, true);
- }
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java
index 880c440..503cf82 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java
@@ -23,8 +23,6 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -38,7 +36,7 @@ import org.slf4j.LoggerFactory;
public class SchemaUtils {
private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class);
- private static Pattern p = Pattern.compile("\\s*|\t|\r|\n");
+ private static final Pattern p = Pattern.compile("\\s*|\t|\r|\n");
private SchemaUtils() {
throw new UnsupportedOperationException("Construct SchemaUtils");
@@ -49,7 +47,6 @@ public class SchemaUtils {
*
* @return all schema list
*/
- @SuppressWarnings("unchecked")
public static List<String> getAllSchemaList() {
List<String> schemaDirList = new ArrayList<>();
File[] schemaDirArr = FileUtils.getAllDir("sql/upgrade");
@@ -61,28 +58,25 @@ public class SchemaUtils {
schemaDirList.add(file.getName());
}
- Collections.sort(schemaDirList, new Comparator() {
- @Override
- public int compare(Object o1, Object o2) {
- try {
- String dir1 = String.valueOf(o1);
- String dir2 = String.valueOf(o2);
- String version1 = dir1.split("_")[0];
- String version2 = dir2.split("_")[0];
- if (version1.equals(version2)) {
- return 0;
- }
-
- if (SchemaUtils.isAGreatVersion(version1, version2)) {
- return 1;
- }
-
- return -1;
-
- } catch (Exception e) {
- logger.error(e.getMessage(), e);
- throw new RuntimeException(e);
+ schemaDirList.sort((o1, o2) -> {
+ try {
+ String dir1 = String.valueOf(o1);
+ String dir2 = String.valueOf(o2);
+ String version1 = dir1.split("_")[0];
+ String version2 = dir2.split("_")[0];
+ if (version1.equals(version2)) {
+ return 0;
}
+
+ if (SchemaUtils.isAGreatVersion(version1, version2)) {
+ return 1;
+ }
+
+ return -1;
+
+ } catch (Exception e) {
+ logger.error(e.getMessage(), e);
+ throw new RuntimeException(e);
}
});
@@ -124,7 +118,7 @@ public class SchemaUtils {
public static String getSoftVersion() {
String softVersion;
try {
- softVersion = FileUtils.readFile2Str(new FileInputStream(new File("sql/soft_version")));
+ softVersion = FileUtils.readFile2Str(new FileInputStream("sql/soft_version"));
softVersion = replaceBlank(softVersion);
} catch (FileNotFoundException e) {
logger.error(e.getMessage(), e);
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java
deleted file mode 100644
index b95417e..0000000
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtils.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.common.utils;
-
-import org.apache.dolphinscheduler.common.Constants;
-
-import org.apache.commons.lang.StringUtils;
-
-/**
- * sensitive log Util
- */
-public class SensitiveLogUtils {
-
- private SensitiveLogUtils() {
- throw new UnsupportedOperationException("Construct SensitiveLogUtils");
- }
-
- /**
- * @param dataSourcePwd data source password
- * @return String
- */
- public static String maskDataSourcePwd(String dataSourcePwd) {
-
- if (!StringUtils.isEmpty(dataSourcePwd)) {
- dataSourcePwd = Constants.PASSWORD_DEFAULT;
- }
- return dataSourcePwd;
- }
-
-}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/PropertyPlaceholderHelper.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/PropertyPlaceholderHelper.java
index 4fb8555..099a6f6 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/PropertyPlaceholderHelper.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/PropertyPlaceholderHelper.java
@@ -16,12 +16,13 @@
*/
package org.apache.dolphinscheduler.common.utils.placeholder;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import java.util.*;
+import static java.util.Objects.requireNonNull;
+
/**
* Utility class for working with Strings that have placeholder values in them. A placeholder takes the form
* {@code ${name}}. Using {@code PropertyPlaceholderHelper} these placeholders can be substituted for
@@ -36,7 +37,7 @@ public class PropertyPlaceholderHelper {
private static final Log logger = LogFactory.getLog(PropertyPlaceholderHelper.class);
- private static final Map<String, String> wellKnownSimplePrefixes = new HashMap<String, String>(4);
+ private static final Map<String, String> wellKnownSimplePrefixes = new HashMap<>(4);
static {
wellKnownSimplePrefixes.put("}", "{");
@@ -58,16 +59,7 @@ public class PropertyPlaceholderHelper {
/**
* Creates a new {@code PropertyPlaceholderHelper} that uses the supplied prefix and suffix.
- * Unresolvable placeholders are ignored.
- * @param placeholderPrefix the prefix that denotes the start of a placeholder
- * @param placeholderSuffix the suffix that denotes the end of a placeholder
- */
- public PropertyPlaceholderHelper(String placeholderPrefix, String placeholderSuffix) {
- this(placeholderPrefix, placeholderSuffix, null, true);
- }
-
- /**
- * Creates a new {@code PropertyPlaceholderHelper} that uses the supplied prefix and suffix.
+ *
* @param placeholderPrefix the prefix that denotes the start of a placeholder
* @param placeholderSuffix the suffix that denotes the end of a placeholder
* @param valueSeparator the separating character between the placeholder variable
@@ -78,15 +70,14 @@ public class PropertyPlaceholderHelper {
public PropertyPlaceholderHelper(String placeholderPrefix, String placeholderSuffix,
String valueSeparator, boolean ignoreUnresolvablePlaceholders) {
- notNull(placeholderPrefix, "'placeholderPrefix' must not be null");
- notNull(placeholderSuffix, "'placeholderSuffix' must not be null");
+ requireNonNull((Object) placeholderPrefix, "'placeholderPrefix' must not be null");
+ requireNonNull((Object) placeholderSuffix, "'placeholderSuffix' must not be null");
this.placeholderPrefix = placeholderPrefix;
this.placeholderSuffix = placeholderSuffix;
String simplePrefixForSuffix = wellKnownSimplePrefixes.get(this.placeholderSuffix);
if (simplePrefixForSuffix != null && this.placeholderPrefix.endsWith(simplePrefixForSuffix)) {
this.simplePrefix = simplePrefixForSuffix;
- }
- else {
+ } else {
this.simplePrefix = this.placeholderPrefix;
}
this.valueSeparator = valueSeparator;
@@ -95,36 +86,20 @@ public class PropertyPlaceholderHelper {
/**
- * Replaces all placeholders of format {@code ${name}} with the corresponding
- * property from the supplied {@link Properties}.
- * @param value the value containing the placeholders to be replaced
- * @param properties the {@code Properties} to use for replacement
- * @return the supplied value with placeholders replaced inline
- */
- public String replacePlaceholders(String value, final Properties properties) {
- notNull(properties, "'properties' must not be null");
- return replacePlaceholders(value, new PlaceholderResolver() {
- @Override
- public String resolvePlaceholder(String placeholderName) {
- return properties.getProperty(placeholderName);
- }
- });
- }
-
- /**
* Replaces all placeholders of format {@code ${name}} with the value returned
* from the supplied {@link PlaceholderResolver}.
+ *
* @param value the value containing the placeholders to be replaced
* @param placeholderResolver the {@code PlaceholderResolver} to use for replacement
* @return the supplied value with placeholders replaced inline
*/
public String replacePlaceholders(String value, PlaceholderResolver placeholderResolver) {
- notNull(value, "'value' must not be null");
+ requireNonNull((Object) value, "'value' must not be null");
return parseStringValue(value, placeholderResolver, new HashSet<String>());
}
protected String parseStringValue(
- String value, PlaceholderResolver placeholderResolver, Set<String> visitedPlaceholders) {
+ String value, PlaceholderResolver placeholderResolver, Set<String> visitedPlaceholders) {
StringBuilder result = new StringBuilder(value);
@@ -136,7 +111,7 @@ public class PropertyPlaceholderHelper {
String originalPlaceholder = placeholder;
if (!visitedPlaceholders.add(originalPlaceholder)) {
throw new IllegalArgumentException(
- "Circular placeholder reference '" + originalPlaceholder + "' in property definitions");
+ "Circular placeholder reference '" + originalPlaceholder + "' in property definitions");
}
// Recursive invocation, parsing placeholders contained in the placeholder key.
placeholder = parseStringValue(placeholder, placeholderResolver, visitedPlaceholders);
@@ -162,18 +137,15 @@ public class PropertyPlaceholderHelper {
logger.trace("Resolved placeholder '" + placeholder + "'");
}
startIndex = result.indexOf(this.placeholderPrefix, startIndex + propVal.length());
- }
- else if (this.ignoreUnresolvablePlaceholders) {
+ } else if (this.ignoreUnresolvablePlaceholders) {
// Proceed with unprocessed value.
startIndex = result.indexOf(this.placeholderPrefix, endIndex + this.placeholderSuffix.length());
- }
- else {
+ } else {
throw new IllegalArgumentException("Could not resolve placeholder '" +
- placeholder + "'" + " in value \"" + value + "\"");
+ placeholder + "'" + " in value \"" + value + "\"");
}
visitedPlaceholders.remove(originalPlaceholder);
- }
- else {
+ } else {
startIndex = -1;
}
}
@@ -189,16 +161,13 @@ public class PropertyPlaceholderHelper {
if (withinNestedPlaceholder > 0) {
withinNestedPlaceholder--;
index = index + this.placeholderSuffix.length();
- }
- else {
+ } else {
return index;
}
- }
- else if (substringMatch(buf, index, this.simplePrefix)) {
+ } else if (substringMatch(buf, index, this.simplePrefix)) {
withinNestedPlaceholder++;
index = index + this.simplePrefix.length();
- }
- else {
+ } else {
index++;
}
}
@@ -213,6 +182,7 @@ public class PropertyPlaceholderHelper {
/**
* Resolve the supplied placeholder name to the replacement value.
+ *
* @param placeholderName the name of the placeholder to resolve
* @return the replacement value, or {@code null} if no replacement is to be made
*/
@@ -222,6 +192,7 @@ public class PropertyPlaceholderHelper {
/**
* Test whether the given string matches the given substring
* at the given index.
+ *
* @param str the original string (or StringBuilder)
* @param index the index in the original string to start matching against
* @param substring the substring to match at the given index
@@ -236,20 +207,5 @@ public class PropertyPlaceholderHelper {
}
return true;
}
-
- /**
- * Assert that an object is not {@code null}.
- * <pre class="code">Assert.notNull(clazz, "The class must not be null");</pre>
- * @param object the object to check
- * @param message the exception message to use if the assertion fails
- * @throws IllegalArgumentException if the object is {@code null}
- */
- public static void notNull(Object object, String message) {
- if (object == null) {
- throw new IllegalArgumentException(message);
- }
- }
-
-
}
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java
index bfb5e28..1974460 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/placeholder/TimePlaceholderUtils.java
@@ -24,7 +24,7 @@ import static org.apache.dolphinscheduler.common.Constants.DIVISION_STRING;
import static org.apache.dolphinscheduler.common.Constants.LEFT_BRACE_CHAR;
import static org.apache.dolphinscheduler.common.Constants.LEFT_BRACE_STRING;
import static org.apache.dolphinscheduler.common.Constants.MULTIPLY_CHAR;
-import static org.apache.dolphinscheduler.common.Constants.MULTIPLY_STRING;
+import static org.apache.dolphinscheduler.common.Constants.STAR;
import static org.apache.dolphinscheduler.common.Constants.N;
import static org.apache.dolphinscheduler.common.Constants.P;
import static org.apache.dolphinscheduler.common.Constants.RIGHT_BRACE_CHAR;
@@ -266,9 +266,9 @@ public class TimePlaceholderUtils {
* @return true or false
*/
private static boolean compare(String peek, String cur) {
- if (MULTIPLY_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || MULTIPLY_STRING.equals(cur) || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) {
+ if (STAR.equals(peek) && (DIVISION_STRING.equals(cur) || STAR.equals(cur) || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) {
return true;
- } else if (DIVISION_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || MULTIPLY_STRING.equals(cur) || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) {
+ } else if (DIVISION_STRING.equals(peek) && (DIVISION_STRING.equals(cur) || STAR.equals(cur) || ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) {
return true;
} else if (ADD_STRING.equals(peek) && (ADD_STRING.equals(cur) || SUBTRACT_STRING.equals(cur))) {
return true;
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java
index 3280a96..3f2c301 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/ConstantsTest.java
@@ -16,7 +16,8 @@
*/
package org.apache.dolphinscheduler.common;
-import org.apache.dolphinscheduler.common.utils.OSUtils;
+import org.apache.commons.lang.SystemUtils;
+
import org.junit.Assert;
import org.junit.Test;
@@ -30,7 +31,7 @@ public class ConstantsTest {
*/
@Test
public void testPID() {
- if (OSUtils.isWindows()) {
+ if (SystemUtils.IS_OS_WINDOWS) {
Assert.assertEquals(Constants.PID, "handle");
} else {
Assert.assertEquals(Constants.PID, "pid");
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java
index d6f3ffd..64ef52e 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java
@@ -40,16 +40,6 @@ public class OSUtilsTest {
}
@Test
- public void physicalMemorySize() {
- double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
- double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize();
- logger.info("availablePhysicalMemorySize : {}", availablePhysicalMemorySize);
- logger.info("totalPhysicalMemorySize : {}", totalPhysicalMemorySize);
- Assert.assertTrue(availablePhysicalMemorySize >= 0.0);
- Assert.assertTrue(totalPhysicalMemorySize >= 0.0);
- }
-
- @Test
public void loadAverage() {
double loadAverage = OSUtils.loadAverage();
logger.info("loadAverage : {}", loadAverage);
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java
index cd7b4f2..f7ad9de 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/FlinkParametersTest.java
@@ -18,7 +18,9 @@ package org.apache.dolphinscheduler.common.task;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.flink.FlinkParameters;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
+
+import org.apache.commons.collections.CollectionUtils;
+
import org.junit.Assert;
import org.junit.Test;
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SparkParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SparkParametersTest.java
index dbd98ed..a3d4d9e 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SparkParametersTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SparkParametersTest.java
@@ -19,7 +19,8 @@ package org.apache.dolphinscheduler.common.task;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.spark.SparkParameters;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
+
+import org.apache.commons.collections.CollectionUtils;
import java.util.LinkedList;
import java.util.List;
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java
index 17e95cf..3f7a980 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/task/SqlParametersTest.java
@@ -23,7 +23,8 @@ import org.apache.dolphinscheduler.common.enums.DataType;
import org.apache.dolphinscheduler.common.enums.Direct;
import org.apache.dolphinscheduler.common.process.Property;
import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
+
+import org.apache.commons.collections.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CollectionUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CollectionUtilsTest.java
deleted file mode 100644
index ccf8ace..0000000
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CollectionUtilsTest.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.dolphinscheduler.common.utils;
-
-import org.apache.dolphinscheduler.common.Constants;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.*;
-
-
-public class CollectionUtilsTest {
-
- @Test
- public void equalLists() {
- Assert.assertTrue(CollectionUtils.equalLists(null,null));
- Assert.assertTrue(CollectionUtils.equalLists(new ArrayList<Integer>(),new ArrayList<Integer>()));
- List<Integer> a = new ArrayList<Integer>();
- a.add(1);
- a.add(2);
- List<Integer> b = new ArrayList<Integer>();
- b.add(1);
- b.add(2);
- Assert.assertTrue(CollectionUtils.equalLists(a, b));
- a.add(1);
- Assert.assertFalse(CollectionUtils.equalLists(a, b));
- b.add(2);
- Assert.assertFalse(CollectionUtils.equalLists(a, b));
- a.add(2);
- b.add(1);
- a.add(4);
- b.add(2);
- Assert.assertFalse(CollectionUtils.equalLists(a, b));
- Assert.assertFalse(CollectionUtils.equalLists(null, new ArrayList<Integer>()));
- Assert.assertFalse(CollectionUtils.equalLists(new ArrayList<Integer>(), null));
- }
-
- @Test
- public void subtract() {
- Set<Integer> a = new HashSet<Integer>();
- a.add(1);
- a.add(2);
- a.add(3);
- Set<Integer> b = new HashSet<Integer>();
- b.add(0);
- b.add(2);
- b.add(4);
- Assert.assertArrayEquals(new Integer[]{1,3},CollectionUtils.subtract(a,b).toArray());
- }
-
- @Test
- public void stringToMap() {
- Map<String, String> a = CollectionUtils.stringToMap("a=b;c=d;", ";");
- Assert.assertNotNull(a);
- Assert.assertTrue(a.size() == 2);
- a = CollectionUtils.stringToMap(null, ";");
- Assert.assertTrue(a.isEmpty());
- a = CollectionUtils.stringToMap("", ";");
- Assert.assertTrue(a.isEmpty());
- a = CollectionUtils.stringToMap("a=b;c=d", "");
- Assert.assertTrue(a.isEmpty());
- a = CollectionUtils.stringToMap("a=b;c=d", null);
- Assert.assertTrue(a.isEmpty());
- a = CollectionUtils.stringToMap("a=b;c=d;e=f", ";");
- Assert.assertEquals(3, a.size());
- a = CollectionUtils.stringToMap("a;b=f", ";");
- Assert.assertTrue(a.isEmpty());
- a = CollectionUtils.stringToMap("a=b;c=d;e=f;", ";", "test");
- Assert.assertEquals(3, a.size());
- Assert.assertNotNull(a.get("testa"));
- }
-
- @Test
- public void getListByExclusion() {
- Assert.assertNotNull(CollectionUtils.getListByExclusion(null, null));
- List<Integer> originList = new ArrayList<>();
- originList.add(1);
- originList.add(2);
- List<Map<String, Object>> ret = CollectionUtils.getListByExclusion(originList, null);
- Assert.assertEquals(2, ret.size());
- ret = CollectionUtils.getListByExclusion(originList, new HashSet<>());
- Assert.assertEquals(2, ret.size());
- Assert.assertFalse(ret.get(0).isEmpty());
- Set<String> exclusion = new HashSet<>();
- exclusion.add(Constants.CLASS);
- ret = CollectionUtils.getListByExclusion(originList, exclusion);
- Assert.assertEquals(2, ret.size());
- Assert.assertTrue(ret.get(0).isEmpty());
- }
-
- @Test
- public void isNotEmpty() {
- List<Integer> list = new ArrayList<>();
- Assert.assertFalse(CollectionUtils.isNotEmpty(list));
- Assert.assertFalse(CollectionUtils.isNotEmpty(null));
- }
- @Test
- public void isEmpty(){
- List<Integer> list = new ArrayList<>();
- Assert.assertTrue(CollectionUtils.isEmpty(list));
- Assert.assertTrue(CollectionUtils.isEmpty(null));
- list.add(1);
- Assert.assertFalse(CollectionUtils.isEmpty(list));
- }
- @Test
- public void isEqualCollection() {
- List<Integer> a = new ArrayList<>();
- a.add(1);
- List<Integer> b = new ArrayList<>();
- b.add(1);
- Assert.assertTrue(CollectionUtils.isEqualCollection(a,b));
- b.add(2);
- Assert.assertFalse(CollectionUtils.isEqualCollection(a,b));
- }
-
- @Test
- public void getCardinalityMap(){
- List<Integer> a = new ArrayList<>();
- a.add(1);
- a.add(2);
- a.add(2);
- a.add(3);
- a.add(3);
- a.add(3);
- Map<Integer,Integer> cardinalityMap = CollectionUtils.getCardinalityMap(a);
- Assert.assertEquals(3, cardinalityMap.size());
- Assert.assertEquals(1, cardinalityMap.get(1).intValue());
- Assert.assertEquals(2, cardinalityMap.get(2).intValue());
- Assert.assertEquals(3, cardinalityMap.get(3).intValue());
- }
-
- @Test
- public void transformToList() {
- List<String> stringList = new ArrayList<>();
- stringList.add("1");
- List<Integer> integers = CollectionUtils.transformToList(stringList, String::length);
- Assert.assertFalse(integers.isEmpty());
- }
-
- @Test
- public void collectionToMap() {
- List<String> stringList = new ArrayList<>();
- stringList.add("1");
- Map<Integer, String> lengthStringMap = CollectionUtils.collectionToMap(stringList, String::length);
- Assert.assertFalse(lengthStringMap.isEmpty());
- }
-}
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
index 4cbd4ae..42113de 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
@@ -37,14 +37,6 @@ import org.powermock.modules.junit4.PowerMockRunner;
public class FileUtilsTest {
@Test
- public void suffix() {
- Assert.assertEquals("java", FileUtils.suffix("ninfor.java"));
- Assert.assertEquals("", FileUtils.suffix(null));
- Assert.assertEquals("", FileUtils.suffix(""));
- Assert.assertEquals("", FileUtils.suffix("ninfor-java"));
- }
-
- @Test
public void testGetDownloadFilename() {
PowerMockito.mockStatic(DateUtils.class);
PowerMockito.when(DateUtils.getCurrentTime(YYYYMMDDHHMMSS)).thenReturn("20190101101059");
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LoggerUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LoggerUtilsTest.java
deleted file mode 100644
index 80f0f58..0000000
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/LoggerUtilsTest.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.common.utils;
-
-import java.io.BufferedReader;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.List;
-import java.util.Optional;
-
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.Test.None;
-import org.junit.runner.RunWith;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({LoggerUtils.class})
-public class LoggerUtilsTest {
- private Logger logger = LoggerFactory.getLogger(LoggerUtilsTest.class);
-
- @Test
- public void buildTaskId() {
-
- String taskId = LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX, 798L,1,4084, 15210);
-
- Assert.assertEquals(" - [taskAppId=TASK-798_1-4084-15210]", taskId);
- }
-
- @Test
- public void getAppIds() {
- List<String> appIdList = LoggerUtils.getAppIds("Running job: application_1_1", logger);
- Assert.assertEquals("application_1_1", appIdList.get(0));
-
- }
-
- @Test
- public void testReadWholeFileContent() throws Exception {
- BufferedReader bufferedReader = PowerMockito.mock(BufferedReader.class);
- PowerMockito.whenNew(BufferedReader.class).withAnyArguments().thenReturn(bufferedReader);
- PowerMockito.when(bufferedReader.readLine()).thenReturn("").thenReturn(null);
- FileInputStream fileInputStream = PowerMockito.mock(FileInputStream.class);
- PowerMockito.whenNew(FileInputStream.class).withAnyArguments().thenReturn(fileInputStream);
-
- InputStreamReader inputStreamReader = PowerMockito.mock(InputStreamReader.class);
- PowerMockito.whenNew(InputStreamReader.class).withAnyArguments().thenReturn(inputStreamReader);
-
- String log = LoggerUtils.readWholeFileContent("/tmp/log");
- Assert.assertNotNull(log);
-
- PowerMockito.when(bufferedReader.readLine()).thenThrow(new IOException());
- log = LoggerUtils.readWholeFileContent("/tmp/log");
- Assert.assertNotNull(log);
- }
-
- @Test(expected = None.class)
- public void testLogError() {
- Optional<Logger> loggerOptional = Optional.of(this.logger);
-
- LoggerUtils.logError(loggerOptional, "error message");
- LoggerUtils.logError(loggerOptional, new RuntimeException("error message"));
- LoggerUtils.logError(loggerOptional, "error message", new RuntimeException("runtime exception"));
- LoggerUtils.logInfo(loggerOptional, "info message");
- }
-}
\ No newline at end of file
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
index 8467cec..ef0fe1e 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java
@@ -16,6 +16,8 @@
*/
package org.apache.dolphinscheduler.common.utils;
+import org.apache.commons.lang.SystemUtils;
+
import java.io.IOException;
import java.util.List;
@@ -36,11 +38,9 @@ public class OSUtilsTest {
@Test
public void testOSMetric() {
- if (!OSUtils.isWindows()) {
+ if (!SystemUtils.IS_OS_WINDOWS) {
double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize();
Assert.assertTrue(availablePhysicalMemorySize >= 0.0d);
- double totalPhysicalMemorySize = OSUtils.totalPhysicalMemorySize();
- Assert.assertTrue(totalPhysicalMemorySize >= 0.0d);
double loadAverage = OSUtils.loadAverage();
logger.info("loadAverage {}", loadAverage);
double memoryUsage = OSUtils.memoryUsage();
@@ -87,7 +87,7 @@ public class OSUtilsTest {
@Test
public void exeCmd() {
- if (OSUtils.isMacOS() || !OSUtils.isWindows()) {
+ if (SystemUtils.IS_OS_MAC || !SystemUtils.IS_OS_WINDOWS) {
try {
String result = OSUtils.exeCmd("echo helloWorld");
Assert.assertEquals("helloWorld\n",result);
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java
index 19b7853..7841e46 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/RetryerUtilsTest.java
@@ -213,113 +213,4 @@ public class RetryerUtilsTest {
testRetryExceptionWithPara(true);
testRetryExceptionWithPara(false);
}
-
- @Test
- public void testRetrySilent() {
- try {
- for (int execTarget = 1; execTarget <= 3; execTarget++) {
- int finalExecTarget = execTarget;
- int[] execTime = {0};
- boolean result = RetryerUtils.retryCallSilent(() -> {
- execTime[0]++;
- return execTime[0] == finalExecTarget;
- });
- Assert.assertEquals(finalExecTarget, execTime[0]);
- Assert.assertTrue(result);
- }
- } catch (Exception e) {
- Assert.fail("Unexpected exception " + e.getMessage());
- }
- int[] execTime = {0};
- try {
- boolean result = RetryerUtils.retryCallSilent(() -> {
- execTime[0]++;
- return execTime[0] == 4;
- });
- Assert.assertFalse(result);
- } catch (Exception e) {
- Assert.fail("Unexpected exception " + e.getMessage());
- }
- }
-
- @Test
- public void testRetrySilentWithPara() {
- try {
- for (int execTarget = 1; execTarget <= 3; execTarget++) {
- int finalExecTarget = execTarget;
- int[] execTime = {0};
- boolean result = RetryerUtils.retryCallSilent(() -> {
- execTime[0]++;
- return execTime[0] == finalExecTarget;
- }, true);
- Assert.assertEquals(finalExecTarget, execTime[0]);
- Assert.assertTrue(result);
- }
- } catch (Exception e) {
- Assert.fail("Unexpected exception " + e.getMessage());
- }
- int[] execTime = {0};
- try {
- boolean result = RetryerUtils.retryCallSilent(() -> {
- execTime[0]++;
- return execTime[0] == 4;
- }, true);
- Assert.assertFalse(result);
- } catch (Exception e) {
- Assert.fail("Unexpected exception " + e.getMessage());
- }
- }
- @Test
- public void testRetrySilentNoCheckResult(){
- try {
- for (int execTarget = 1; execTarget <= 5; execTarget++) {
- int[] execTime = {0};
- boolean result = RetryerUtils.retryCallSilent(() -> {
- execTime[0]++;
- return execTime[0] > 1;
- }, false);
- Assert.assertEquals(1, execTime[0]);
- Assert.assertFalse(result);
- }
- } catch (Exception e) {
- Assert.fail("Unexpected exception " + e.getMessage());
- }
- }
- private void testRetrySilentExceptionWithPara(boolean checkResult) {
- try {
- for (int execTarget = 1; execTarget <= 3; execTarget++) {
- int finalExecTarget = execTarget;
- int[] execTime = {0};
- boolean result = RetryerUtils.retryCallSilent(() -> {
- execTime[0]++;
- if (execTime[0] != finalExecTarget) {
- throw new IllegalArgumentException(String.valueOf(execTime[0]));
- }
- return true;
- }, checkResult);
- Assert.assertEquals(finalExecTarget, execTime[0]);
- Assert.assertTrue(result);
- }
- } catch (Exception e) {
- Assert.fail("Unexpected exception " + e.getMessage());
- }
- int[] execTime = {0};
- try {
- boolean result = RetryerUtils.retryCallSilent(() -> {
- execTime[0]++;
- if (execTime[0] != 4) {
- throw new IllegalArgumentException(String.valueOf(execTime[0]));
- }
- return true;
- }, checkResult);
- Assert.assertFalse(result);
- } catch (Exception e) {
- Assert.fail("Unexpected exception " + e.getMessage());
- }
- }
- @Test
- public void testRetrySilentException() {
- testRetrySilentExceptionWithPara(true);
- testRetrySilentExceptionWithPara(false);
- }
}
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java
index 7885806..b385523 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java
@@ -105,7 +105,8 @@ public class SchemaUtilsTest {
List<String> real = SchemaUtils.getAllSchemaList();
List<String> expect = Arrays.asList("1.0.1_schema", "1.0.2_schema",
"1.1.0_schema", "1.2.0_schema");
- Assert.assertTrue(CollectionUtils.isEqualCollection(real, expect));
+ boolean result = org.apache.commons.collections.CollectionUtils.isEqualCollection(real, expect);
+ Assert.assertTrue(result);
//normal
files = new File[0];
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtilsTest.java
deleted file mode 100644
index 03880b6..0000000
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SensitiveLogUtilsTest.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.dolphinscheduler.common.utils;
-
-
-import org.apache.dolphinscheduler.common.Constants;
-import org.junit.Assert;
-import org.junit.Test;
-
-
-public class SensitiveLogUtilsTest {
-
- @Test
- public void testMaskDataSourcePwd() {
-
- String password = "123456";
- String emptyPassword = "";
-
- Assert.assertEquals(Constants.PASSWORD_DEFAULT, SensitiveLogUtils.maskDataSourcePwd(password));
- Assert.assertEquals("", SensitiveLogUtils.maskDataSourcePwd(emptyPassword));
-
- }
-}
diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml
index 0d03129..f8ccb3a 100644
--- a/dolphinscheduler-dao/pom.xml
+++ b/dolphinscheduler-dao/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-dao</artifactId>
<name>${project.artifactId}</name>
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java
index b6aa9e5..afaa0bd 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/PluginDao.java
@@ -19,11 +19,12 @@ package org.apache.dolphinscheduler.dao;
import static java.util.Objects.requireNonNull;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
import org.apache.dolphinscheduler.dao.entity.PluginDefine;
import org.apache.dolphinscheduler.dao.mapper.PluginDefineMapper;
+import org.apache.commons.collections.CollectionUtils;
+
import java.util.List;
import org.slf4j.Logger;
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
index 68fd446..3eba405 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
@@ -16,6 +16,7 @@
*/
package org.apache.dolphinscheduler.dao.upgrade;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ConditionType;
import org.apache.dolphinscheduler.common.enums.DbType;
@@ -26,7 +27,6 @@ import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SchemaUtils;
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java
index e692645..3501291 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/DagHelper.java
@@ -25,11 +25,12 @@ import org.apache.dolphinscheduler.common.process.ProcessDag;
import org.apache.dolphinscheduler.common.task.conditions.ConditionsParameters;
import org.apache.dolphinscheduler.common.task.switchtask.SwitchParameters;
import org.apache.dolphinscheduler.common.task.switchtask.SwitchResultVo;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.commons.collections.CollectionUtils;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -87,8 +88,7 @@ public class DagHelper {
List<TaskNode> destFlowNodeList = new ArrayList<>();
List<String> startNodeList = startNodeNameList;
- if (taskDependType != TaskDependType.TASK_POST
- && CollectionUtils.isEmpty(startNodeList)) {
+ if (taskDependType != TaskDependType.TASK_POST && CollectionUtils.isEmpty(startNodeList)) {
logger.error("start node list is empty! cannot continue run the process ");
return destFlowNodeList;
}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java
index 4531579..483c052 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java
@@ -16,7 +16,7 @@
*/
package org.apache.dolphinscheduler.dao.utils;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
+import org.apache.commons.collections.CollectionUtils;
import java.util.Arrays;
import java.util.HashMap;
diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java
index e2ba437..dad9638 100644
--- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java
+++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java
@@ -23,10 +23,10 @@ import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UserType;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.ResourcesUser;
import org.apache.dolphinscheduler.dao.entity.Tenant;
diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml
index 1c82132..00e0773 100644
--- a/dolphinscheduler-dist/pom.xml
+++ b/dolphinscheduler-dist/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/dolphinscheduler-dist/src/main/provisio/dolphinscheduler.xml b/dolphinscheduler-dist/src/main/provisio/dolphinscheduler.xml
index 9ecd9ef..d7119d0 100644
--- a/dolphinscheduler-dist/src/main/provisio/dolphinscheduler.xml
+++ b/dolphinscheduler-dist/src/main/provisio/dolphinscheduler.xml
@@ -75,11 +75,6 @@
<unpack/>
</artifact>
</artifactSet>
- <artifactSet to="lib/plugin/registry/zookeeper">
- <artifact id="${project.groupId}:dolphinscheduler-registry-zookeeper:zip:${project.version}">
- <unpack/>
- </artifact>
- </artifactSet>
<!-- Task Plugins -->
<artifactSet to="lib/plugin/task/datax">
<artifact id="${project.groupId}:dolphinscheduler-task-datax:zip:${project.version}">
diff --git a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConnectionStateListener.java b/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConnectionStateListener.java
deleted file mode 100644
index cda98ef..0000000
--- a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConnectionStateListener.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.registry.zookeeper;
-
-import org.apache.dolphinscheduler.spi.register.RegistryConnectListener;
-import org.apache.dolphinscheduler.spi.register.RegistryConnectState;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.state.ConnectionState;
-import org.apache.curator.framework.state.ConnectionStateListener;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class ZookeeperConnectionStateListener implements ConnectionStateListener {
-
- private static final Logger logger = LoggerFactory.getLogger(ZookeeperConnectionStateListener.class);
-
- private RegistryConnectListener registryConnectListener;
-
- public ZookeeperConnectionStateListener(RegistryConnectListener registryConnectListener) {
- this.registryConnectListener = registryConnectListener;
- }
-
- @Override
- public void stateChanged(CuratorFramework client, ConnectionState newState) {
-
- if (newState == ConnectionState.LOST) {
- logger.error("connection lost from zookeeper");
- registryConnectListener.notify(RegistryConnectState.LOST);
- } else if (newState == ConnectionState.RECONNECTED) {
- logger.info("reconnected to zookeeper");
- registryConnectListener.notify(RegistryConnectState.RECONNECTED);
- } else if (newState == ConnectionState.SUSPENDED) {
- logger.warn("zookeeper connection SUSPENDED");
- registryConnectListener.notify(RegistryConnectState.SUSPENDED);
- }
-
- }
-
-}
diff --git a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryPlugin.java b/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryPlugin.java
deleted file mode 100644
index 85723ad..0000000
--- a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryPlugin.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.registry.zookeeper;
-
-import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin;
-import org.apache.dolphinscheduler.spi.register.RegistryFactory;
-
-import com.google.common.collect.ImmutableList;
-
-/**
- * zookeeper registry plugin
- */
-public class ZookeeperRegistryPlugin implements DolphinSchedulerPlugin {
-
- @Override
- public Iterable<RegistryFactory> getRegisterFactorys() {
- return ImmutableList.of(new ZookeeperRegistryFactory());
- }
-}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-api/pom.xml
new file mode 100644
index 0000000..1338128
--- /dev/null
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/pom.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to Apache Software Foundation (ASF) under one or more contributor
+ ~ license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright
+ ~ ownership. Apache Software Foundation (ASF) licenses this file to you under
+ ~ the Apache License, Version 2.0 (the "License"); you may
+ ~ not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>dolphinscheduler-registry</artifactId>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <version>2.0.1-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>dolphinscheduler-registry-api</artifactId>
+</project>
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectionListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectionListener.java
new file mode 100644
index 0000000..eaebc81
--- /dev/null
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectionListener.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to Apache Software Foundation (ASF) under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Apache Software Foundation (ASF) licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.dolphinscheduler.registry.api;
+
+@FunctionalInterface
+public interface ConnectionListener {
+ void onUpdate(ConnectionState newState);
+}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectionState.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectionState.java
new file mode 100644
index 0000000..fef3bca
--- /dev/null
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/ConnectionState.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to Apache Software Foundation (ASF) under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Apache Software Foundation (ASF) licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.dolphinscheduler.registry.api;
+
+public enum ConnectionState {
+ CONNECTED,
+ RECONNECTED,
+ SUSPENDED,
+ DISCONNECTED
+}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java
new file mode 100644
index 0000000..be781db
--- /dev/null
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Event.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Apache Software Foundation (ASF) under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Apache Software Foundation (ASF) licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.dolphinscheduler.registry.api;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import lombok.ToString;
+import lombok.experimental.Accessors;
+
+@Getter
+@Setter
+@Builder
+@ToString
+@NoArgsConstructor
+@AllArgsConstructor
+@Accessors(fluent = true)
+public class Event {
+ private String key;
+ private String path;
+ private String data;
+ private Type type;
+
+ public enum Type {
+ ADD,
+ REMOVE,
+ UPDATE
+ }
+}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java
new file mode 100644
index 0000000..6057a7e
--- /dev/null
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/Registry.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to Apache Software Foundation (ASF) under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Apache Software Foundation (ASF) licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.dolphinscheduler.registry.api;
+
+import java.io.Closeable;
+import java.util.Collection;
+import java.util.Map;
+
+public interface Registry extends Closeable {
+ void start(Map<String, String> config);
+
+ boolean subscribe(String path, SubscribeListener listener);
+
+ void unsubscribe(String path);
+
+ void addConnectionStateListener(ConnectionListener listener);
+
+ String get(String key);
+
+ void put(String key, String value, boolean deleteOnDisconnect);
+
+ void delete(String key);
+
+ Collection<String> children(String key);
+
+ boolean exists(String key);
+
+ boolean acquireLock(String key);
+
+ boolean releaseLock(String key);
+}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryException.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryException.java
new file mode 100644
index 0000000..b88fe25
--- /dev/null
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to Apache Software Foundation (ASF) under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Apache Software Foundation (ASF) licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.dolphinscheduler.registry.api;
+
+public final class RegistryException extends RuntimeException {
+
+ public RegistryException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public RegistryException(String message) {
+ super(message);
+ }
+}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryFactory.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryFactory.java
new file mode 100644
index 0000000..6903e95
--- /dev/null
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryFactory.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to Apache Software Foundation (ASF) under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Apache Software Foundation (ASF) licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.dolphinscheduler.registry.api;
+
+public interface RegistryFactory {
+ String name();
+
+ Registry create();
+}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryFactoryLoader.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryFactoryLoader.java
new file mode 100644
index 0000000..45d4596
--- /dev/null
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/RegistryFactoryLoader.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to Apache Software Foundation (ASF) under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Apache Software Foundation (ASF) licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.dolphinscheduler.registry.api;
+
+import static java.util.stream.Collectors.toMap;
+
+import java.util.Map;
+import java.util.ServiceLoader;
+import java.util.function.Function;
+import java.util.stream.StreamSupport;
+
+public final class RegistryFactoryLoader {
+ public static Map<String, RegistryFactory> load() {
+ final ServiceLoader<RegistryFactory> factories = ServiceLoader.load(RegistryFactory.class);
+ return StreamSupport.stream(factories.spliterator(), false)
+ .collect(toMap(RegistryFactory::name, Function.identity()));
+ }
+}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java
new file mode 100644
index 0000000..2432eb1
--- /dev/null
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-api/src/main/java/org/apache/dolphinscheduler/registry/api/SubscribeListener.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to Apache Software Foundation (ASF) under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Apache Software Foundation (ASF) licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.dolphinscheduler.registry.api;
+
+public interface SubscribeListener {
+ void notify(Event event);
+}
diff --git a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/pom.xml
similarity index 87%
rename from dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml
rename to dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/pom.xml
index 28311e7..58d3f31 100644
--- a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/pom.xml
@@ -17,18 +17,19 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
- <artifactId>dolphinscheduler-registry-plugin</artifactId>
+ <artifactId>dolphinscheduler-registry-plugins</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
-
<artifactId>dolphinscheduler-registry-zookeeper</artifactId>
- <!-- can be load as a Alert Plugin when development and run server in IDE -->
- <packaging>dolphinscheduler-plugin</packaging>
<dependencies>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-registry-api</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
@@ -55,7 +56,6 @@
<artifactId>slf4j-api</artifactId>
</dependency>
-
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
@@ -74,11 +74,5 @@
<classifier>runtime</classifier>
<scope>test</scope>
</dependency>
-
</dependencies>
-
- <build>
- <finalName>dolphinscheduler-registry-zookeeper-${project.version}</finalName>
- </build>
-
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConfiguration.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConfiguration.java
similarity index 100%
rename from dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConfiguration.java
rename to dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConfiguration.java
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConnectionStateListener.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConnectionStateListener.java
new file mode 100644
index 0000000..9526e0e
--- /dev/null
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperConnectionStateListener.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.registry.zookeeper;
+
+import org.apache.dolphinscheduler.registry.api.ConnectionListener;
+import org.apache.dolphinscheduler.registry.api.ConnectionState;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.state.ConnectionStateListener;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RequiredArgsConstructor
+public final class ZookeeperConnectionStateListener implements ConnectionStateListener {
+ private final ConnectionListener listener;
+
+ @Override
+ public void stateChanged(CuratorFramework client,
+ org.apache.curator.framework.state.ConnectionState newState) {
+ switch (newState) {
+ case LOST:
+ log.warn("Registry disconnected");
+ listener.onUpdate(ConnectionState.DISCONNECTED);
+ break;
+ case RECONNECTED:
+ log.info("Registry reconnected");
+ listener.onUpdate(ConnectionState.RECONNECTED);
+ break;
+ case SUSPENDED:
+ log.warn("Registry suspended");
+ listener.onUpdate(ConnectionState.SUSPENDED);
+ break;
+ default:
+ break;
+ }
+ }
+}
diff --git a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java
similarity index 53%
rename from dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java
rename to dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java
index e84666a..89cb280 100644
--- a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistry.java
@@ -28,21 +28,19 @@ import static org.apache.dolphinscheduler.plugin.registry.zookeeper.ZookeeperCon
import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import org.apache.dolphinscheduler.spi.register.DataChangeEvent;
-import org.apache.dolphinscheduler.spi.register.ListenerManager;
-import org.apache.dolphinscheduler.spi.register.Registry;
-import org.apache.dolphinscheduler.spi.register.RegistryConnectListener;
-import org.apache.dolphinscheduler.spi.register.RegistryException;
-import org.apache.dolphinscheduler.spi.register.SubscribeListener;
+import org.apache.dolphinscheduler.registry.api.ConnectionListener;
+import org.apache.dolphinscheduler.registry.api.Event;
+import org.apache.dolphinscheduler.registry.api.Registry;
+import org.apache.dolphinscheduler.registry.api.RegistryException;
+import org.apache.dolphinscheduler.registry.api.SubscribeListener;
import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.framework.api.ACLProvider;
-import org.apache.curator.framework.api.transaction.TransactionOp;
+import org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.curator.framework.recipes.cache.TreeCache;
import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
-import org.apache.curator.framework.recipes.cache.TreeCacheListener;
import org.apache.curator.framework.recipes.locks.InterProcessMutex;
import org.apache.curator.retry.ExponentialBackoffRetry;
import org.apache.curator.utils.CloseableUtils;
@@ -56,28 +54,18 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import com.google.common.base.Strings;
-public class ZookeeperRegistry implements Registry {
+public final class ZookeeperRegistry implements Registry {
private CuratorFramework client;
- /**
- * treeCache map
- * k-subscribe key
- * v-listener
- */
- private Map<String, TreeCache> treeCacheMap = new HashMap<>();
-
- /**
- * Distributed lock map
- */
- private ThreadLocal<Map<String, InterProcessMutex>> threadLocalLockMap = new ThreadLocal<>();
-
- /**
- * build retry policy
- */
+ private final Map<String, TreeCache> treeCacheMap = new ConcurrentHashMap<>();
+
+ private static final ThreadLocal<Map<String, InterProcessMutex>> threadLocalLockMap = new ThreadLocal<>();
+
private static RetryPolicy buildRetryPolicy(Map<String, String> registerData) {
int baseSleepTimeMs = BASE_SLEEP_TIME.getParameterValue(registerData.get(BASE_SLEEP_TIME.getName()));
int maxRetries = MAX_RETRIES.getParameterValue(registerData.get(MAX_RETRIES.getName()));
@@ -85,35 +73,32 @@ public class ZookeeperRegistry implements Registry {
return new ExponentialBackoffRetry(baseSleepTimeMs, maxRetries, maxSleepMs);
}
- /**
- * build digest
- */
private static void buildDigest(CuratorFrameworkFactory.Builder builder, String digest) {
builder.authorization(DIGEST.getName(), digest.getBytes(StandardCharsets.UTF_8))
- .aclProvider(new ACLProvider() {
- @Override
- public List<ACL> getDefaultAcl() {
- return ZooDefs.Ids.CREATOR_ALL_ACL;
- }
-
- @Override
- public List<ACL> getAclForPath(final String path) {
- return ZooDefs.Ids.CREATOR_ALL_ACL;
- }
- });
+ .aclProvider(new ACLProvider() {
+ @Override
+ public List<ACL> getDefaultAcl() {
+ return ZooDefs.Ids.CREATOR_ALL_ACL;
+ }
+
+ @Override
+ public List<ACL> getAclForPath(final String path) {
+ return ZooDefs.Ids.CREATOR_ALL_ACL;
+ }
+ });
}
@Override
- public void init(Map<String, String> registerData) {
-
- CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder()
- .connectString(SERVERS.getParameterValue(registerData.get(SERVERS.getName())))
- .retryPolicy(buildRetryPolicy(registerData))
- .namespace(NAME_SPACE.getParameterValue(registerData.get(NAME_SPACE.getName())))
- .sessionTimeoutMs(SESSION_TIMEOUT_MS.getParameterValue(registerData.get(SESSION_TIMEOUT_MS.getName())))
- .connectionTimeoutMs(CONNECTION_TIMEOUT_MS.getParameterValue(registerData.get(CONNECTION_TIMEOUT_MS.getName())));
-
- String digest = DIGEST.getParameterValue(registerData.get(DIGEST.getName()));
+ public void start(Map<String, String> config) {
+ CuratorFrameworkFactory.Builder builder =
+ CuratorFrameworkFactory.builder()
+ .connectString(SERVERS.getParameterValue(config.get(SERVERS.getName())))
+ .retryPolicy(buildRetryPolicy(config))
+ .namespace(NAME_SPACE.getParameterValue(config.get(NAME_SPACE.getName())))
+ .sessionTimeoutMs(SESSION_TIMEOUT_MS.getParameterValue(config.get(SESSION_TIMEOUT_MS.getName())))
+ .connectionTimeoutMs(CONNECTION_TIMEOUT_MS.getParameterValue(config.get(CONNECTION_TIMEOUT_MS.getName())));
+
+ String digest = DIGEST.getParameterValue(config.get(DIGEST.getName()));
if (!Strings.isNullOrEmpty(digest)) {
buildDigest(builder, digest);
}
@@ -121,7 +106,7 @@ public class ZookeeperRegistry implements Registry {
client.start();
try {
- if (!client.blockUntilConnected(BLOCK_UNTIL_CONNECTED_WAIT_MS.getParameterValue(registerData.get(BLOCK_UNTIL_CONNECTED_WAIT_MS.getName())), MILLISECONDS)) {
+ if (!client.blockUntilConnected(BLOCK_UNTIL_CONNECTED_WAIT_MS.getParameterValue(config.get(BLOCK_UNTIL_CONNECTED_WAIT_MS.getName())), MILLISECONDS)) {
client.close();
throw new RegistryException("zookeeper connect timeout");
}
@@ -132,55 +117,26 @@ public class ZookeeperRegistry implements Registry {
}
@Override
- public void addConnectionStateListener(RegistryConnectListener registryConnectListener) {
- client.getConnectionStateListenable().addListener(new ZookeeperConnectionStateListener(registryConnectListener));
+ public void addConnectionStateListener(ConnectionListener listener) {
+ client.getConnectionStateListenable().addListener(new ZookeeperConnectionStateListener(listener));
}
@Override
- public boolean subscribe(String path, SubscribeListener subscribeListener) {
- if (null != treeCacheMap.get(path)) {
- return false;
- }
- TreeCache treeCache = new TreeCache(client, path);
- TreeCacheListener treeCacheListener = (client, event) -> {
- TreeCacheEvent.Type type = event.getType();
- DataChangeEvent eventType = null;
- String dataPath = null;
- switch (type) {
- case NODE_ADDED:
- dataPath = event.getData().getPath();
- eventType = DataChangeEvent.ADD;
- break;
- case NODE_UPDATED:
- eventType = DataChangeEvent.UPDATE;
- dataPath = event.getData().getPath();
- break;
- case NODE_REMOVED:
- eventType = DataChangeEvent.REMOVE;
- dataPath = event.getData().getPath();
- break;
- default:
- }
- if (null != eventType && null != dataPath) {
- ListenerManager.dataChange(path, dataPath, new String(event.getData().getData()), eventType);
- }
- };
- treeCache.getListenable().addListener(treeCacheListener);
- treeCacheMap.put(path, treeCache);
+ public boolean subscribe(String path, SubscribeListener listener) {
+ final TreeCache treeCache = treeCacheMap.computeIfAbsent(path, $ -> new TreeCache(client, path));
+ treeCache.getListenable().addListener(($, event) -> listener.notify(new EventAdaptor(event, path)));
try {
treeCache.start();
} catch (Exception e) {
- throw new RegistryException("start zookeeper tree cache error", e);
+ treeCacheMap.remove(path);
+ throw new RegistryException("Failed to subscribe listener for key: " + path, e);
}
- ListenerManager.addListener(path, subscribeListener);
return true;
}
@Override
public void unsubscribe(String path) {
- TreeCache treeCache = treeCacheMap.get(path);
- treeCache.close();
- ListenerManager.removeListener(path);
+ CloseableUtils.closeQuietly(treeCacheMap.get(path));
}
@Override
@@ -193,12 +149,7 @@ public class ZookeeperRegistry implements Registry {
}
@Override
- public void remove(String key) {
- delete(key);
- }
-
- @Override
- public boolean isExisted(String key) {
+ public boolean exists(String key) {
try {
return null != client.checkExists().forPath(key);
} catch (Exception e) {
@@ -207,47 +158,22 @@ public class ZookeeperRegistry implements Registry {
}
@Override
- public void persist(String key, String value) {
- try {
- if (isExisted(key)) {
- update(key, value);
- return;
- }
- client.create().creatingParentsIfNeeded().withMode(CreateMode.PERSISTENT).forPath(key, value.getBytes(StandardCharsets.UTF_8));
+ public void put(String key, String value, boolean deleteOnDisconnect) {
+ final CreateMode mode = deleteOnDisconnect ? CreateMode.EPHEMERAL : CreateMode.PERSISTENT;
- } catch (Exception e) {
- throw new RegistryException("zookeeper persist error", e);
- }
- }
-
- @Override
- public void persistEphemeral(String key, String value) {
try {
- if (isExisted(key)) {
- update(key, value);
- return;
- }
- client.create().creatingParentsIfNeeded().withMode(CreateMode.EPHEMERAL).forPath(key, value.getBytes(StandardCharsets.UTF_8));
+ client.create()
+ .orSetData()
+ .creatingParentsIfNeeded()
+ .withMode(mode)
+ .forPath(key, value.getBytes(StandardCharsets.UTF_8));
} catch (Exception e) {
- throw new RegistryException("zookeeper persist ephemeral error", e);
+ throw new RegistryException("Failed to put registry key: " + key, e);
}
}
@Override
- public void update(String key, String value) {
- try {
- if (!isExisted(key)) {
- return;
- }
- TransactionOp transactionOp = client.transactionOp();
- client.transaction().forOperations(transactionOp.check().forPath(key), transactionOp.setData().forPath(key, value.getBytes(StandardCharsets.UTF_8)));
- } catch (Exception e) {
- throw new RegistryException("zookeeper update error", e);
- }
- }
-
- @Override
- public List<String> getChildren(String key) {
+ public List<String> children(String key) {
try {
List<String> result = client.getChildren().forPath(key);
result.sort(Comparator.reverseOrder());
@@ -258,23 +184,20 @@ public class ZookeeperRegistry implements Registry {
}
@Override
- public boolean delete(String nodePath) {
+ public void delete(String nodePath) {
try {
client.delete()
- .deletingChildrenIfNeeded()
- .forPath(nodePath);
- } catch (KeeperException.NoNodeException ignore) {
- // the node is not exist, we can believe the node has been removed
-
+ .deletingChildrenIfNeeded()
+ .forPath(nodePath);
+ } catch (KeeperException.NoNodeException ignored) {
+ // Is already deleted or does not exist
} catch (Exception e) {
- throw new RegistryException("zookeeper delete key error", e);
+ throw new RegistryException("Failed to delete registry key: " + nodePath, e);
}
- return true;
}
@Override
public boolean acquireLock(String key) {
-
InterProcessMutex interProcessMutex = new InterProcessMutex(client, key);
try {
interProcessMutex.acquire();
@@ -291,7 +214,6 @@ public class ZookeeperRegistry implements Registry {
throw new RegistryException("zookeeper release lock error", e);
}
}
-
}
@Override
@@ -311,22 +233,35 @@ public class ZookeeperRegistry implements Registry {
return true;
}
- public CuratorFramework getClient() {
- return client;
- }
-
@Override
public void close() {
- treeCacheMap.forEach((key, value) -> value.close());
- waitForCacheClose(500);
+ treeCacheMap.values().forEach(CloseableUtils::closeQuietly);
CloseableUtils.closeQuietly(client);
}
- private void waitForCacheClose(long millis) {
- try {
- Thread.sleep(millis);
- } catch (final InterruptedException ex) {
- Thread.currentThread().interrupt();
+ static final class EventAdaptor extends Event {
+ public EventAdaptor(TreeCacheEvent event, String key) {
+ key(key);
+
+ switch (event.getType()) {
+ case NODE_ADDED:
+ type(Type.ADD);
+ break;
+ case NODE_UPDATED:
+ type(Type.UPDATE);
+ break;
+ case NODE_REMOVED:
+ type(Type.REMOVE);
+ break;
+ default:
+ break;
+ }
+
+ final ChildData data = event.getData();
+ if (data != null) {
+ path(data.getPath());
+ data(new String(data.getData()));
+ }
}
}
}
diff --git a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryFactory.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryFactory.java
similarity index 76%
rename from dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryFactory.java
rename to dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryFactory.java
index 1ecf3e0..949df21 100644
--- a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryFactory.java
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/main/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryFactory.java
@@ -17,16 +17,16 @@
package org.apache.dolphinscheduler.plugin.registry.zookeeper;
-import org.apache.dolphinscheduler.spi.register.Registry;
-import org.apache.dolphinscheduler.spi.register.RegistryFactory;
+import org.apache.dolphinscheduler.registry.api.Registry;
+import org.apache.dolphinscheduler.registry.api.RegistryFactory;
-/**
- * Zookeeper registry factory
- */
-public class ZookeeperRegistryFactory implements RegistryFactory {
+import com.google.auto.service.AutoService;
+
+@AutoService(RegistryFactory.class)
+public final class ZookeeperRegistryFactory implements RegistryFactory {
@Override
- public String getName() {
+ public String name() {
return "zookeeper";
}
diff --git a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTest.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTest.java
similarity index 87%
rename from dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTest.java
rename to dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTest.java
index a5dc33b..8442c02 100644
--- a/dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTest.java
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTest.java
@@ -17,8 +17,8 @@
package org.apache.dolphinscheduler.plugin.registry.zookeeper;
-import org.apache.dolphinscheduler.spi.register.DataChangeEvent;
-import org.apache.dolphinscheduler.spi.register.SubscribeListener;
+import org.apache.dolphinscheduler.registry.api.Event;
+import org.apache.dolphinscheduler.registry.api.SubscribeListener;
import org.apache.curator.test.TestingServer;
@@ -50,18 +50,18 @@ public class ZookeeperRegistryTest {
server = new TestingServer(true);
Map<String, String> registryConfig = new HashMap<>();
registryConfig.put(ZookeeperConfiguration.SERVERS.getName(), server.getConnectString());
- registry.init(registryConfig);
- registry.persist("/sub", "");
+ registry.start(registryConfig);
+ registry.put("/sub", "", false);
}
@Test
public void persistTest() {
- registry.persist("/nodes/m1", "");
- registry.persist("/nodes/m2", "");
- Assert.assertEquals(Arrays.asList("m2", "m1"), registry.getChildren("/nodes"));
- Assert.assertTrue(registry.isExisted("/nodes/m1"));
+ registry.put("/nodes/m1", "", false);
+ registry.put("/nodes/m2", "", false);
+ Assert.assertEquals(Arrays.asList("m2", "m1"), registry.children("/nodes"));
+ Assert.assertTrue(registry.exists("/nodes/m1"));
registry.delete("/nodes/m2");
- Assert.assertFalse(registry.isExisted("/nodes/m2"));
+ Assert.assertFalse(registry.exists("/nodes/m2"));
}
@Test
@@ -112,10 +112,9 @@ public class ZookeeperRegistryTest {
}
- class TestListener implements SubscribeListener {
-
+ static class TestListener implements SubscribeListener {
@Override
- public void notify(String path, String data, DataChangeEvent dataChangeEvent) {
+ public void notify(Event event) {
logger.info("I'm test listener");
}
}
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/pom.xml b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/pom.xml
new file mode 100644
index 0000000..065c767
--- /dev/null
+++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/pom.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to Apache Software Foundation (ASF) under one or more contributor
+ ~ license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright
+ ~ ownership. Apache Software Foundation (ASF) licenses this file to you under
+ ~ the Apache License, Version 2.0 (the "License"); you may
+ ~ not use this file except in compliance with the License.
+ ~ You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>dolphinscheduler-registry</artifactId>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <version>2.0.1-SNAPSHOT</version>
+ </parent>
+ <artifactId>dolphinscheduler-registry-plugins</artifactId>
+ <modelVersion>4.0.0</modelVersion>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>dolphinscheduler-registry-zookeeper</module>
+ </modules>
+</project>
diff --git a/dolphinscheduler-registry-plugin/pom.xml b/dolphinscheduler-registry/pom.xml
similarity index 72%
rename from dolphinscheduler-registry-plugin/pom.xml
rename to dolphinscheduler-registry/pom.xml
index 0f33159..ff6796c 100644
--- a/dolphinscheduler-registry-plugin/pom.xml
+++ b/dolphinscheduler-registry/pom.xml
@@ -19,23 +19,28 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-registry-plugin</artifactId>
+ <artifactId>dolphinscheduler-registry</artifactId>
<packaging>pom</packaging>
+ <modules>
+ <module>dolphinscheduler-registry-api</module>
+ <module>dolphinscheduler-registry-plugins</module>
+ </modules>
+
<dependencies>
- <!-- dolphinscheduler -->
<dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-spi</artifactId>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>1.18.22</version>
<scope>provided</scope>
</dependency>
</dependencies>
-
- <modules>
- <module>dolphinscheduler-registry-zookeeper</module>
- </modules>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml
index 0b7668e..4067fb6 100644
--- a/dolphinscheduler-remote/pom.xml
+++ b/dolphinscheduler-remote/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml
index d8c83c9..cd0412c 100644
--- a/dolphinscheduler-server/pom.xml
+++ b/dolphinscheduler-server/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-server</artifactId>
<name>dolphinscheduler-server</name>
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java
index 85c9e99..8413fed 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java
@@ -17,17 +17,18 @@
package org.apache.dolphinscheduler.server.log;
-import ch.qos.logback.classic.pattern.MessageConverter;
-import ch.qos.logback.classic.spi.ILoggingEvent;
+import static org.apache.dolphinscheduler.common.Constants.STAR;
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.utils.SensitiveLogUtils;
import org.apache.commons.lang.StringUtils;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import ch.qos.logback.classic.pattern.MessageConverter;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+
/**
* sensitive data log converter
*/
@@ -69,7 +70,7 @@ public class SensitiveDataConverter extends MessageConverter {
*
* @param logMsg original log
*/
- private String passwordHandler(Pattern pwdPattern, String logMsg) {
+ static String passwordHandler(Pattern pwdPattern, String logMsg) {
Matcher matcher = pwdPattern.matcher(logMsg);
@@ -79,7 +80,7 @@ public class SensitiveDataConverter extends MessageConverter {
String password = matcher.group();
- String maskPassword = SensitiveLogUtils.maskDataSourcePwd(password);
+ String maskPassword = StringUtils.repeat(STAR, StringUtils.length(password));
matcher.appendReplacement(sb, maskPassword);
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java
index d1448b3..e0753b7 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/CommonHostManager.java
@@ -18,7 +18,6 @@
package org.apache.dolphinscheduler.server.master.dispatch.host;
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.HeartBeat;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext;
@@ -26,6 +25,7 @@ import org.apache.dolphinscheduler.server.master.dispatch.enums.ExecutorType;
import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker;
import org.apache.dolphinscheduler.server.master.registry.ServerNodeManager;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java
index 6549f9d..2f421e0 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/LowerWeightHostManager.java
@@ -17,8 +17,8 @@
package org.apache.dolphinscheduler.server.master.dispatch.host;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.HeartBeat;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory;
@@ -27,13 +27,12 @@ import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWeight
import org.apache.dolphinscheduler.server.master.dispatch.host.assign.HostWorker;
import org.apache.dolphinscheduler.server.master.dispatch.host.assign.LowerWeightRoundRobin;
import org.apache.dolphinscheduler.spi.utils.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.util.Optional;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
+import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -41,12 +40,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
-import javax.annotation.PostConstruct;
-import javax.annotation.PreDestroy;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
* lower weight host manager
*/
@@ -175,9 +168,9 @@ public class LowerWeightHostManager extends CommonHostManager {
return Optional.empty();
}
return Optional.of(
- new HostWeight(HostWorker.of(addr, heartBeat.getWorkerHostWeight(), workerGroup),
- heartBeat.getCpuUsage(), heartBeat.getMemoryUsage(), heartBeat.getLoadAverage(),
- heartBeat.getStartupTime()));
+ new HostWeight(HostWorker.of(addr, heartBeat.getWorkerHostWeight(), workerGroup),
+ heartBeat.getCpuUsage(), heartBeat.getMemoryUsage(), heartBeat.getLoadAverage(),
+ heartBeat.getStartupTime()));
}
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java
index 087a5ff..d07f62a 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/dispatch/host/assign/AbstractSelector.java
@@ -16,7 +16,7 @@
*/
package org.apache.dolphinscheduler.server.master.dispatch.host.assign;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
+import org.apache.commons.collections.CollectionUtils;
import java.util.Collection;
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
index c157fb0..ee6349a 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClient.java
@@ -32,6 +32,7 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.registry.api.ConnectionState;
import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory;
import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
@@ -41,11 +42,10 @@ import org.apache.dolphinscheduler.server.utils.ProcessUtils;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext;
import org.apache.dolphinscheduler.service.registry.RegistryClient;
-import org.apache.dolphinscheduler.spi.register.RegistryConnectListener;
-import org.apache.dolphinscheduler.spi.register.RegistryConnectState;
import org.apache.commons.lang.StringUtils;
+import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
@@ -79,6 +79,7 @@ public class MasterRegistryClient {
@Autowired
private ProcessService processService;
+ @Autowired
private RegistryClient registryClient;
/**
@@ -103,13 +104,12 @@ public class MasterRegistryClient {
public void init(ConcurrentHashMap<Integer, WorkflowExecuteThread> processInstanceExecMaps) {
this.startupTime = System.currentTimeMillis();
- this.registryClient = RegistryClient.getInstance();
this.heartBeatExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("HeartBeatExecutor"));
this.processInstanceExecMaps = processInstanceExecMaps;
}
public void start() {
- String nodeLock = registryClient.getMasterStartUpLockPath();
+ String nodeLock = Constants.REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS;
try {
// create distributed lock with the root node path of the lock space as /dolphinscheduler/lock/failover/startup-masters
@@ -117,7 +117,7 @@ public class MasterRegistryClient {
// master registry
registry();
String registryPath = getMasterPath();
- registryClient.handleDeadServer(registryPath, NodeType.MASTER, Constants.DELETE_OP);
+ registryClient.handleDeadServer(Collections.singleton(registryPath), NodeType.MASTER, Constants.DELETE_OP);
// init system node
@@ -143,7 +143,8 @@ public class MasterRegistryClient {
}
public void closeRegistry() {
- unRegistry();
+ // TODO unsubscribe MasterRegistryDataListener
+ deregister();
}
/**
@@ -167,7 +168,7 @@ public class MasterRegistryClient {
return;
}
// handle dead server
- registryClient.handleDeadServer(path, nodeType, Constants.ADD_OP);
+ registryClient.handleDeadServer(Collections.singleton(path), nodeType, Constants.ADD_OP);
}
//failover server
if (failover) {
@@ -209,9 +210,9 @@ public class MasterRegistryClient {
private String getFailoverLockPath(NodeType nodeType) {
switch (nodeType) {
case MASTER:
- return registryClient.getMasterFailoverLockPath();
+ return Constants.REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS;
case WORKER:
- return registryClient.getWorkerFailoverLockPath();
+ return Constants.REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_WORKERS;
default:
return "";
}
@@ -289,20 +290,20 @@ public class MasterRegistryClient {
ProcessInstance processInstance = processService.findProcessInstanceDetailById(taskInstance.getProcessInstanceId());
if (workerHost == null
- || !checkOwner
- || processInstance.getHost().equalsIgnoreCase(getLocalAddress())) {
+ || !checkOwner
+ || processInstance.getHost().equalsIgnoreCase(getLocalAddress())) {
// only failover the task owned myself if worker down.
if (processInstance == null) {
logger.error("failover error, the process {} of task {} do not exists.",
- taskInstance.getProcessInstanceId(), taskInstance.getId());
+ taskInstance.getProcessInstanceId(), taskInstance.getId());
continue;
}
taskInstance.setProcessInstance(processInstance);
TaskExecutionContext taskExecutionContext = TaskExecutionContextBuilder.get()
- .buildTaskInstanceRelatedInfo(taskInstance)
- .buildProcessInstanceRelatedInfo(processInstance)
- .create();
+ .buildTaskInstanceRelatedInfo(taskInstance)
+ .buildProcessInstanceRelatedInfo(processInstance)
+ .create();
// only kill yarn job if exists , the local thread has exited
ProcessUtils.killYarnJob(taskExecutionContext);
@@ -348,14 +349,6 @@ public class MasterRegistryClient {
logger.info("master failover end");
}
- public void blockAcquireMutex() {
- registryClient.getLock(registryClient.getMasterLockPath());
- }
-
- public void releaseLock() {
- registryClient.releaseLock(registryClient.getMasterLockPath());
- }
-
/**
* registry
*/
@@ -364,36 +357,24 @@ public class MasterRegistryClient {
localNodePath = getMasterPath();
int masterHeartbeatInterval = masterConfig.getMasterHeartbeatInterval();
HeartBeatTask heartBeatTask = new HeartBeatTask(startupTime,
- masterConfig.getMasterMaxCpuloadAvg(),
- masterConfig.getMasterReservedMemory(),
- Sets.newHashSet(getMasterPath()),
- Constants.MASTER_TYPE,
- registryClient);
+ masterConfig.getMasterMaxCpuloadAvg(),
+ masterConfig.getMasterReservedMemory(),
+ Sets.newHashSet(getMasterPath()),
+ Constants.MASTER_TYPE,
+ registryClient);
registryClient.persistEphemeral(localNodePath, heartBeatTask.getHeartBeatInfo());
- registryClient.addConnectionStateListener(new MasterRegistryConnectStateListener());
+ registryClient.addConnectionStateListener(newState -> {
+ if (newState == ConnectionState.RECONNECTED || newState == ConnectionState.SUSPENDED) {
+ registryClient.persistEphemeral(localNodePath, "");
+ }
+ });
this.heartBeatExecutor.scheduleAtFixedRate(heartBeatTask, masterHeartbeatInterval, masterHeartbeatInterval, TimeUnit.SECONDS);
logger.info("master node : {} registry to ZK successfully with heartBeatInterval : {}s", address, masterHeartbeatInterval);
}
- class MasterRegistryConnectStateListener implements RegistryConnectListener {
-
- @Override
- public void notify(RegistryConnectState newState) {
- if (RegistryConnectState.RECONNECTED == newState) {
- registryClient.persistEphemeral(localNodePath, "");
- }
- if (RegistryConnectState.SUSPENDED == newState) {
- registryClient.persistEphemeral(localNodePath, "");
- }
- }
- }
-
- /**
- * remove registry info
- */
- public void unRegistry() {
+ public void deregister() {
try {
String address = getLocalAddress();
String localNodePath = getMasterPath();
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java
index 4fd50a0..cb5b6be 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryDataListener.java
@@ -22,42 +22,43 @@ import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHED
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.NodeType;
+import org.apache.dolphinscheduler.registry.api.Event;
+import org.apache.dolphinscheduler.registry.api.SubscribeListener;
import org.apache.dolphinscheduler.service.bean.SpringApplicationContext;
-import org.apache.dolphinscheduler.spi.register.DataChangeEvent;
-import org.apache.dolphinscheduler.spi.register.SubscribeListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Strings;
+
public class MasterRegistryDataListener implements SubscribeListener {
private static final Logger logger = LoggerFactory.getLogger(MasterRegistryDataListener.class);
- private MasterRegistryClient masterRegistryClient;
+ private final MasterRegistryClient masterRegistryClient;
public MasterRegistryDataListener() {
masterRegistryClient = SpringApplicationContext.getBean(MasterRegistryClient.class);
}
@Override
- public void notify(String path, String data, DataChangeEvent event) {
+ public void notify(Event event) {
+ final String path = event.path();
+ if (Strings.isNullOrEmpty(path)) {
+ return;
+ }
//monitor master
if (path.startsWith(REGISTRY_DOLPHINSCHEDULER_MASTERS + Constants.SINGLE_SLASH)) {
- handleMasterEvent(event, path);
+ handleMasterEvent(event);
} else if (path.startsWith(REGISTRY_DOLPHINSCHEDULER_WORKERS + Constants.SINGLE_SLASH)) {
//monitor worker
- handleWorkerEvent(event, path);
+ handleWorkerEvent(event);
}
}
- /**
- * monitor master
- *
- * @param event event
- * @param path path
- */
- public void handleMasterEvent(DataChangeEvent event, String path) {
- switch (event) {
+ public void handleMasterEvent(Event event) {
+ final String path = event.path();
+ switch (event.type()) {
case ADD:
logger.info("master node added : {}", path);
break;
@@ -69,14 +70,9 @@ public class MasterRegistryDataListener implements SubscribeListener {
}
}
- /**
- * monitor worker
- *
- * @param event event
- * @param path path
- */
- public void handleWorkerEvent(DataChangeEvent event, String path) {
- switch (event) {
+ public void handleWorkerEvent(Event event) {
+ final String path = event.path();
+ switch (event.type()) {
case ADD:
logger.info("worker node added : {}", path);
break;
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java
index 1bceeb7..b7e904b 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/ServerNodeManager.java
@@ -27,16 +27,18 @@ import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.dao.AlertDao;
import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper;
+import org.apache.dolphinscheduler.registry.api.Event;
+import org.apache.dolphinscheduler.registry.api.Event.Type;
+import org.apache.dolphinscheduler.registry.api.SubscribeListener;
import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory;
import org.apache.dolphinscheduler.service.queue.MasterPriorityQueue;
import org.apache.dolphinscheduler.service.registry.RegistryClient;
-import org.apache.dolphinscheduler.spi.register.DataChangeEvent;
-import org.apache.dolphinscheduler.spi.register.SubscribeListener;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -101,10 +103,8 @@ public class ServerNodeManager implements InitializingBean {
*/
private ScheduledExecutorService executorService;
- /**
- * zk client
- */
- private RegistryClient registryClient = RegistryClient.getInstance();
+ @Autowired
+ private RegistryClient registryClient;
/**
* eg : /node/worker/group/127.0.0.1:xxx
@@ -153,11 +153,11 @@ public class ServerNodeManager implements InitializingBean {
*/
executorService = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("ServerNodeManagerExecutor"));
executorService.scheduleWithFixedDelay(new WorkerNodeInfoAndGroupDbSyncTask(), 0, 10, TimeUnit.SECONDS);
- /**
+ /*
* init MasterNodeListener listener
*/
registryClient.subscribe(REGISTRY_DOLPHINSCHEDULER_MASTERS, new MasterDataListener());
- /**
+ /*
* init WorkerNodeListener listener
*/
registryClient.subscribe(REGISTRY_DOLPHINSCHEDULER_WORKERS, new WorkerDataListener());
@@ -167,15 +167,15 @@ public class ServerNodeManager implements InitializingBean {
* load nodes from zookeeper
*/
public void load() {
- /**
+ /*
* master nodes from zookeeper
*/
updateMasterNodes();
- /**
+ /*
* worker group nodes from zookeeper
*/
- Set<String> workerGroups = registryClient.getWorkerGroupDirectly();
+ Collection<String> workerGroups = registryClient.getWorkerGroupDirectly();
for (String workerGroup : workerGroups) {
syncWorkerGroupNodes(workerGroup, registryClient.getWorkerGroupNodesDirectly(workerGroup));
}
@@ -218,25 +218,28 @@ public class ServerNodeManager implements InitializingBean {
class WorkerDataListener implements SubscribeListener {
@Override
- public void notify(String path, String data, DataChangeEvent dataChangeEvent) {
+ public void notify(Event event) {
+ final String path = event.path();
+ final Type type = event.type();
+ final String data = event.data();
if (registryClient.isWorkerPath(path)) {
try {
- if (dataChangeEvent == DataChangeEvent.ADD) {
+ if (type == Type.ADD) {
logger.info("worker group node : {} added.", path);
String group = parseGroup(path);
- Set<String> currentNodes = registryClient.getWorkerGroupNodesDirectly(group);
+ Collection<String> currentNodes = registryClient.getWorkerGroupNodesDirectly(group);
logger.info("currentNodes : {}", currentNodes);
syncWorkerGroupNodes(group, currentNodes);
- } else if (dataChangeEvent == DataChangeEvent.REMOVE) {
+ } else if (type == Type.REMOVE) {
logger.info("worker group node : {} down.", path);
String group = parseGroup(path);
- Set<String> currentNodes = registryClient.getWorkerGroupNodesDirectly(group);
+ Collection<String> currentNodes = registryClient.getWorkerGroupNodesDirectly(group);
syncWorkerGroupNodes(group, currentNodes);
alertDao.sendServerStopedAlert(1, path, "WORKER");
- } else if (dataChangeEvent == DataChangeEvent.UPDATE) {
+ } else if (type == Type.UPDATE) {
logger.debug("worker group node : {} update, data: {}", path, data);
String group = parseGroup(path);
- Set<String> currentNodes = registryClient.getWorkerGroupNodesDirectly(group);
+ Collection<String> currentNodes = registryClient.getWorkerGroupNodesDirectly(group);
syncWorkerGroupNodes(group, currentNodes);
String node = parseNode(path);
@@ -268,19 +271,18 @@ public class ServerNodeManager implements InitializingBean {
}
}
- /**
- * master node listener
- */
class MasterDataListener implements SubscribeListener {
@Override
- public void notify(String path, String data, DataChangeEvent dataChangeEvent) {
+ public void notify(Event event) {
+ final String path = event.path();
+ final Type type = event.type();
if (registryClient.isMasterPath(path)) {
try {
- if (dataChangeEvent.equals(DataChangeEvent.ADD)) {
+ if (type.equals(Type.ADD)) {
logger.info("master node : {} added.", path);
updateMasterNodes();
}
- if (dataChangeEvent.equals(DataChangeEvent.REMOVE)) {
+ if (type.equals(Type.REMOVE)) {
logger.info("master node : {} down.", path);
updateMasterNodes();
alertDao.sendServerStopedAlert(1, path, "MASTER");
@@ -295,10 +297,10 @@ public class ServerNodeManager implements InitializingBean {
private void updateMasterNodes() {
SLOT_LIST.clear();
this.masterNodes.clear();
- String nodeLock = registryClient.getMasterLockPath();
+ String nodeLock = Constants.REGISTRY_DOLPHINSCHEDULER_LOCK_MASTERS;
try {
registryClient.getLock(nodeLock);
- Set<String> currentNodes = registryClient.getMasterNodesDirectly();
+ Collection<String> currentNodes = registryClient.getMasterNodesDirectly();
List<Server> masterNodes = registryClient.getServerList(NodeType.MASTER);
syncMasterNodes(currentNodes, masterNodes);
} catch (Exception e) {
@@ -328,7 +330,7 @@ public class ServerNodeManager implements InitializingBean {
*
* @param nodes master nodes
*/
- private void syncMasterNodes(Set<String> nodes, List<Server> masterNodes) {
+ private void syncMasterNodes(Collection<String> nodes, List<Server> masterNodes) {
masterLock.lock();
try {
this.masterNodes.addAll(nodes);
@@ -353,7 +355,7 @@ public class ServerNodeManager implements InitializingBean {
* @param workerGroup worker group
* @param nodes worker nodes
*/
- private void syncWorkerGroupNodes(String workerGroup, Set<String> nodes) {
+ private void syncWorkerGroupNodes(String workerGroup, Collection<String> nodes) {
workerGroupLock.lock();
try {
workerGroup = workerGroup.toLowerCase();
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
index f0a9562..8c19a6a 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteThread.java
@@ -17,42 +17,23 @@
package org.apache.dolphinscheduler.server.master.runner;
-import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
-import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
-import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVERY_START_NODE_STRING;
-import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_START_NODES;
-import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
-
+import com.google.common.collect.HashBasedTable;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Table;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.enums.CommandType;
-import org.apache.dolphinscheduler.common.enums.DependResult;
-import org.apache.dolphinscheduler.common.enums.Direct;
-import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.common.enums.FailureStrategy;
-import org.apache.dolphinscheduler.common.enums.Flag;
-import org.apache.dolphinscheduler.common.enums.Priority;
-import org.apache.dolphinscheduler.common.enums.StateEvent;
-import org.apache.dolphinscheduler.common.enums.StateEventType;
-import org.apache.dolphinscheduler.common.enums.TaskDependType;
-import org.apache.dolphinscheduler.common.enums.TaskTimeoutStrategy;
-import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
+import org.apache.dolphinscheduler.common.enums.*;
import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
import org.apache.dolphinscheduler.common.process.ProcessDag;
import org.apache.dolphinscheduler.common.process.Property;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.NetUtils;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
-import org.apache.dolphinscheduler.dao.entity.Environment;
-import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
-import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
-import org.apache.dolphinscheduler.dao.entity.ProjectUser;
-import org.apache.dolphinscheduler.dao.entity.Schedule;
-import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
-import org.apache.dolphinscheduler.dao.entity.TaskInstance;
+import org.apache.dolphinscheduler.dao.entity.*;
import org.apache.dolphinscheduler.dao.utils.DagHelper;
import org.apache.dolphinscheduler.remote.command.HostUpdateCommand;
import org.apache.dolphinscheduler.remote.utils.Host;
@@ -65,29 +46,14 @@ import org.apache.dolphinscheduler.service.alert.ProcessAlertManager;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
import org.apache.dolphinscheduler.service.queue.PeerTaskInstancePriorityQueue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.lang.StringUtils;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
+import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import com.google.common.collect.HashBasedTable;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Table;
+import static org.apache.dolphinscheduler.common.Constants.*;
/**
* master exec thread,split dag
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
index 4afa2cc..fb14d96 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
@@ -29,8 +29,6 @@ import org.apache.dolphinscheduler.common.task.sql.SqlParameters;
import org.apache.dolphinscheduler.common.task.sqoop.SqoopParameters;
import org.apache.dolphinscheduler.common.task.sqoop.sources.SourceMysqlParameter;
import org.apache.dolphinscheduler.common.task.sqoop.targets.TargetMysqlParameter;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
-import org.apache.dolphinscheduler.common.utils.EnumUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
@@ -51,6 +49,7 @@ import org.apache.dolphinscheduler.spi.task.request.SQLTaskExecutionContext;
import org.apache.dolphinscheduler.spi.task.request.SqoopTaskExecutionContext;
import org.apache.dolphinscheduler.spi.task.request.UdfFuncRequest;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import java.util.HashMap;
@@ -63,6 +62,9 @@ import java.util.stream.Stream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Enums;
+import com.google.common.base.Strings;
+
public abstract class BaseTaskProcessor implements ITaskProcessor {
protected Logger logger = LoggerFactory.getLogger(getClass());
@@ -300,8 +302,8 @@ public abstract class BaseTaskProcessor implements ITaskProcessor {
sqlTaskExecutionContext.setDefaultFS(HadoopUtils.getInstance().getDefaultFS());
// whether udf type
- boolean udfTypeFlag = EnumUtils.isValidEnum(UdfType.class, sqlParameters.getType())
- && !StringUtils.isEmpty(sqlParameters.getUdfs());
+ boolean udfTypeFlag = Enums.getIfPresent(UdfType.class, Strings.nullToEmpty(sqlParameters.getType())).isPresent()
+ && !StringUtils.isEmpty(sqlParameters.getUdfs());
if (udfTypeFlag) {
String[] udfFunIds = sqlParameters.getUdfs().split(",");
@@ -371,4 +373,4 @@ public abstract class BaseTaskProcessor implements ITaskProcessor {
return resourcesMap;
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/AbstractMonitor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/AbstractMonitor.java
index 5e31343..2640d8d6 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/AbstractMonitor.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/AbstractMonitor.java
@@ -16,8 +16,7 @@
*/
package org.apache.dolphinscheduler.server.monitor;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
-
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/RegistryMonitorImpl.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/RegistryMonitorImpl.java
index 34d6d9d..9e3bb0c 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/RegistryMonitorImpl.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/monitor/RegistryMonitorImpl.java
@@ -19,36 +19,26 @@ package org.apache.dolphinscheduler.server.monitor;
import org.apache.dolphinscheduler.service.registry.RegistryClient;
+import java.util.Collection;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
-/**
- * zk monitor server impl
- */
@Component
public class RegistryMonitorImpl extends AbstractMonitor {
- /**
- * zookeeper operator
- */
- private RegistryClient registryClient = RegistryClient.getInstance();
-
- /**
- * get active nodes map by path
- *
- * @param path path
- * @return active nodes map
- */
+ @Autowired
+ private RegistryClient registryClient;
+
@Override
protected Map<String, String> getActiveNodesByPath(String path) {
Map<String, String> maps = new HashMap<>();
- List<String> childrenList = registryClient.getChildrenKeys(path);
+ Collection<String> childrenList = registryClient.getChildrenKeys(path);
if (childrenList == null) {
return maps;
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
index 61e8c40..67fd07a 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/registry/HeartBeatTask.java
@@ -33,11 +33,11 @@ public class HeartBeatTask implements Runnable {
private final Logger logger = LoggerFactory.getLogger(HeartBeatTask.class);
- private Set<String> heartBeatPaths;
- private RegistryClient registryClient;
+ private final Set<String> heartBeatPaths;
+ private final RegistryClient registryClient;
private WorkerManagerThread workerManagerThread;
- private String serverType;
- private HeartBeat heartBeat;
+ private final String serverType;
+ private final HeartBeat heartBeat;
public HeartBeatTask(long startupTime,
double maxCpuloadAvg,
@@ -89,7 +89,7 @@ public class HeartBeatTask implements Runnable {
}
for (String heartBeatPath : heartBeatPaths) {
- registryClient.update(heartBeatPath, heartBeat.encodeHeartBeat());
+ registryClient.persistEphemeral(heartBeatPath, heartBeat.encodeHeartBeat());
}
} catch (Throwable ex) {
logger.error("error write heartbeat info", ex);
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ArgsUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ArgsUtils.java
deleted file mode 100644
index d71eb54..0000000
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ArgsUtils.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.server.utils;
-
-public class ArgsUtils {
-
- private ArgsUtils() throws IllegalStateException {
- throw new IllegalStateException("Utility class");
- }
-
- public static String escape(String arg) {
- return arg.replace(" ", "\\ ").replace("\"", "\\\"").replace("'", "\\'");
- }
-
-}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
index 3ca9120..063b153 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ProcessUtils.java
@@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.server.utils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.HadoopUtils;
@@ -30,7 +29,9 @@ import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.dolphinscheduler.service.queue.entity.TaskExecutionContext;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.SystemUtils;
import java.io.File;
import java.nio.charset.StandardCharsets;
@@ -153,7 +154,7 @@ public class ProcessUtils {
List<String> pidList = new ArrayList<>();
Matcher mat = null;
// pstree pid get sub pids
- if (OSUtils.isMacOS()) {
+ if (SystemUtils.IS_OS_MAC) {
String pids = OSUtils.exeCmd(String.format("%s -sp %d", Constants.PSTREE, processId));
if (null != pids) {
mat = MACPATTERN.matcher(pids);
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
index fa186d0..96ec36b 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackService.java
@@ -52,17 +52,11 @@ public class TaskCallbackService {
private static final ConcurrentHashMap<Integer, NettyRemoteChannel> REMOTE_CHANNELS = new ConcurrentHashMap<>();
/**
- * zookeeper registry center
- */
- private RegistryClient registryClient;
-
- /**
* netty remoting client
*/
private final NettyRemotingClient nettyRemotingClient;
public TaskCallbackService() {
- this.registryClient = RegistryClient.getInstance();
final NettyClientConfig clientConfig = new NettyClientConfig();
this.nettyRemotingClient = new NettyRemotingClient(clientConfig);
this.nettyRemotingClient.registerProcessor(CommandType.DB_TASK_ACK, new DBTaskAckProcessor());
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
index 76d70a4..8721152 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java
@@ -148,11 +148,9 @@ public class TaskExecuteProcessor implements NettyRequestProcessor {
OSUtils.createUserIfAbsent(taskExecutionContext.getTenantCode());
}
} catch (Throwable ex) {
- String errorLog = String.format("create execLocalPath : %s", execLocalPath);
- LoggerUtils.logError(Optional.of(logger), errorLog, ex);
+ logger.error("create execLocalPath: {}", execLocalPath, ex);
TaskExecutionContextCacheManager.removeByTaskInstanceId(taskExecutionContext.getTaskInstanceId());
}
- FileUtils.taskLoggerThreadLocal.remove();
taskCallbackService.addRemoteChannel(taskExecutionContext.getTaskInstanceId(),
new NettyRemoteChannel(channel, command.getOpaque()));
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
index b59e3ec..54ec49c 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryClient.java
@@ -33,6 +33,7 @@ import org.apache.dolphinscheduler.service.registry.RegistryClient;
import org.apache.commons.lang.StringUtils;
+import java.io.IOException;
import java.util.Set;
import java.util.StringJoiner;
import java.util.concurrent.Executors;
@@ -73,6 +74,7 @@ public class WorkerRegistryClient {
*/
private ScheduledExecutorService heartBeatExecutor;
+ @Autowired
private RegistryClient registryClient;
/**
@@ -86,7 +88,6 @@ public class WorkerRegistryClient {
public void initWorkRegistry() {
this.workerGroups = workerConfig.getWorkerGroups();
this.startupTime = System.currentTimeMillis();
- this.registryClient = RegistryClient.getInstance();
this.heartBeatExecutor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("HeartBeatExecutor"));
}
@@ -121,7 +122,7 @@ public class WorkerRegistryClient {
/**
* remove registry info
*/
- public void unRegistry() {
+ public void unRegistry() throws IOException {
try {
String address = getLocalAddress();
Set<String> workerZkPaths = getWorkerZkPaths();
@@ -161,7 +162,7 @@ public class WorkerRegistryClient {
return workerPaths;
}
- public void handleDeadServer(Set<String> nodeSet, NodeType nodeType, String opType) throws Exception {
+ public void handleDeadServer(Set<String> nodeSet, NodeType nodeType, String opType) {
registryClient.handleDeadServer(nodeSet, nodeType, opType);
}
diff --git a/dolphinscheduler-server/src/main/resources/config/install_config.conf b/dolphinscheduler-server/src/main/resources/config/install_config.conf
index 2e74b1e..96174d9 100755
--- a/dolphinscheduler-server/src/main/resources/config/install_config.conf
+++ b/dolphinscheduler-server/src/main/resources/config/install_config.conf
@@ -93,11 +93,6 @@ dbname="dolphinscheduler"
# ---------------------------------------------------------
# Registry Server
# ---------------------------------------------------------
-# Registry Server plugin dir. DolphinScheduler will find and load the registry plugin jar package from this dir.
-# For now default registry server is zookeeper, so the default value is `lib/plugin/registry/zookeeper`.
-# If you want to implement your own registry server, please see https://dolphinscheduler.apache.org/en-us/docs/dev/user_doc/registry_spi.html
-registryPluginDir="lib/plugin/registry/zookeeper"
-
# Registry Server plugin name, should be a substring of `registryPluginDir`, DolphinScheduler use this for verifying configuration consistency
registryPluginName="zookeeper"
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java
index fbacf0f..a8733f2 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java
@@ -17,23 +17,16 @@
package org.apache.dolphinscheduler.server.log;
+import static org.apache.dolphinscheduler.server.log.SensitiveDataConverter.passwordHandler;
+
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.utils.SensitiveLogUtils;
-import java.util.Map;
-import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.slf4j.Marker;
-
-import ch.qos.logback.classic.Level;
-import ch.qos.logback.classic.spi.ILoggingEvent;
-import ch.qos.logback.classic.spi.IThrowableProxy;
-import ch.qos.logback.classic.spi.LoggerContextVO;
public class SensitiveDataConverterTest {
@@ -58,86 +51,7 @@ public class SensitiveDataConverterTest {
@Test
public void convert() {
- SensitiveDataConverter sensitiveDataConverter = new SensitiveDataConverter();
- String result = sensitiveDataConverter.convert(new ILoggingEvent() {
- @Override
- public String getThreadName() {
- return null;
- }
-
- @Override
- public Level getLevel() {
- return Level.INFO;
- }
-
- @Override
- public String getMessage() {
- return null;
- }
-
- @Override
- public Object[] getArgumentArray() {
- return new Object[0];
- }
-
- @Override
- public String getFormattedMessage() {
- return logMsg;
- }
-
- @Override
- public String getLoggerName() {
- return null;
- }
-
- @Override
- public LoggerContextVO getLoggerContextVO() {
- return null;
- }
-
- @Override
- public IThrowableProxy getThrowableProxy() {
- return null;
- }
-
- @Override
- public StackTraceElement[] getCallerData() {
- return new StackTraceElement[0];
- }
-
- @Override
- public boolean hasCallerData() {
- return false;
- }
-
- @Override
- public Marker getMarker() {
- return null;
- }
-
- @Override
- public Map<String, String> getMDCPropertyMap() {
- return null;
- }
-
- @Override
- public Map<String, String> getMdc() {
- return null;
- }
-
- @Override
- public long getTimeStamp() {
- return 0;
- }
-
- @Override
- public void prepareForDeferredProcessing() {
-
- }
- });
-
Assert.assertNotEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg));
-
}
/**
@@ -153,28 +67,4 @@ public class SensitiveDataConverterTest {
}
- /**
- * password regex test
- *
- * @param logMsg original log
- */
- private static String passwordHandler(Pattern pattern, String logMsg) {
-
- Matcher matcher = pattern.matcher(logMsg);
-
- StringBuffer sb = new StringBuffer(logMsg.length());
-
- while (matcher.find()) {
-
- String password = matcher.group();
-
- String maskPassword = SensitiveLogUtils.maskDataSourcePwd(password);
-
- matcher.appendReplacement(sb, maskPassword);
- }
- matcher.appendTail(sb);
-
- return sb.toString();
- }
-
}
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java
index 80f75af..619dba8 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/ExecutorDispatcherTest.java
@@ -56,7 +56,7 @@ public class ExecutorDispatcherTest {
}
@Test
- public void testDispatch() throws ExecuteException {
+ public void testDispatch() throws Exception {
int port = 30000;
final NettyServerConfig serverConfig = new NettyServerConfig();
serverConfig.setListenPort(port);
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java
index a91c6e2..c643818 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistryClientTest.java
@@ -20,10 +20,6 @@ package org.apache.dolphinscheduler.server.master.registry;
import static org.mockito.BDDMockito.given;
import static org.mockito.Mockito.doNothing;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.concurrent.ScheduledExecutorService;
-
import org.apache.dolphinscheduler.common.enums.CommandType;
import org.apache.dolphinscheduler.common.enums.NodeType;
import org.apache.dolphinscheduler.common.model.Server;
@@ -32,13 +28,17 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.server.master.config.MasterConfig;
import org.apache.dolphinscheduler.service.process.ProcessService;
import org.apache.dolphinscheduler.service.registry.RegistryClient;
+
+import java.util.Arrays;
+import java.util.Date;
+import java.util.concurrent.ScheduledExecutorService;
+
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
-import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
@@ -58,6 +58,7 @@ public class MasterRegistryClientTest {
@Mock
private MasterConfig masterConfig;
+ @Mock
private RegistryClient registryClient;
@Mock
@@ -68,13 +69,10 @@ public class MasterRegistryClientTest {
@Before
public void before() throws Exception {
- PowerMockito.suppress(PowerMockito.constructor(RegistryClient.class));
- registryClient = PowerMockito.mock(RegistryClient.class);
given(registryClient.getLock(Mockito.anyString())).willReturn(true);
- given(registryClient.getMasterFailoverLockPath()).willReturn("/path");
given(registryClient.releaseLock(Mockito.anyString())).willReturn(true);
given(registryClient.getHostByEventDataPath(Mockito.anyString())).willReturn("127.0.0.1:8080");
- doNothing().when(registryClient).handleDeadServer(Mockito.anyString(), Mockito.any(NodeType.class), Mockito.anyString());
+ doNothing().when(registryClient).handleDeadServer(Mockito.anySet(), Mockito.any(NodeType.class), Mockito.anyString());
ReflectionTestUtils.setField(masterRegistryClient, "registryClient", registryClient);
ProcessInstance processInstance = new ProcessInstance();
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java
index d545c02..6631d3d 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ProcessUtilsTest.java
@@ -25,6 +25,8 @@ import org.apache.dolphinscheduler.common.utils.HadoopUtils;
import org.apache.dolphinscheduler.common.utils.OSUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
+import org.apache.commons.lang.SystemUtils;
+
import java.util.ArrayList;
import java.util.List;
@@ -36,11 +38,12 @@ import org.mockito.MockitoAnnotations;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
+import org.powermock.reflect.Whitebox;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@RunWith(PowerMockRunner.class)
-@PrepareForTest({System.class, OSUtils.class, HadoopUtils.class, PropertyUtils.class})
+@PrepareForTest({System.class, OSUtils.class, HadoopUtils.class, PropertyUtils.class, SystemUtils.class})
public class ProcessUtilsTest {
private static final Logger logger = LoggerFactory.getLogger(ProcessUtils.class);
@@ -53,11 +56,8 @@ public class ProcessUtilsTest {
@Test
public void getPidsStr() throws Exception {
int processId = 1;
- String pidList = ProcessUtils.getPidsStr(processId);
- Assert.assertNotEquals("The child process of process 1 should not be empty", pidList, "");
-
PowerMockito.mockStatic(OSUtils.class);
- when(OSUtils.isMacOS()).thenReturn(true);
+ Whitebox.setInternalState(SystemUtils.class, "IS_OS_MAC", true);
when(OSUtils.exeCmd(String.format("%s -p %d", Constants.PSTREE, processId))).thenReturn(null);
String pidListMac = ProcessUtils.getPidsStr(processId);
Assert.assertEquals("", pidListMac);
diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml
index 015b3a5..d339f7f 100644
--- a/dolphinscheduler-service/pom.xml
+++ b/dolphinscheduler-service/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -41,6 +41,14 @@
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-spi</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-registry-zookeeper</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-common</artifactId>
+ </dependency>
<dependency>
<groupId>org.quartz-scheduler</groupId>
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java
index f207a22..f682780 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java
@@ -149,7 +149,6 @@ public class LogClientService implements AutoCloseable {
public byte[] getLogBytes(String host, int port, String path) {
logger.info("log path {}", path);
GetLogBytesRequestCommand request = new GetLogBytesRequestCommand(path);
- byte[] result = null;
final Host address = new Host(host, port);
try {
Command command = request.convert2Command();
@@ -157,14 +156,14 @@ public class LogClientService implements AutoCloseable {
if (response != null) {
GetLogBytesResponseCommand getLog = JSONUtils.parseObject(
response.getBody(), GetLogBytesResponseCommand.class);
- return getLog.getData();
+ return getLog.getData() == null ? new byte[0] : getLog.getData();
}
} catch (Exception e) {
logger.error("get log size error", e);
} finally {
this.client.closeChannel(address);
}
- return result;
+ return new byte[0];
}
/**
@@ -199,4 +198,4 @@ public class LogClientService implements AutoCloseable {
public boolean isRunning() {
return isRunning;
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java
index a8f73f0..491c191 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/permission/PermissionCheck.java
@@ -20,11 +20,12 @@ package org.apache.dolphinscheduler.service.permission;
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.service.exceptions.ServiceException;
import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.commons.collections.CollectionUtils;
+
import java.util.List;
import org.slf4j.Logger;
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
index d054555..198af0b 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java
@@ -17,29 +17,13 @@
package org.apache.dolphinscheduler.service.process;
-import static java.util.stream.Collectors.toSet;
-import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_END_DATE;
-import static org.apache.dolphinscheduler.common.Constants.CMDPARAM_COMPLEMENT_DATA_START_DATE;
-import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_EMPTY_SUB_PROCESS;
-import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_FATHER_PARAMS;
-import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_RECOVER_PROCESS_ID_STRING;
-import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS;
-import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_DEFINE_CODE;
-import static org.apache.dolphinscheduler.common.Constants.CMD_PARAM_SUB_PROCESS_PARENT_INSTANCE_ID;
-import static org.apache.dolphinscheduler.common.Constants.LOCAL_PARAMS;
-
+import com.facebook.presto.jdbc.internal.guava.collect.Lists;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.enums.AuthorizationType;
-import org.apache.dolphinscheduler.common.enums.CommandType;
-import org.apache.dolphinscheduler.common.enums.Direct;
-import org.apache.dolphinscheduler.common.enums.ExecutionStatus;
-import org.apache.dolphinscheduler.common.enums.FailureStrategy;
-import org.apache.dolphinscheduler.common.enums.Flag;
-import org.apache.dolphinscheduler.common.enums.ReleaseState;
-import org.apache.dolphinscheduler.spi.enums.ResourceType;
-import org.apache.dolphinscheduler.common.enums.TaskDependType;
-import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
-import org.apache.dolphinscheduler.common.enums.WarningType;
+import org.apache.dolphinscheduler.common.enums.*;
import org.apache.dolphinscheduler.common.graph.DAG;
import org.apache.dolphinscheduler.common.model.DateInterval;
import org.apache.dolphinscheduler.common.model.TaskNode;
@@ -50,83 +34,27 @@ import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.AbstractParameters;
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters;
-import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
+import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
-import org.apache.dolphinscheduler.common.utils.DateUtils;
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.common.utils.ParameterUtils;
-import org.apache.dolphinscheduler.common.utils.TaskParametersUtils;
-import org.apache.dolphinscheduler.dao.entity.Command;
-import org.apache.dolphinscheduler.dao.entity.DagData;
-import org.apache.dolphinscheduler.dao.entity.DataSource;
-import org.apache.dolphinscheduler.dao.entity.Environment;
-import org.apache.dolphinscheduler.dao.entity.ErrorCommand;
-import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
-import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
-import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
-import org.apache.dolphinscheduler.dao.entity.ProcessInstanceMap;
-import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
-import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
-import org.apache.dolphinscheduler.dao.entity.Project;
-import org.apache.dolphinscheduler.dao.entity.ProjectUser;
-import org.apache.dolphinscheduler.dao.entity.Resource;
-import org.apache.dolphinscheduler.dao.entity.Schedule;
-import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
-import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
-import org.apache.dolphinscheduler.dao.entity.TaskInstance;
-import org.apache.dolphinscheduler.dao.entity.Tenant;
-import org.apache.dolphinscheduler.dao.entity.UdfFunc;
-import org.apache.dolphinscheduler.dao.entity.User;
-import org.apache.dolphinscheduler.dao.mapper.CommandMapper;
-import org.apache.dolphinscheduler.dao.mapper.DataSourceMapper;
-import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper;
-import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper;
-import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionLogMapper;
-import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
-import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper;
-import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
-import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationLogMapper;
-import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper;
-import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
-import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
-import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
-import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
-import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
-import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
-import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
-import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
-import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
-import org.apache.dolphinscheduler.dao.mapper.UserMapper;
+import org.apache.dolphinscheduler.dao.entity.*;
+import org.apache.dolphinscheduler.dao.mapper.*;
import org.apache.dolphinscheduler.dao.utils.DagHelper;
import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.service.log.LogClientService;
import org.apache.dolphinscheduler.service.quartz.cron.CronUtils;
-
-import org.apache.commons.lang.StringUtils;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.EnumMap;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Objects;
-import java.util.Set;
-import java.util.stream.Collectors;
-
+import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
-import com.facebook.presto.jdbc.internal.guava.collect.Lists;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.node.ObjectNode;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.toSet;
+import static org.apache.dolphinscheduler.common.Constants.*;
/**
* process relative dao that some mappers in this.
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java
index f195d62..3e7007a 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/quartz/cron/CronUtils.java
@@ -28,10 +28,11 @@ import static com.cronutils.model.CronType.QUARTZ;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.CycleEnum;
import org.apache.dolphinscheduler.common.thread.Stopper;
-import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.entity.Schedule;
+import org.apache.commons.collections.CollectionUtils;
+
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Calendar;
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryCenter.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryCenter.java
deleted file mode 100644
index ba74f88..0000000
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryCenter.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.service.registry;
-
-import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_DEAD_SERVERS;
-import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_MASTERS;
-import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_WORKERS;
-
-import org.apache.dolphinscheduler.common.IStoppable;
-import org.apache.dolphinscheduler.common.utils.PropertyUtils;
-import org.apache.dolphinscheduler.spi.plugin.DolphinPluginLoader;
-import org.apache.dolphinscheduler.spi.plugin.DolphinPluginManagerConfig;
-import org.apache.dolphinscheduler.spi.register.Registry;
-import org.apache.dolphinscheduler.spi.register.RegistryConnectListener;
-import org.apache.dolphinscheduler.spi.register.RegistryException;
-import org.apache.dolphinscheduler.spi.register.RegistryPluginManager;
-import org.apache.dolphinscheduler.spi.register.SubscribeListener;
-
-import org.apache.commons.lang.StringUtils;
-
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.ImmutableList;
-
-/**
- * All business parties use this class to access the registry
- */
-public class RegistryCenter {
-
- private static final Logger logger = LoggerFactory.getLogger(RegistryCenter.class);
-
- private final AtomicBoolean isStarted = new AtomicBoolean(false);
-
- private Registry registry;
-
- private IStoppable stoppable;
-
- /**
- * nodes namespace
- */
- protected static String NODES;
-
- private RegistryPluginManager registryPluginManager;
-
- protected static final String EMPTY = "";
-
- private static final String REGISTRY_PREFIX = "registry";
-
- private static final String REGISTRY_PLUGIN_BINDING = "registry.plugin.binding";
-
- private static final String REGISTRY_PLUGIN_DIR = "registry.plugin.dir";
-
- private static final String MAVEN_LOCAL_REPOSITORY = "maven.local.repository";
-
- private static final String REGISTRY_PLUGIN_NAME = "plugin.name";
-
- /**
- * default registry plugin dir
- */
- private static final String REGISTRY_PLUGIN_PATH = "lib/plugin/registry";
-
- private static final String REGISTRY_CONFIG_FILE_PATH = "/registry.properties";
-
- /**
- * init node persist
- */
- public void init() {
- if (isStarted.compareAndSet(false, true)) {
- PropertyUtils.loadPropertyFile(REGISTRY_CONFIG_FILE_PATH);
- Map<String, String> registryConfig = PropertyUtils.getPropertiesByPrefix(REGISTRY_PREFIX);
-
- if (null == registryConfig || registryConfig.isEmpty()) {
- throw new RegistryException("registry config param is null");
- }
- if (null == registryPluginManager) {
- installRegistryPlugin(registryConfig.get(REGISTRY_PLUGIN_NAME));
- registry = registryPluginManager.getRegistry();
- }
-
- registry.init(registryConfig);
- initNodes();
-
- }
- }
-
- /**
- * init nodes
- */
- private void initNodes() {
- persist(REGISTRY_DOLPHINSCHEDULER_MASTERS, EMPTY);
- persist(REGISTRY_DOLPHINSCHEDULER_WORKERS, EMPTY);
- persist(REGISTRY_DOLPHINSCHEDULER_DEAD_SERVERS, EMPTY);
- }
-
- /**
- * install registry plugin
- */
- private void installRegistryPlugin(String registryPluginName) {
- DolphinPluginManagerConfig registryPluginManagerConfig = new DolphinPluginManagerConfig();
- registryPluginManagerConfig.setPlugins(PropertyUtils.getString(REGISTRY_PLUGIN_BINDING));
- if (StringUtils.isNotBlank(PropertyUtils.getString(REGISTRY_PLUGIN_DIR))) {
- registryPluginManagerConfig.setInstalledPluginsDir(PropertyUtils.getString(REGISTRY_PLUGIN_DIR, REGISTRY_PLUGIN_PATH).trim());
- }
-
- if (StringUtils.isNotBlank(PropertyUtils.getString(MAVEN_LOCAL_REPOSITORY))) {
- registryPluginManagerConfig.setMavenLocalRepository(PropertyUtils.getString(MAVEN_LOCAL_REPOSITORY).trim());
- }
-
- registryPluginManager = new RegistryPluginManager(registryPluginName);
-
- DolphinPluginLoader registryPluginLoader = new DolphinPluginLoader(registryPluginManagerConfig, ImmutableList.of(registryPluginManager));
- try {
- registryPluginLoader.loadPlugins();
- } catch (Exception e) {
- throw new RuntimeException("Load registry Plugin Failed !", e);
- }
- }
-
- /**
- * close
- */
- public void close() {
- if (isStarted.compareAndSet(true, false) && registry != null) {
- registry.close();
- }
- }
-
- public void persist(String key, String value) {
- registry.persist(key, value);
- }
-
- public void persistEphemeral(String key, String value) {
- registry.persistEphemeral(key, value);
- }
-
- public void remove(String key) {
- registry.remove(key);
- }
-
- public void update(String key, String value) {
- registry.update(key, value);
- }
-
- public String get(String key) {
- return registry.get(key);
- }
-
- public void subscribe(String path, SubscribeListener subscribeListener) {
- registry.subscribe(path, subscribeListener);
- }
-
- public void addConnectionStateListener(RegistryConnectListener registryConnectListener) {
- registry.addConnectionStateListener(registryConnectListener);
- }
-
- public boolean isExisted(String key) {
- return registry.isExisted(key);
- }
-
- public boolean getLock(String key) {
- return registry.acquireLock(key);
- }
-
- public boolean releaseLock(String key) {
- return registry.releaseLock(key);
- }
-
- /**
- * @return get dead server node parent path
- */
- public String getDeadZNodeParentPath() {
- return REGISTRY_DOLPHINSCHEDULER_DEAD_SERVERS;
- }
-
- public void setStoppable(IStoppable stoppable) {
- this.stoppable = stoppable;
- }
-
- public IStoppable getStoppable() {
- return stoppable;
- }
-
- /**
- * whether master path
- *
- * @param path path
- * @return result
- */
- public boolean isMasterPath(String path) {
- return path != null && path.contains(REGISTRY_DOLPHINSCHEDULER_MASTERS);
- }
-
- /**
- * get worker group path
- *
- * @param workerGroup workerGroup
- * @return worker group path
- */
- public String getWorkerGroupPath(String workerGroup) {
- return REGISTRY_DOLPHINSCHEDULER_WORKERS + "/" + workerGroup;
- }
-
- /**
- * whether worker path
- *
- * @param path path
- * @return result
- */
- public boolean isWorkerPath(String path) {
- return path != null && path.contains(REGISTRY_DOLPHINSCHEDULER_WORKERS);
- }
-
- /**
- * get children nodes
- *
- * @param key key
- * @return children nodes
- */
- public List<String> getChildrenKeys(final String key) {
- return registry.getChildren(key);
- }
-
-}
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java
index 210ce21..f384678 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/registry/RegistryClient.java
@@ -22,60 +22,72 @@ import static org.apache.dolphinscheduler.common.Constants.COLON;
import static org.apache.dolphinscheduler.common.Constants.DELETE_OP;
import static org.apache.dolphinscheduler.common.Constants.DIVISION_STRING;
import static org.apache.dolphinscheduler.common.Constants.MASTER_TYPE;
+import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_DEAD_SERVERS;
import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_MASTERS;
import static org.apache.dolphinscheduler.common.Constants.REGISTRY_DOLPHINSCHEDULER_WORKERS;
import static org.apache.dolphinscheduler.common.Constants.SINGLE_SLASH;
import static org.apache.dolphinscheduler.common.Constants.UNDERLINE;
import static org.apache.dolphinscheduler.common.Constants.WORKER_TYPE;
+import static com.google.common.base.Preconditions.checkArgument;
+
import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.IStoppable;
import org.apache.dolphinscheduler.common.enums.NodeType;
import org.apache.dolphinscheduler.common.model.Server;
import org.apache.dolphinscheduler.common.utils.HeartBeat;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-
-import org.apache.commons.lang.StringUtils;
-
+import org.apache.dolphinscheduler.common.utils.PropertyUtils;
+import org.apache.dolphinscheduler.registry.api.ConnectionListener;
+import org.apache.dolphinscheduler.registry.api.Registry;
+import org.apache.dolphinscheduler.registry.api.RegistryException;
+import org.apache.dolphinscheduler.registry.api.RegistryFactory;
+import org.apache.dolphinscheduler.registry.api.RegistryFactoryLoader;
+import org.apache.dolphinscheduler.registry.api.SubscribeListener;
+
+import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collections;
+import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+
+import javax.annotation.PostConstruct;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
-/**
- * registry client singleton
- */
-public class RegistryClient extends RegistryCenter {
+import com.google.common.base.Strings;
+@Component
+public class RegistryClient {
private static final Logger logger = LoggerFactory.getLogger(RegistryClient.class);
- private static RegistryClient registryClient = new RegistryClient();
-
- private RegistryClient() {
- super.init();
- }
-
- public static RegistryClient getInstance() {
- return registryClient;
+ private static final String EMPTY = "";
+ private static final String REGISTRY_PREFIX = "registry";
+ private static final String REGISTRY_PLUGIN_NAME = "plugin.name";
+ private static final String REGISTRY_CONFIG_FILE_PATH = "/registry.properties";
+ private final AtomicBoolean isStarted = new AtomicBoolean(false);
+ private Registry registry;
+ private IStoppable stoppable;
+
+ @PostConstruct
+ public void afterConstruct() {
+ start();
+ initNodes();
}
- /**
- * get active master num
- *
- * @return active master number
- */
public int getActiveMasterNum() {
- List<String> childrenList = new ArrayList<>();
+ Collection<String> childrenList = new ArrayList<>();
try {
// read master node parent path from conf
- if (isExisted(getNodeParentPath(NodeType.MASTER))) {
- childrenList = getChildrenKeys(getNodeParentPath(NodeType.MASTER));
+ if (exists(rootNodePath(NodeType.MASTER))) {
+ childrenList = getChildrenKeys(rootNodePath(NodeType.MASTER));
}
} catch (Exception e) {
logger.error("getActiveMasterNum error", e);
@@ -83,15 +95,9 @@ public class RegistryClient extends RegistryCenter {
return childrenList.size();
}
- /**
- * get server list.
- *
- * @param nodeType zookeeper node type
- * @return server list
- */
public List<Server> getServerList(NodeType nodeType) {
- Map<String, String> serverMaps = getServerMaps(nodeType);
- String parentPath = getNodeParentPath(nodeType);
+ Map<String, String> serverMaps = getServerMaps(nodeType, false);
+ String parentPath = rootNodePath(nodeType);
List<Server> serverList = new ArrayList<>();
for (Map.Entry<String, String> entry : serverMaps.entrySet()) {
@@ -119,40 +125,11 @@ public class RegistryClient extends RegistryCenter {
return serverList;
}
- /**
- * get server nodes.
- *
- * @param nodeType registry node type
- * @return result : list<node>
- */
- public List<String> getServerNodes(NodeType nodeType) {
- String path = getNodeParentPath(nodeType);
- List<String> serverList = getChildrenKeys(path);
- if (nodeType == NodeType.WORKER) {
- List<String> workerList = new ArrayList<>();
- for (String group : serverList) {
- List<String> groupServers = getChildrenKeys(path + SINGLE_SLASH + group);
- for (String groupServer : groupServers) {
- workerList.add(group + SINGLE_SLASH + groupServer);
- }
- }
- serverList = workerList;
- }
- return serverList;
- }
-
- /**
- * get server list map.
- *
- * @param nodeType zookeeper node type
- * @param hostOnly host only
- * @return result : {host : resource info}
- */
public Map<String, String> getServerMaps(NodeType nodeType, boolean hostOnly) {
Map<String, String> serverMap = new HashMap<>();
try {
- String path = getNodeParentPath(nodeType);
- List<String> serverList = getServerNodes(nodeType);
+ String path = rootNodePath(nodeType);
+ Collection<String> serverList = getServerNodes(nodeType);
for (String server : serverList) {
String host = server;
if (nodeType == NodeType.WORKER && hostOnly) {
@@ -167,298 +144,194 @@ public class RegistryClient extends RegistryCenter {
return serverMap;
}
- /**
- * get server list map.
- *
- * @param nodeType zookeeper node type
- * @return result : {host : resource info}
- */
- public Map<String, String> getServerMaps(NodeType nodeType) {
- return getServerMaps(nodeType, false);
+ public boolean checkNodeExists(String host, NodeType nodeType) {
+ return getServerMaps(nodeType, true).keySet()
+ .stream()
+ .anyMatch(it -> it.contains(host));
}
- /**
- * get server node set.
- *
- * @param nodeType zookeeper node type
- * @param hostOnly host only
- * @return result : set<host>
- */
- public Set<String> getServerNodeSet(NodeType nodeType, boolean hostOnly) {
- Set<String> serverSet = new HashSet<>();
- try {
- List<String> serverList = getServerNodes(nodeType);
- for (String server : serverList) {
- String host = server;
- if (nodeType == NodeType.WORKER && hostOnly) {
- host = server.split(SINGLE_SLASH)[1];
- }
- serverSet.add(host);
+ public void handleDeadServer(Collection<String> nodes, NodeType nodeType, String opType) {
+ nodes.forEach(node -> {
+ final String host = getHostByEventDataPath(node);
+ final String type = nodeType == NodeType.MASTER ? MASTER_TYPE : WORKER_TYPE;
+
+ if (opType.equals(DELETE_OP)) {
+ removeDeadServerByHost(host, type);
+ } else if (opType.equals(ADD_OP)) {
+ String deadServerPath = REGISTRY_DOLPHINSCHEDULER_DEAD_SERVERS + SINGLE_SLASH + type + UNDERLINE + host;
+ // Add dead server info to zk dead server path : /dead-servers/
+ registry.put(deadServerPath, type + UNDERLINE + host, false);
+ logger.info("{} server dead , and {} added to zk dead server path success", nodeType, node);
}
- } catch (Exception e) {
- logger.error("get server node set failed", e);
- }
- return serverSet;
+ });
}
- /**
- * get server node list.
- *
- * @param nodeType zookeeper node type
- * @param hostOnly host only
- * @return result : list<host>
- */
- public List<String> getServerNodeList(NodeType nodeType, boolean hostOnly) {
- Set<String> serverSet = getServerNodeSet(nodeType, hostOnly);
- List<String> serverList = new ArrayList<>(serverSet);
- Collections.sort(serverList);
- return serverList;
- }
+ public boolean checkIsDeadServer(String node, String serverType) {
+ // ip_sequence_no
+ String[] zNodesPath = node.split("/");
+ String ipSeqNo = zNodesPath[zNodesPath.length - 1];
+ String deadServerPath = REGISTRY_DOLPHINSCHEDULER_DEAD_SERVERS + SINGLE_SLASH + serverType + UNDERLINE + ipSeqNo;
- /**
- * check the zookeeper node already exists
- *
- * @param host host
- * @param nodeType zookeeper node type
- * @return true if exists
- */
- public boolean checkNodeExists(String host, NodeType nodeType) {
- String path = getNodeParentPath(nodeType);
- if (StringUtils.isEmpty(path)) {
- logger.error("check zk node exists error, host:{}, zk node type:{}",
- host, nodeType);
- return false;
- }
- Map<String, String> serverMaps = getServerMaps(nodeType, true);
- for (String hostKey : serverMaps.keySet()) {
- if (hostKey.contains(host)) {
- return true;
- }
- }
- return false;
+ return !exists(node) || exists(deadServerPath);
}
- /**
- * @return get worker node parent path
- */
- protected String getWorkerNodeParentPath() {
- return Constants.REGISTRY_DOLPHINSCHEDULER_WORKERS;
+ public Collection<String> getMasterNodesDirectly() {
+ return getChildrenKeys(REGISTRY_DOLPHINSCHEDULER_MASTERS);
}
- /**
- * @return get master node parent path
- */
- protected String getMasterNodeParentPath() {
- return Constants.REGISTRY_DOLPHINSCHEDULER_MASTERS;
+ public Collection<String> getWorkerGroupDirectly() {
+ return getChildrenKeys(REGISTRY_DOLPHINSCHEDULER_WORKERS);
}
- /**
- * @return get dead server node parent path
- */
- protected String getDeadNodeParentPath() {
- return Constants.REGISTRY_DOLPHINSCHEDULER_DEAD_SERVERS;
+ public Collection<String> getWorkerGroupNodesDirectly(String workerGroup) {
+ return getChildrenKeys(REGISTRY_DOLPHINSCHEDULER_WORKERS + "/" + workerGroup);
}
/**
- * @return get master lock path
+ * get host ip:port, path format: parentPath/ip:port
+ *
+ * @param path path
+ * @return host ip:port, string format: parentPath/ip:port
*/
- public String getMasterLockPath() {
- return Constants.REGISTRY_DOLPHINSCHEDULER_LOCK_MASTERS;
+ public String getHostByEventDataPath(String path) {
+ checkArgument(!Strings.isNullOrEmpty(path), "path cannot be null or empty");
+
+ final String[] pathArray = path.split(SINGLE_SLASH);
+
+ checkArgument(pathArray.length >= 1, "cannot parse path: %s", path);
+
+ return pathArray[pathArray.length - 1];
}
- /**
- * @param nodeType zookeeper node type
- * @return get zookeeper node parent path
- */
- public String getNodeParentPath(NodeType nodeType) {
- String path = "";
- switch (nodeType) {
- case MASTER:
- return getMasterNodeParentPath();
- case WORKER:
- return getWorkerNodeParentPath();
- case DEAD_SERVER:
- return getDeadNodeParentPath();
- default:
- break;
+ public void close() throws IOException {
+ if (isStarted.compareAndSet(true, false) && registry != null) {
+ registry.close();
}
- return path;
}
- /**
- * @return get master start up lock path
- */
- public String getMasterStartUpLockPath() {
- return Constants.REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_STARTUP_MASTERS;
+ public void persistEphemeral(String key, String value) {
+ registry.put(key, value, true);
}
- /**
- * @return get master failover lock path
- */
- public String getMasterFailoverLockPath() {
- return Constants.REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_MASTERS;
+ public void remove(String key) {
+ registry.delete(key);
}
- /**
- * @return get worker failover lock path
- */
- public String getWorkerFailoverLockPath() {
- return Constants.REGISTRY_DOLPHINSCHEDULER_LOCK_FAILOVER_WORKERS;
+ public String get(String key) {
+ return registry.get(key);
}
- /**
- * opType(add): if find dead server , then add to zk deadServerPath
- * opType(delete): delete path from zk
- *
- * @param node node path
- * @param nodeType master or worker
- * @param opType delete or add
- * @throws Exception errors
- */
- public void handleDeadServer(String node, NodeType nodeType, String opType) throws Exception {
- String host = getHostByEventDataPath(node);
- String type = (nodeType == NodeType.MASTER) ? MASTER_TYPE : WORKER_TYPE;
-
- //check server restart, if restart , dead server path in zk should be delete
- if (opType.equals(DELETE_OP)) {
- removeDeadServerByHost(host, type);
-
- } else if (opType.equals(ADD_OP)) {
- String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + host;
- if (!isExisted(deadServerPath)) {
- //add dead server info to zk dead server path : /dead-servers/
+ public void subscribe(String path, SubscribeListener listener) {
+ registry.subscribe(path, listener);
+ }
- persist(deadServerPath, (type + UNDERLINE + host));
+ public void addConnectionStateListener(ConnectionListener listener) {
+ registry.addConnectionStateListener(listener);
+ }
- logger.info("{} server dead , and {} added to zk dead server path success",
- nodeType, node);
- }
- }
+ public boolean exists(String key) {
+ return registry.exists(key);
+ }
+ public boolean getLock(String key) {
+ return registry.acquireLock(key);
}
- /**
- * check dead server or not , if dead, stop self
- *
- * @param node node path
- * @param serverType master or worker prefix
- * @return true if not exists
- * @throws Exception errors
- */
- public boolean checkIsDeadServer(String node, String serverType) throws Exception {
- // ip_sequence_no
- String[] zNodesPath = node.split("\\/");
- String ipSeqNo = zNodesPath[zNodesPath.length - 1];
- String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + serverType + UNDERLINE + ipSeqNo;
+ public boolean releaseLock(String key) {
+ return registry.releaseLock(key);
+ }
- return !isExisted(node) || isExisted(deadServerPath);
+ public void setStoppable(IStoppable stoppable) {
+ this.stoppable = stoppable;
}
- /**
- * get master nodes directly
- *
- * @return master nodes
- */
- public Set<String> getMasterNodesDirectly() {
- List<String> masters = getChildrenKeys(REGISTRY_DOLPHINSCHEDULER_MASTERS);
- return new HashSet<>(masters);
+ public IStoppable getStoppable() {
+ return stoppable;
}
- /**
- * get worker nodes directly
- *
- * @return master nodes
- */
- public Set<String> getWorkerNodesDirectly() {
- List<String> workers = getChildrenKeys(REGISTRY_DOLPHINSCHEDULER_WORKERS);
- return new HashSet<>(workers);
+ public boolean isMasterPath(String path) {
+ return path != null && path.startsWith(REGISTRY_DOLPHINSCHEDULER_MASTERS);
}
- /**
- * get worker group directly
- *
- * @return worker group nodes
- */
- public Set<String> getWorkerGroupDirectly() {
- List<String> workers = getChildrenKeys(REGISTRY_DOLPHINSCHEDULER_WORKERS);
- return new HashSet<>(workers);
+ public boolean isWorkerPath(String path) {
+ return path != null && path.startsWith(REGISTRY_DOLPHINSCHEDULER_WORKERS);
}
- /**
- * get worker group nodes
- */
- public Set<String> getWorkerGroupNodesDirectly(String workerGroup) {
- List<String> workers = getChildrenKeys(getWorkerGroupPath(workerGroup));
- return new HashSet<>(workers);
+ public Collection<String> getChildrenKeys(final String key) {
+ return registry.children(key);
}
- /**
- * opType(add): if find dead server , then add to zk deadServerPath
- * opType(delete): delete path from zk
- *
- * @param nodeSet node path set
- * @param nodeType master or worker
- * @param opType delete or add
- * @throws Exception errors
- */
- public void handleDeadServer(Set<String> nodeSet, NodeType nodeType, String opType) throws Exception {
+ public Set<String> getServerNodeSet(NodeType nodeType, boolean hostOnly) {
+ try {
+ return getServerNodes(nodeType).stream().map(server -> {
+ if (nodeType == NodeType.WORKER && hostOnly) {
+ return server.split(SINGLE_SLASH)[1];
+ }
+ return server;
+ }).collect(Collectors.toSet());
+ } catch (Exception e) {
+ throw new RegistryException("Failed to get server node: " + nodeType, e);
+ }
+ }
- String type = (nodeType == NodeType.MASTER) ? MASTER_TYPE : WORKER_TYPE;
- for (String node : nodeSet) {
- String host = getHostByEventDataPath(node);
- //check server restart, if restart , dead server path in zk should be delete
- if (opType.equals(DELETE_OP)) {
- removeDeadServerByHost(host, type);
+ private void start() {
+ if (isStarted.compareAndSet(false, true)) {
+ PropertyUtils.loadPropertyFile(REGISTRY_CONFIG_FILE_PATH);
+ final Map<String, String> registryConfig = PropertyUtils.getPropertiesByPrefix(REGISTRY_PREFIX);
- } else if (opType.equals(ADD_OP)) {
- String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + host;
- if (!isExisted(deadServerPath)) {
- //add dead server info to zk dead server path : /dead-servers/
- persist(deadServerPath, (type + UNDERLINE + host));
- logger.info("{} server dead , and {} added to registry dead server path success",
- nodeType, node);
- }
+ if (null == registryConfig || registryConfig.isEmpty()) {
+ throw new RegistryException("registry config param is null");
}
-
+ final String pluginName = registryConfig.get(REGISTRY_PLUGIN_NAME);
+ final Map<String, RegistryFactory> factories = RegistryFactoryLoader.load();
+ if (!factories.containsKey(pluginName)) {
+ throw new RegistryException("No such registry plugin: " + pluginName);
+ }
+ registry = factories.get(pluginName).create();
+ registry.start(registryConfig);
}
+ }
+ private void initNodes() {
+ registry.put(REGISTRY_DOLPHINSCHEDULER_MASTERS, EMPTY, false);
+ registry.put(REGISTRY_DOLPHINSCHEDULER_WORKERS, EMPTY, false);
+ registry.put(REGISTRY_DOLPHINSCHEDULER_DEAD_SERVERS, EMPTY, false);
}
- /**
- * get host ip:port, string format: parentPath/ip:port
- *
- * @param path path
- * @return host ip:port, string format: parentPath/ip:port
- */
- public String getHostByEventDataPath(String path) {
- if (StringUtils.isEmpty(path)) {
- logger.error("empty path!");
- return "";
- }
- String[] pathArray = path.split(SINGLE_SLASH);
- if (pathArray.length < 1) {
- logger.error("parse ip error: {}", path);
- return "";
+ private String rootNodePath(NodeType type) {
+ switch (type) {
+ case MASTER:
+ return Constants.REGISTRY_DOLPHINSCHEDULER_MASTERS;
+ case WORKER:
+ return Constants.REGISTRY_DOLPHINSCHEDULER_WORKERS;
+ case DEAD_SERVER:
+ return Constants.REGISTRY_DOLPHINSCHEDULER_DEAD_SERVERS;
+ default:
+ throw new IllegalStateException("Should not reach here");
}
- return pathArray[pathArray.length - 1];
+ }
+ private Collection<String> getServerNodes(NodeType nodeType) {
+ final String path = rootNodePath(nodeType);
+ final Collection<String> serverList = getChildrenKeys(path);
+ if (nodeType != NodeType.WORKER) {
+ return serverList;
+ }
+ return serverList.stream().flatMap(group ->
+ getChildrenKeys(path + SINGLE_SLASH + group)
+ .stream()
+ .map(it -> group + SINGLE_SLASH + it)
+ ).collect(Collectors.toList());
}
- /**
- * remove dead server by host
- *
- * @param host host
- * @param serverType serverType
- */
- public void removeDeadServerByHost(String host, String serverType) {
- List<String> deadServers = getChildrenKeys(getDeadZNodeParentPath());
+ private void removeDeadServerByHost(String host, String serverType) {
+ Collection<String> deadServers = getChildrenKeys(REGISTRY_DOLPHINSCHEDULER_DEAD_SERVERS);
for (String serverPath : deadServers) {
if (serverPath.startsWith(serverType + UNDERLINE + host)) {
- String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath;
+ String server = REGISTRY_DOLPHINSCHEDULER_DEAD_SERVERS + SINGLE_SLASH + serverPath;
remove(server);
logger.info("{} server {} deleted from zk dead server path success", serverType, host);
}
}
}
-
}
diff --git a/dolphinscheduler-service/src/main/resources/registry.properties b/dolphinscheduler-service/src/main/resources/registry.properties
index b00dfc0..f534a65 100644
--- a/dolphinscheduler-service/src/main/resources/registry.properties
+++ b/dolphinscheduler-service/src/main/resources/registry.properties
@@ -15,18 +15,5 @@
# limitations under the License.
#
-#registry.plugin.dir config the Registry Plugin dir.
-registry.plugin.dir=lib/plugin/registry
-
registry.plugin.name=zookeeper
registry.servers=127.0.0.1:2181
-
-#maven.local.repository=/usr/local/localRepository
-
-#registry.plugin.binding config the Registry Plugin need be load when development and run in IDE
-#registry.plugin.binding=./dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml
-
-#registry timeout
-#registry.session.timeout.ms=30000
-#registry.connection.timeout.ms=7500
-#registry.block.until.connected.wait=600
\ No newline at end of file
diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/registry/RegistryClientTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/registry/RegistryClientTest.java
deleted file mode 100644
index 2dc2193..0000000
--- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/registry/RegistryClientTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.service.registry;
-
-import static org.apache.dolphinscheduler.common.Constants.ADD_OP;
-import static org.apache.dolphinscheduler.common.Constants.DELETE_OP;
-
-import static org.mockito.BDDMockito.given;
-
-import org.apache.dolphinscheduler.common.enums.NodeType;
-import org.apache.dolphinscheduler.spi.register.Registry;
-
-import java.util.Arrays;
-
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.mockito.Mockito;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-
-import com.google.common.collect.Sets;
-
-@RunWith(PowerMockRunner.class)
-@PrepareForTest({ RegistryClient.class })
-public class RegistryClientTest {
-
- private RegistryClient registryClient;
-
- @Test
- public void test() throws Exception {
- Registry registry = PowerMockito.mock(Registry.class);
- PowerMockito.doNothing().when(registry).persist(Mockito.anyString(), Mockito.anyString());
- PowerMockito.doNothing().when(registry).update(Mockito.anyString(), Mockito.anyString());
- PowerMockito.when(registry.releaseLock(Mockito.anyString())).thenReturn(true);
- PowerMockito.when(registry.getChildren("/dead-servers")).thenReturn(Arrays.asList("worker_127.0.0.1:8089"));
-
- PowerMockito.suppress(PowerMockito.constructor(RegistryClient.class));
- registryClient = PowerMockito.mock(RegistryClient.class);
- registryClient.persist("/key", "");
- registryClient.update("/key", "");
- registryClient.releaseLock("/key");
- registryClient.getChildrenKeys("/key");
- registryClient.handleDeadServer(Sets.newHashSet("ma/127.0.0.1:8089"), NodeType.WORKER, DELETE_OP);
- registryClient.handleDeadServer(Sets.newHashSet("ma/127.0.0.1:8089"), NodeType.WORKER, ADD_OP);
- //registryClient.removeDeadServerByHost("127.0.0.1:8089","master");
- registryClient.handleDeadServer("ma/127.0.0.1:8089", NodeType.WORKER, DELETE_OP);
- registryClient.handleDeadServer("ma/127.0.0.1:8089", NodeType.WORKER, ADD_OP);
- registryClient.checkIsDeadServer("master/127.0.0.1","master");
- given(registry.getChildren("/nodes/worker")).willReturn(Arrays.asList("worker_127.0.0.1:8089"));
- given(registry.getChildren("/nodes/worker/worker_127.0.0.1:8089")).willReturn(Arrays.asList("default"));
-
- registryClient.checkNodeExists("127.0.0.1",NodeType.WORKER);
-
- registryClient.getServerList(NodeType.MASTER);
-
- }
-
-}
diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/registry/RegistryPluginTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/registry/RegistryPluginTest.java
deleted file mode 100644
index a35252c..0000000
--- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/registry/RegistryPluginTest.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.service.registry;
-
-import org.apache.dolphinscheduler.spi.plugin.DolphinPluginLoader;
-import org.apache.dolphinscheduler.spi.plugin.DolphinPluginManagerConfig;
-import org.apache.dolphinscheduler.spi.register.RegistryPluginManager;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.google.common.collect.ImmutableList;
-
-public class RegistryPluginTest {
-
- @Test
- public void testLoadPlugin() throws Exception {
- DolphinPluginManagerConfig registryPluginManagerConfig = new DolphinPluginManagerConfig();
- String path = DolphinPluginLoader.class.getClassLoader().getResource("").getPath();
-
- String registryPluginZkPath = path + "../../../dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml";
- registryPluginManagerConfig.setPlugins(registryPluginZkPath);
- RegistryPluginManager registryPluginManager = new RegistryPluginManager("zookeeper");
-
- DolphinPluginLoader registryPluginLoader = new DolphinPluginLoader(registryPluginManagerConfig, ImmutableList.of(registryPluginManager));
- registryPluginLoader.loadPlugins();
- Assert.assertNotNull(registryPluginManager.getRegistry());
-
- }
-}
diff --git a/dolphinscheduler-spi/pom.xml b/dolphinscheduler-spi/pom.xml
index 3212ad1..1a21502 100644
--- a/dolphinscheduler-spi/pom.xml
+++ b/dolphinscheduler-spi/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-spi</artifactId>
<name>${project.artifactId}</name>
@@ -112,4 +112,4 @@
</dependency>
</dependencies>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/DolphinSchedulerPlugin.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/DolphinSchedulerPlugin.java
index 1c5f1c5..a073c53 100644
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/DolphinSchedulerPlugin.java
+++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/DolphinSchedulerPlugin.java
@@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.spi;
import static java.util.Collections.emptyList;
import org.apache.dolphinscheduler.spi.alert.AlertChannelFactory;
-import org.apache.dolphinscheduler.spi.register.RegistryFactory;
import org.apache.dolphinscheduler.spi.task.TaskChannelFactory;
/**
@@ -43,14 +42,6 @@ public interface DolphinSchedulerPlugin {
}
/**
- * get registry plugin factory
- * @return registry factory
- */
- default Iterable<RegistryFactory> getRegisterFactorys() {
- return emptyList();
- }
-
- /**
* get task plugin factory
* @return registry factory
*/
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/ConnectStateListener.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/ConnectStateListener.java
deleted file mode 100644
index 6675ef6..0000000
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/ConnectStateListener.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.spi.register;
-
-public interface ConnectStateListener {
-
- void notify(RegistryConnectState state);
-}
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/DataChangeEvent.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/DataChangeEvent.java
deleted file mode 100644
index a6aa32d..0000000
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/DataChangeEvent.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.spi.register;
-
-/**
- * Monitor the type of data changes
- */
-public enum DataChangeEvent {
-
- ADD("ADD", 1),
- REMOVE("REMOVE", 2),
- UPDATE("UPDATE",3);
-
- private String type;
-
- private int value;
-
- DataChangeEvent(String type, int value) {
- this.type = type;
- this.value = value;
- }
-}
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/ListenerManager.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/ListenerManager.java
deleted file mode 100644
index 94b13e6..0000000
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/ListenerManager.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.spi.register;
-
-import java.util.HashMap;
-
-/**
- * The registry node monitors subscriptions
- */
-public class ListenerManager {
-
- /**
- * All message subscriptions must be subscribed uniformly at startup.
- * A node path only supports one listener
- */
- private static HashMap<String, SubscribeListener> listeners = new HashMap<>();
-
- /**
- * Check whether the key has been monitored
- */
- public static boolean checkHasListeners(String path) {
- return null != listeners.get(path);
- }
-
- /**
- * add listener(A node can only be monitored by one listener)
- */
- public static void addListener(String path, SubscribeListener listener) {
- listeners.put(path, listener);
- }
-
- /**
- * remove listener
- */
- public static void removeListener(String path) {
- listeners.remove(path);
- }
-
- /**
- *
- *After the data changes, it is distributed to the corresponding listener for processing
- */
- public static void dataChange(String key,String path, String data, DataChangeEvent dataChangeEvent) {
- SubscribeListener notifyListener = listeners.get(key);
- if (null == notifyListener) {
- return;
- }
- notifyListener.notify(path, data, dataChangeEvent);
- }
-
-}
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/Registry.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/Registry.java
deleted file mode 100644
index 11fe25a..0000000
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/Registry.java
+++ /dev/null
@@ -1,102 +0,0 @@
-package org.apache.dolphinscheduler.spi.register;/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * The final display of all registry component data must follow a tree structure.
- * Therefore, some registry may need to do a layer of internal conversion, such as Etcd
- */
-public interface Registry {
-
- /**
- * initialize registry center.
- */
- void init(Map<String, String> registerData);
-
- /**
- * close registry
- */
- void close();
-
- /**
- * subscribe registry data change, a path can only be monitored by one listener
- */
- boolean subscribe(String path, SubscribeListener subscribeListener);
-
- /**
- * unsubscribe
- */
- void unsubscribe(String path);
-
- /**
- * Registry status monitoring, globally unique. Only one is allowed to subscribe.
- */
- void addConnectionStateListener(RegistryConnectListener registryConnectListener);
-
- /**
- * get key
- */
- String get(String key);
-
- /**
- * delete
- */
- void remove(String key);
-
- /**
- * persist data
- */
- void persist(String key, String value);
-
- /**
- *persist ephemeral data
- */
- void persistEphemeral(String key, String value);
-
- /**
- * update data
- */
- void update(String key, String value);
-
- /**
- * get children keys
- */
- List<String> getChildren(String path);
-
- /**
- * Judge node is exist or not.
- */
- boolean isExisted(String key);
-
- /**
- * delete kay
- */
- boolean delete(String key);
-
- /**
- * Obtain a distributed lock
- * todo It is best to add expiration time, and automatically release the lock after expiration
- */
- boolean acquireLock(String key);
-
- /**
- * release key
- */
- boolean releaseLock(String key);
-}
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryConnectListener.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryConnectListener.java
deleted file mode 100644
index 83385f8..0000000
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryConnectListener.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.spi.register;
-
-public interface RegistryConnectListener {
-
- void notify(RegistryConnectState newState);
-}
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryConnectState.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryConnectState.java
deleted file mode 100644
index e085e6d..0000000
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryConnectState.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.spi.register;
-
-/**
- * All registry connection status must be converted to this
- */
-public enum RegistryConnectState {
- CONNECTED("connected", 1),
- RECONNECTED("reconnected", 2),
- SUSPENDED("suspended", 3),
- LOST("lost", 4);
-
- private String description;
-
- private int state;
-
- RegistryConnectState(String description, int state) {
- this.description = description;
- this.state = state;
- }
-}
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryException.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryException.java
deleted file mode 100644
index 884f005..0000000
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.spi.register;
-
-/**
- * registry exception
- */
-public class RegistryException extends RuntimeException {
-
- public RegistryException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public RegistryException(String message) {
- super(message);
- }
-}
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryFactory.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryFactory.java
deleted file mode 100644
index 244c0f4..0000000
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryFactory.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.spi.register;
-
-/**
- * Registry the component factory, all registry must implement this interface
- */
-public interface RegistryFactory {
-
- /**
- * get registry component name
- */
- String getName();
-
- /**
- * get registry
- */
- Registry create();
-}
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryPluginManager.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryPluginManager.java
deleted file mode 100644
index 211795f..0000000
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/RegistryPluginManager.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.spi.register;
-
-import org.apache.dolphinscheduler.spi.DolphinSchedulerPlugin;
-import org.apache.dolphinscheduler.spi.classloader.ThreadContextClassLoader;
-import org.apache.dolphinscheduler.spi.plugin.AbstractDolphinPluginManager;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * The plug-in address of the registry needs to be configured.
- * Multi-registries are not supported.
- * When the plug-in directory contains multiple plug-ins, only the configured plug-in will be used.
- * todo It’s not good to put it here, consider creating a separate API module for each plugin
- */
-public class RegistryPluginManager extends AbstractDolphinPluginManager {
-
- private static final Logger logger = LoggerFactory.getLogger(RegistryPluginManager.class);
-
- private RegistryFactory registryFactory;
-
- public static Registry registry;
-
- private String registerPluginName;
-
- public RegistryPluginManager(String registerPluginName) {
- this.registerPluginName = registerPluginName;
- }
-
- @Override
- public void installPlugin(DolphinSchedulerPlugin dolphinSchedulerPlugin) {
- for (RegistryFactory registryFactory : dolphinSchedulerPlugin.getRegisterFactorys()) {
- logger.info("Registering Registry Plugin '{}'", registryFactory.getName());
- if (registerPluginName.equals(registryFactory.getName())) {
- this.registryFactory = registryFactory;
- loadRegistry();
- return;
- }
- }
- if (null == registry) {
- throw new RegistryException(String.format("not found %s registry plugin ", registerPluginName));
- }
- }
-
- /**
- * load registry
- */
- private void loadRegistry() {
- try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(registryFactory.getClass().getClassLoader())) {
- registry = registryFactory.create();
- }
- }
-
- /**
- * get registry
- * @return registry
- */
- public Registry getRegistry() {
- if (null == registry) {
- throw new RegistryException("not install registry");
- }
- return registry;
- }
-
-}
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/SubscribeListener.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/SubscribeListener.java
deleted file mode 100644
index 3db7f2e..0000000
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/register/SubscribeListener.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.spi.register;
-
-/**
- * Registration center subscription. All listeners must implement this interface
- */
-public interface SubscribeListener {
-
- /**
- * Processing logic when the subscription node changes
- */
- void notify(String path, String data, DataChangeEvent dataChangeEvent);
-
-}
diff --git a/dolphinscheduler-standalone-server/pom.xml b/dolphinscheduler-standalone-server/pom.xml
index a6558a9..e16c3d0 100644
--- a/dolphinscheduler-standalone-server/pom.xml
+++ b/dolphinscheduler-standalone-server/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/server/StandaloneServer.java b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/server/StandaloneServer.java
index bc42871..f3ab194 100644
--- a/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/server/StandaloneServer.java
+++ b/dolphinscheduler-standalone-server/src/main/java/org/apache/dolphinscheduler/server/StandaloneServer.java
@@ -85,15 +85,6 @@ public class StandaloneServer {
private static void startRegistry() throws Exception {
final TestingServer server = new TestingServer(true);
System.setProperty("registry.servers", server.getConnectString());
-
- final Path registryPath = Paths.get(
- StandaloneServer.class.getProtectionDomain().getCodeSource().getLocation().getPath(),
- "../../../dolphinscheduler-registry-plugin/dolphinscheduler-registry-zookeeper/pom.xml"
- ).toAbsolutePath();
- if (Files.exists(registryPath)) {
- System.setProperty("registry.plugin.binding", registryPath.toString());
- System.setProperty("registry.plugin.dir", "");
- }
}
private static void startDatabase() throws IOException, SQLException {
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml
index c361dbf..d5b25a2 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-task-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>jar</packaging>
@@ -284,4 +284,4 @@
<build>
<finalName>dolphinscheduler-task-api-${project.version}</finalName>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml
index 3d07854..1321f2f 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-task-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml
index 0a8d385..6e55c5a 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-task-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -42,4 +42,4 @@
<build>
<finalName>dolphinscheduler-task-flink-${project.version}</finalName>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml
index 47aacba..2daf812 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-http/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-task-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml
index 40b53ee..f409aa3 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-task-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -44,4 +44,4 @@
<finalName>dolphinscheduler-task-mr-${project.version}</finalName>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/pom.xml
index 7103add..291b697 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-pigeon/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-task-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -108,4 +108,4 @@
<finalName>dolphinscheduler-task-pigeon-${project.version}</finalName>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml
index a29f7e0..8295239 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-procedure/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-task-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml
index 5a83f4a..3c043f0 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-python/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-task-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -42,4 +42,4 @@
<build>
<finalName>dolphinscheduler-task-python-${project.version}</finalName>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/pom.xml
index 78a610d..e6f5fd4 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-shell/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-task-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -47,4 +47,4 @@
<build>
<finalName>dolphinscheduler-task-shell-${project.version}</finalName>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml
index 0cea039..46b1ca1 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-task-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dolphinscheduler-task-spark</artifactId>
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml
index e914cd8..4074c40 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-task-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -49,4 +49,4 @@
<build>
<finalName>dolphinscheduler-task-sql-${project.version}</finalName>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml
index 81632a7..f1bafbd 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sqoop/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler-task-plugin</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -43,4 +43,4 @@
<build>
<finalName>dolphinscheduler-task-sqoop-${project.version}</finalName>
</build>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-task-plugin/pom.xml b/dolphinscheduler-task-plugin/pom.xml
index 6221ab6..d219193 100644
--- a/dolphinscheduler-task-plugin/pom.xml
+++ b/dolphinscheduler-task-plugin/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -42,4 +42,4 @@
</modules>
-</project>
\ No newline at end of file
+</project>
diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml
index a50b82e..f6ae7fc 100644
--- a/dolphinscheduler-ui/pom.xml
+++ b/dolphinscheduler-ui/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pom.xml b/pom.xml
index 636db76..ad231e0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -15,11 +15,12 @@
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler</artifactId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.0.1-SNAPSHOT</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<url>http://dolphinscheduler.apache.org</url>
@@ -38,7 +39,7 @@
<connection>scm:git:https://github.com/apache/dolphinscheduler.git</connection>
<developerConnection>scm:git:https://github.com/apache/dolphinscheduler.git</developerConnection>
<url>https://github.com/apache/dolphinscheduler</url>
- <tag>2.0.0-alpha</tag>
+ <tag>HEAD</tag>
</scm>
<mailingLists>
<mailingList>
@@ -126,6 +127,8 @@
<reflections.version>0.9.12</reflections.version>
<byte-buddy.version>1.9.16</byte-buddy.version>
<java-websocket.version>1.5.1</java-websocket.version>
+ <py4j.version>0.10.9</py4j.version>
+ <auto-service.version>1.0.1</auto-service.version>
</properties>
<dependencyManagement>
@@ -229,7 +232,12 @@
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-registry-plugin</artifactId>
+ <artifactId>dolphinscheduler-registry</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-registry-zookeeper</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
@@ -264,6 +272,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-registry-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<version>${curator.version}</version>
@@ -677,6 +691,12 @@
<artifactId>javax.mail</artifactId>
<version>1.6.2</version>
</dependency>
+ <dependency>
+ <groupId>com.google.auto.service</groupId>
+ <artifactId>auto-service</artifactId>
+ <version>${auto-service.version}</version>
+ <optional>true</optional>
+ </dependency>
</dependencies>
</dependencyManagement>
@@ -1220,7 +1240,7 @@
<modules>
<module>dolphinscheduler-spi</module>
<module>dolphinscheduler-alert-plugin</module>
- <module>dolphinscheduler-registry-plugin</module>
+ <module>dolphinscheduler-registry</module>
<module>dolphinscheduler-task-plugin</module>
<module>dolphinscheduler-ui</module>
<module>dolphinscheduler-server</module>
diff --git a/script/env/dolphinscheduler_env.sh b/script/env/dolphinscheduler_env.sh
index e62f54e..1798a5d 100755
--- a/script/env/dolphinscheduler_env.sh
+++ b/script/env/dolphinscheduler_env.sh
@@ -20,7 +20,7 @@ export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
export SPARK_HOME1=/opt/soft/spark1
export SPARK_HOME2=/opt/soft/spark2
export PYTHON_HOME=/opt/soft/python
-export JAVA_HOME=/opt/soft/java
+export JAVA_HOME=${JAVA_HOME:-/opt/soft/java}
export HIVE_HOME=/opt/soft/hive
export FLINK_HOME=/opt/soft/flink
export DATAX_HOME=/opt/soft/datax