You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ke...@apache.org on 2022/09/02 04:03:27 UTC
[dolphinscheduler] branch dev updated: Add Kubernetes configmap reload to all components (#11730)
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 277f137358 Add Kubernetes configmap reload to all components (#11730)
277f137358 is described below
commit 277f137358628bd248483b6b5e865f41b3aaaa11
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Fri Sep 2 12:03:19 2022 +0800
Add Kubernetes configmap reload to all components (#11730)
---
docs/docs/en/architecture/configuration.md | 37 +++++++------
docs/docs/zh/architecture/configuration.md | 53 ++++++++++--------
.../dolphinscheduler-alert-server/pom.xml | 13 +++--
.../src/main/resources/application.yaml | 2 -
.../src/main/resources/bootstrap.yaml | 28 ++++++++++
dolphinscheduler-api/pom.xml | 5 ++
.../src/main/resources/application.yaml | 2 -
.../src/main/resources/bootstrap.yaml | 28 ++++++++++
dolphinscheduler-dist/release-docs/LICENSE | 2 +
.../server/log/LoggerRequestProcessor.java | 17 +++---
dolphinscheduler-master/pom.xml | 64 ++++++++++++----------
.../src/main/resources/application.yaml | 2 -
.../src/main/resources/bootstrap.yaml | 28 ++++++++++
dolphinscheduler-standalone-server/pom.xml | 4 --
.../src/main/resources/application.yaml | 2 -
.../src/main/resources/bootstrap.yaml | 28 ++++++++++
dolphinscheduler-worker/pom.xml | 41 ++++++++------
.../src/main/resources/application.yaml | 2 -
.../src/main/resources/bootstrap.yaml | 28 ++++++++++
tools/dependencies/known-dependencies.txt | 11 ++++
20 files changed, 284 insertions(+), 113 deletions(-)
diff --git a/docs/docs/en/architecture/configuration.md b/docs/docs/en/architecture/configuration.md
index 48d8aa80ab..cfa853c896 100644
--- a/docs/docs/en/architecture/configuration.md
+++ b/docs/docs/en/architecture/configuration.md
@@ -17,14 +17,14 @@ The directory structure of DolphinScheduler is as follows:
│
├── licenses directory of licenses
│
-├── bin directory of DolphinScheduler application commands, configrations scripts
+├── bin directory of DolphinScheduler application commands, configrations scripts
│ ├── dolphinscheduler-daemon.sh script to start or shut down DolphinScheduler application
│ ├── env directory of scripts to load environment variables
│ │ ├── dolphinscheduler_env.sh script to export environment variables [eg: JAVA_HOME,HADOOP_HOME, HIVE_HOME ...] when you start or stop service using script `dolphinscheduler-daemon.sh`
│ │ └── install_env.sh script to export environment variables for DolphinScheduler installation when you use scripts `install.sh` `start-all.sh` `stop-all.sh` `status-all.sh`
│ ├── install.sh script to auto-setup services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
-│ ├── remove-zk-node.sh script to cleanup ZooKeeper caches
-│ ├── scp-hosts.sh script to copy installation files to target hosts
+│ ├── remove-zk-node.sh script to cleanup ZooKeeper caches
+│ ├── scp-hosts.sh script to copy installation files to target hosts
│ ├── start-all.sh script to start all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
│ ├── status-all.sh script to check the status of all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
│ └── stop-all.sh script to shut down all services when you deploy DolphinScheduler in `psuedo-cluster` mode or `cluster` mode
@@ -34,7 +34,8 @@ The directory structure of DolphinScheduler is as follows:
│ │ └── start.sh script to start DolphinScheduler alert-server
│ ├── conf
│ │ ├── application.yaml configurations of alert-server
-│ │ ├── common.properties configurations of common-service like storage, credentials, etc.
+│ │ ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this,
+│ │ ├── common.properties configurations of common-service like storage, credentials, etc.
│ │ ├── dolphinscheduler_env.sh script to load environment variables for alert-server
│ │ └── logback-spring.xml configurations of alert-service log
│ └── libs directory of alert-server libs
@@ -44,17 +45,19 @@ The directory structure of DolphinScheduler is as follows:
│ │ └── start.sh script to start DolphinScheduler api-server
│ ├── conf
│ │ ├── application.yaml configurations of api-server
+│ │ ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this,
│ │ ├── common.properties configurations of common-service like storage, credentials, etc.
│ │ ├── dolphinscheduler_env.sh script to load environment variables for api-server
│ │ └── logback-spring.xml configurations of api-service log
│ ├── libs directory of api-server libs
-│ └── ui directory of api-server related front-end web resources
+│ └── ui directory of api-server related front-end web resources
│
├── master-server directory of DolphinScheduler master-server commands, configrations scripts and libs
-│ ├── bin
+│ ├── bin
│ │ └── start.sh script to start DolphinScheduler master-server
│ ├── conf
│ │ ├── application.yaml configurations of master-server
+│ │ ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this,
│ │ ├── common.properties configurations of common-service like storage, credentials, etc.
│ │ ├── dolphinscheduler_env.sh script to load environment variables for master-server
│ │ └── logback-spring.xml configurations of master-service log
@@ -65,13 +68,14 @@ The directory structure of DolphinScheduler is as follows:
│ │ └── start.sh script to start DolphinScheduler standalone-server
│ ├── conf
│ │ ├── application.yaml configurations of standalone-server
+│ │ ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this,
│ │ ├── common.properties configurations of common-service like storage, credentials, etc.
│ │ ├── dolphinscheduler_env.sh script to load environment variables for standalone-server
│ │ ├── logback-spring.xml configurations of standalone-service log
│ │ └── sql .sql files to create or upgrade DolphinScheduler metadata
│ ├── libs directory of standalone-server libs
│ └── ui directory of standalone-server related front-end web resources
-│
+│
├── tools directory of DolphinScheduler metadata tools commands, configrations scripts and libs
│ ├── bin
│ │ └── upgrade-schema.sh script to initialize or upgrade DolphinScheduler metadata
@@ -80,12 +84,13 @@ The directory structure of DolphinScheduler is as follows:
│ │ └── common.properties configurations of common-service like storage, credentials, etc.
│ ├── libs directory of tool libs
│ └── sql .sql files to create or upgrade DolphinScheduler metadata
-│
+│
├── worker-server directory of DolphinScheduler worker-server commands, configrations scripts and libs
│ ├── bin
│ │ └── start.sh script to start DolphinScheduler worker-server
│ ├── conf
│ │ ├── application.yaml configurations of worker-server
+│ │ ├── bootstrap.yaml configurations for Spring Cloud bootstrap, mostly you don't need to modify this,
│ │ ├── common.properties configurations of common-service like storage, credentials, etc.
│ │ ├── dolphinscheduler_env.sh script to load environment variables for worker-server
│ │ └── logback-spring.xml configurations of worker-service log
@@ -107,14 +112,14 @@ Currently, DolphinScheduler just makes a basic config, remember to config furthe
Default simplified parameters are:
```bash
export DOLPHINSCHEDULER_OPTS="
--server
--Xmx16g
--Xms1g
--Xss512k
--XX:+UseConcMarkSweepGC
--XX:+CMSParallelRemarkEnabled
--XX:+UseFastAccessorMethods
--XX:+UseCMSInitiatingOccupancyOnly
+-server
+-Xmx16g
+-Xms1g
+-Xss512k
+-XX:+UseConcMarkSweepGC
+-XX:+CMSParallelRemarkEnabled
+-XX:+UseFastAccessorMethods
+-XX:+UseCMSInitiatingOccupancyOnly
-XX:CMSInitiatingOccupancyFraction=70
"
```
diff --git a/docs/docs/zh/architecture/configuration.md b/docs/docs/zh/architecture/configuration.md
index 9e18e7832d..fb985a96d3 100644
--- a/docs/docs/zh/architecture/configuration.md
+++ b/docs/docs/zh/architecture/configuration.md
@@ -13,13 +13,13 @@ DolphinScheduler的目录结构如下：
│
├── licenses licenses存放目录
│
-├── bin DolphinScheduler命令和环境变量配置存放目录
+├── bin DolphinScheduler命令和环境变量配置存放目录
│ ├── dolphinscheduler-daemon.sh 启动/关闭DolphinScheduler服务脚本
│ ├── env 环境变量配置存放目录
-│ │ ├── dolphinscheduler_env.sh 当使用`dolphinscheduler-daemon.sh`脚本起停服务时，运行此脚本加载环境变量配置文件 [如：JAVA_HOME,HADOOP_HOME, HIVE_HOME ...]
-│ │ └── install_env.sh 当使用`install.sh` `start-all.sh` `stop-all.sh` `status-all.sh`脚本时，运行此脚本为DolphinScheduler安装加载环境变量配置
-│ ├── install.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时，运行此脚本自动安装服务
-│ ├── remove-zk-node.sh 清理zookeeper缓存文件脚本
+│ │ ├── dolphinscheduler_env.sh 当使用`dolphinscheduler-daemon.sh`脚本起停服务时，运行此脚本加载环境变量配置文件 [如：JAVA_HOME,HADOOP_HOME, HIVE_HOME ...]
+│ │ └── install_env.sh 当使用`install.sh` `start-all.sh` `stop-all.sh` `status-all.sh`脚本时，运行此脚本为DolphinScheduler安装加载环境变量配置
+│ ├── install.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时，运行此脚本自动安装服务
+│ ├── remove-zk-node.sh 清理zookeeper缓存文件脚本
│ ├── scp-hosts.sh 安装文件传输脚本
│ ├── start-all.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时，运行此脚本启动所有服务
│ ├── status-all.sh 当使用`集群`模式或`伪集群`模式部署DolphinScheduler时，运行此脚本获取所有服务状态
@@ -30,7 +30,8 @@ DolphinScheduler的目录结构如下：
│ │ └── start.sh DolphinScheduler alert-server启动脚本
│ ├── conf
│ │ ├── application.yaml alert-server配置文件
-│ │ ├── common.properties 公共服务（存储等信息）配置文件
+│ │ ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改
+│ │ ├── common.properties 公共服务（存储等信息）配置文件
│ │ ├── dolphinscheduler_env.sh alert-server环境变量配置加载脚本
│ │ └── logback-spring.xml alert-service日志配置文件
│ └── libs alert-server依赖jar包存放目录
@@ -40,17 +41,19 @@ DolphinScheduler的目录结构如下：
│ │ └── start.sh DolphinScheduler api-server启动脚本
│ ├── conf
│ │ ├── application.yaml api-server配置文件
+│ │ ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改
│ │ ├── common.properties 公共服务（存储等信息）配置文件
│ │ ├── dolphinscheduler_env.sh api-server环境变量配置加载脚本
│ │ └── logback-spring.xml api-service日志配置文件
│ ├── libs api-server依赖jar包存放目录
-│ └── ui api-server相关前端WEB资源存放目录
+│ └── ui api-server相关前端WEB资源存放目录
│
├── master-server DolphinScheduler master-server命令、配置和依赖存放目录
-│ ├── bin
+│ ├── bin
│ │ └── start.sh DolphinScheduler master-server启动脚本
│ ├── conf
│ │ ├── application.yaml master-server配置文件
+│ │ ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改
│ │ ├── common.properties 公共服务（存储等信息）配置文件
│ │ ├── dolphinscheduler_env.sh master-server环境变量配置加载脚本
│ │ └── logback-spring.xml master-service日志配置文件
@@ -61,13 +64,14 @@ DolphinScheduler的目录结构如下：
│ │ └── start.sh DolphinScheduler standalone-server启动脚本
│ ├── conf
│ │ ├── application.yaml standalone-server配置文件
+│ │ ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改
│ │ ├── common.properties 公共服务（存储等信息）配置文件
│ │ ├── dolphinscheduler_env.sh standalone-server环境变量配置加载脚本
│ │ ├── logback-spring.xml standalone-service日志配置文件
│ │ └── sql DolphinScheduler元数据创建/升级sql文件
│ ├── libs standalone-server依赖jar包存放目录
│ └── ui standalone-server相关前端WEB资源存放目录
-│
+│
├── tools DolphinScheduler元数据工具命令、配置和依赖存放目录
│ ├── bin
│ │ └── upgrade-schema.sh DolphinScheduler元数据创建/升级脚本
@@ -76,12 +80,13 @@ DolphinScheduler的目录结构如下：
│ │ └── common.properties 公共服务（存储等信息）配置文件
│ ├── libs 元数据工具依赖jar包存放目录
│ └── sql DolphinScheduler元数据创建/升级sql文件
-│
+│
├── worker-server DolphinScheduler worker-server命令、配置和依赖存放目录
│ ├── bin
│ │ └── start.sh DolphinScheduler worker-server启动脚本
│ ├── conf
│ │ ├── application.yaml worker-server配置文件
+│ │ ├── bootstrap.yaml Spring Cloud 启动阶段配置文件, 通常不需要修改
│ │ ├── common.properties 公共服务（存储等信息）配置文件
│ │ ├── dolphinscheduler_env.sh worker-server环境变量配置加载脚本
│ │ └── logback-spring.xml worker-service日志配置文件
@@ -93,21 +98,21 @@ DolphinScheduler的目录结构如下：
# 配置文件详解
## dolphinscheduler-daemon.sh [启动/关闭DolphinScheduler服务脚本]
-dolphinscheduler-daemon.sh脚本负责DolphinScheduler的启动&关闭.
+dolphinscheduler-daemon.sh脚本负责DolphinScheduler的启动&关闭.
start-all.sh/stop-all.sh最终也是通过dolphinscheduler-daemon.sh对集群进行启动/关闭操作.
目前DolphinScheduler只是做了一个基本的设置,JVM参数请根据各自资源的实际情况自行设置.
默认简化参数如下:
```bash
export DOLPHINSCHEDULER_OPTS="
--server
--Xmx16g
--Xms1g
--Xss512k
--XX:+UseConcMarkSweepGC
--XX:+CMSParallelRemarkEnabled
--XX:+UseFastAccessorMethods
--XX:+UseCMSInitiatingOccupancyOnly
+-server
+-Xmx16g
+-Xms1g
+-Xss512k
+-XX:+UseConcMarkSweepGC
+-XX:+CMSParallelRemarkEnabled
+-XX:+UseFastAccessorMethods
+-XX:+UseCMSInitiatingOccupancyOnly
-XX:CMSInitiatingOccupancyFraction=70
"
```
@@ -155,7 +160,7 @@ DolphinScheduler使用Zookeeper进行集群管理、容错、事件监听等功
默认配置如下：
-|参数 |默认值| 描述|
+|参数 |默认值| 描述|
|--|--|--|
|registry.zookeeper.namespace|dolphinscheduler|Zookeeper集群使用的namespace|
|registry.zookeeper.connect-string|localhost:2181| Zookeeper集群连接信息|
@@ -213,7 +218,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn相关的配置
## Api-server相关配置
位置：`api-server/conf/application.yaml`
-|参数 |默认值| 描述|
+|参数 |默认值| 描述|
|--|--|--|
|server.port|12345|api服务通讯端口|
|server.servlet.session.timeout|120m|session超时时间|
@@ -241,7 +246,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn相关的配置
## Master Server相关配置
位置：`master-server/conf/application.yaml`
-|参数 |默认值| 描述|
+|参数 |默认值| 描述|
|--|--|--|
|master.listen-port|5678|master监听端口|
|master.fetch-command-num|10|master拉取command数量|
@@ -262,7 +267,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn相关的配置
## Worker Server相关配置
位置：`worker-server/conf/application.yaml`
-|参数 |默认值| 描述|
+|参数 |默认值| 描述|
|--|--|--|
|worker.listen-port|1234|worker监听端口|
|worker.exec-threads|100|worker工作线程数量,用于限制并行的任务实例数量|
@@ -280,7 +285,7 @@ common.properties配置文件目前主要是配置hadoop/s3/yarn相关的配置
## Alert Server相关配置
位置：`alert-server/conf/application.yaml`
-|参数 |默认值| 描述|
+|参数 |默认值| 描述|
|--|--|--|
|server.port|50053|Alert Server监听端口|
|alert.port|50052|alert监听端口|
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml
index 6ed92bf63d..587619f9f1 100644
--- a/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/pom.xml
@@ -15,8 +15,7 @@
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
-
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0"
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -25,8 +24,8 @@
<version>dev-SNAPSHOT</version>
</parent>
<artifactId>dolphinscheduler-alert-server</artifactId>
- <name>${project.artifactId}</name>
<packaging>jar</packaging>
+ <name>${project.artifactId}</name>
<dependencies>
<!-- dolphinscheduler -->
@@ -93,6 +92,12 @@
<groupId>org.codehaus.janino</groupId>
<artifactId>janino</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>org.springframework.cloud</groupId>
+ <artifactId>spring-cloud-starter-kubernetes-fabric8-config</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
@@ -123,10 +128,10 @@
<executions>
<execution>
<id>dolphinscheduler-alert-server</id>
- <phase>package</phase>
<goals>
<goal>single</goal>
</goals>
+ <phase>package</phase>
<configuration>
<finalName>alert-server</finalName>
<descriptors>
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
index 40cb4c7ce5..1a65013090 100644
--- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
@@ -16,8 +16,6 @@
#
spring:
- application:
- name: alert-server
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"
diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/bootstrap.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/bootstrap.yaml
new file mode 100644
index 0000000000..347cdd07bb
--- /dev/null
+++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/bootstrap.yaml
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+spring:
+ application:
+ name: alert-server
+ cloud:
+ kubernetes:
+ reload:
+ enabled: true
+ mode: event
+ config:
+ sources:
+ - name: ${spring.application.name}
diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml
index 74de5d9b1f..1ded902ced 100644
--- a/dolphinscheduler-api/pom.xml
+++ b/dolphinscheduler-api/pom.xml
@@ -181,6 +181,11 @@
<artifactId>spring-ldap</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.springframework.cloud</groupId>
+ <artifactId>spring-cloud-starter-kubernetes-fabric8-config</artifactId>
+ </dependency>
+
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>
diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml
index 874e638c12..9a8381454b 100644
--- a/dolphinscheduler-api/src/main/resources/application.yaml
+++ b/dolphinscheduler-api/src/main/resources/application.yaml
@@ -28,8 +28,6 @@ server:
max-http-form-post-size: 5000000
spring:
- application:
- name: api-server
banner:
charset: UTF-8
jackson:
diff --git a/dolphinscheduler-api/src/main/resources/bootstrap.yaml b/dolphinscheduler-api/src/main/resources/bootstrap.yaml
new file mode 100644
index 0000000000..04804ab540
--- /dev/null
+++ b/dolphinscheduler-api/src/main/resources/bootstrap.yaml
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+spring:
+ application:
+ name: api-server
+ cloud:
+ kubernetes:
+ reload:
+ enabled: true
+ mode: event
+ config:
+ sources:
+ - name: ${spring.application.name}
diff --git a/dolphinscheduler-dist/release-docs/LICENSE b/dolphinscheduler-dist/release-docs/LICENSE
index 52c7758455..4b4ea79e77 100644
--- a/dolphinscheduler-dist/release-docs/LICENSE
+++ b/dolphinscheduler-dist/release-docs/LICENSE
@@ -521,6 +521,8 @@ The text of each license is also included at licenses/LICENSE-[project].txt.
oshi-core 6.1.1: https://mvnrepository.com/artifact/com.github.oshi/oshi-core/6.1.1, MIT
unirest-java 3.7.04-standalone: https://mvnrepository.com/artifact/com.konghq/unirest-java/3.7.04, MIT
classgraph 4.8.83: https://mvnrepository.com/artifact/io.github.classgraph/classgraph, MIT
+ bcpkix-jdk15on 1.68: https://mvnrepository.com/artifact/org.bouncycastle/bcpkix-jdk15on MIT
+ bcprov-jdk15on 1.68: https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15on MIT
========================================================================
MPL 1.1 licenses
diff --git a/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
index ea60fe918c..490f6d76eb 100644
--- a/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
+++ b/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
@@ -59,8 +59,6 @@ import org.springframework.stereotype.Component;
import io.netty.channel.Channel;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
/**
* logger request process logic
*/
@@ -80,7 +78,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
public void process(Channel channel, Command command) {
logger.info("received command : {}", command);
- //request task log command type
+ // request task log command type
final CommandType commandType = command.getType();
switch (commandType) {
case GET_LOG_BYTES_REQUEST:
@@ -120,7 +118,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
final int MaxResponseLogSize = 65535;
int totalLogByteSize = 0;
for (String line : lines) {
- //If a single line of log is exceed max response size, cut off the line
+ // If a single line of log is exceed max response size, cut off the line
final int lineByteSize = line.getBytes(StandardCharsets.UTF_8).length;
if (lineByteSize >= MaxResponseLogSize) {
builder.append(line, 0, MaxResponseLogSize)
@@ -136,7 +134,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
break;
}
}
- RollViewLogResponseCommand rollViewLogRequestResponse = new RollViewLogResponseCommand(builder.toString());
+ RollViewLogResponseCommand rollViewLogRequestResponse =
+ new RollViewLogResponseCommand(builder.toString());
channel.writeAndFlush(rollViewLogRequestResponse.convert2Command(command.getOpaque()));
break;
case REMOVE_TAK_LOG_REQUEST:
@@ -161,7 +160,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
channel.writeAndFlush(removeTaskLogResponse.convert2Command(command.getOpaque()));
break;
case GET_APP_ID_REQUEST:
- GetAppIdRequestCommand getAppIdRequestCommand = JSONUtils.parseObject(command.getBody(), GetAppIdRequestCommand.class);
+ GetAppIdRequestCommand getAppIdRequestCommand =
+ JSONUtils.parseObject(command.getBody(), GetAppIdRequestCommand.class);
String logPath = getAppIdRequestCommand.getLogPath();
if (!checkPathSecurity(logPath)) {
throw new IllegalArgumentException("Illegal path");
@@ -203,8 +203,9 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
* @return byte array of file
*/
private byte[] getFileContentBytes(String filePath) {
- try (InputStream in = new FileInputStream(filePath);
- ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
+ try (
+ InputStream in = new FileInputStream(filePath);
+ ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
byte[] buf = new byte[1024];
int len;
while ((len = in.read(buf)) != -1) {
diff --git a/dolphinscheduler-master/pom.xml b/dolphinscheduler-master/pom.xml
index 558741fea1..d204d987fd 100644
--- a/dolphinscheduler-master/pom.xml
+++ b/dolphinscheduler-master/pom.xml
@@ -18,18 +18,29 @@
~ under the License.
~
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
<parent>
- <artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler</artifactId>
<version>dev-SNAPSHOT</version>
</parent>
- <modelVersion>4.0.0</modelVersion>
<artifactId>dolphinscheduler-master</artifactId>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-bom</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
@@ -66,12 +77,12 @@
<artifactId>spring-boot-starter-cache</artifactId>
<exclusions>
<exclusion>
- <artifactId>log4j-api</artifactId>
<groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-api</artifactId>
</exclusion>
<exclusion>
- <artifactId>log4j-to-slf4j</artifactId>
<groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-to-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -93,8 +104,8 @@
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
- <artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.jackson</groupId>
@@ -118,20 +129,20 @@
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
</exclusion>
<exclusion>
- <artifactId>jersey-client</artifactId>
<groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
</exclusion>
<exclusion>
- <artifactId>jersey-core</artifactId>
<groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
- <artifactId>jaxb-api</artifactId>
<groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
</exclusion>
<exclusion>
- <artifactId>log4j</artifactId>
<groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -149,8 +160,8 @@
<artifactId>jdk.tools</artifactId>
</exclusion>
<exclusion>
- <artifactId>servlet-api</artifactId>
<groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet</groupId>
@@ -214,16 +225,16 @@
<artifactId>jsp-api</artifactId>
</exclusion>
<exclusion>
- <artifactId>jersey-json</artifactId>
<groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
</exclusion>
<exclusion>
- <artifactId>jersey-server</artifactId>
<groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
- <artifactId>jersey-core</artifactId>
<groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -240,25 +251,20 @@
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-worker</artifactId>
- <scope>test</scope> <!-- master should never depend on worker, this is only for tests -->
+ <scope>test</scope>
+ <!-- master should never depend on worker, this is only for tests -->
</dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-log-server</artifactId>
</dependency>
- </dependencies>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-bom</artifactId>
- <version>${project.version}</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
+ <dependency>
+ <groupId>org.springframework.cloud</groupId>
+ <artifactId>spring-cloud-starter-kubernetes-fabric8-config</artifactId>
+ </dependency>
+
+ </dependencies>
<build>
<plugins>
@@ -278,10 +284,10 @@
<executions>
<execution>
<id>dolphinscheduler-master-server</id>
- <phase>package</phase>
<goals>
<goal>single</goal>
</goals>
+ <phase>package</phase>
<configuration>
<finalName>master-server</finalName>
<descriptors>
diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml
index 4a00768278..1e5ba5d9b9 100644
--- a/dolphinscheduler-master/src/main/resources/application.yaml
+++ b/dolphinscheduler-master/src/main/resources/application.yaml
@@ -17,8 +17,6 @@
spring:
banner:
charset: UTF-8
- application:
- name: master-server
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"
diff --git a/dolphinscheduler-master/src/main/resources/bootstrap.yaml b/dolphinscheduler-master/src/main/resources/bootstrap.yaml
new file mode 100644
index 0000000000..896590736c
--- /dev/null
+++ b/dolphinscheduler-master/src/main/resources/bootstrap.yaml
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+spring:
+ application:
+ name: master-server
+ cloud:
+ kubernetes:
+ reload:
+ enabled: true
+ mode: event
+ config:
+ sources:
+ - name: ${spring.application.name}
diff --git a/dolphinscheduler-standalone-server/pom.xml b/dolphinscheduler-standalone-server/pom.xml
index 8b69742f9f..7bc450fd8a 100644
--- a/dolphinscheduler-standalone-server/pom.xml
+++ b/dolphinscheduler-standalone-server/pom.xml
@@ -77,10 +77,6 @@
</exclusions>
</dependency>
- <dependency>
- <groupId>org.springframework.cloud</groupId>
- <artifactId>spring-cloud-starter</artifactId>
- </dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-starter-kubernetes-fabric8-config</artifactId>
diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
index 02bd42b32b..5b4c6fdac3 100644
--- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml
+++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
@@ -16,8 +16,6 @@
#
spring:
- application:
- name: standalone-server
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"
diff --git a/dolphinscheduler-standalone-server/src/main/resources/bootstrap.yaml b/dolphinscheduler-standalone-server/src/main/resources/bootstrap.yaml
new file mode 100644
index 0000000000..6d20924c47
--- /dev/null
+++ b/dolphinscheduler-standalone-server/src/main/resources/bootstrap.yaml
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+spring:
+ application:
+ name: standalone-server
+ cloud:
+ kubernetes:
+ reload:
+ enabled: true
+ mode: event
+ config:
+ sources:
+ - name: ${spring.application.name}
diff --git a/dolphinscheduler-worker/pom.xml b/dolphinscheduler-worker/pom.xml
index d648d1d410..e558deb9c7 100644
--- a/dolphinscheduler-worker/pom.xml
+++ b/dolphinscheduler-worker/pom.xml
@@ -18,18 +18,29 @@
~ under the License.
~
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
<parent>
- <artifactId>dolphinscheduler</artifactId>
<groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler</artifactId>
<version>dev-SNAPSHOT</version>
</parent>
- <modelVersion>4.0.0</modelVersion>
<artifactId>dolphinscheduler-worker</artifactId>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.dolphinscheduler</groupId>
+ <artifactId>dolphinscheduler-bom</artifactId>
+ <version>${project.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<dependencies>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
@@ -66,8 +77,8 @@
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
<exclusion>
- <artifactId>log4j-to-slf4j</artifactId>
<groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-to-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
@@ -93,19 +104,13 @@
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-log-server</artifactId>
</dependency>
- </dependencies>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-bom</artifactId>
- <version>${project.version}</version>
- <type>pom</type>
- <scope>import</scope>
- </dependency>
- </dependencies>
- </dependencyManagement>
+ <dependency>
+ <groupId>org.springframework.cloud</groupId>
+ <artifactId>spring-cloud-starter-kubernetes-fabric8-config</artifactId>
+ </dependency>
+
+ </dependencies>
<build>
<plugins>
@@ -124,10 +129,10 @@
<executions>
<execution>
<id>dolphinscheduler-worker-server</id>
- <phase>package</phase>
<goals>
<goal>single</goal>
</goals>
+ <phase>package</phase>
<configuration>
<finalName>worker-server</finalName>
<descriptors>
diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml
index fb8a1cb704..4effad23b1 100644
--- a/dolphinscheduler-worker/src/main/resources/application.yaml
+++ b/dolphinscheduler-worker/src/main/resources/application.yaml
@@ -17,8 +17,6 @@
spring:
banner:
charset: UTF-8
- application:
- name: worker-server
jackson:
time-zone: UTC
date-format: "yyyy-MM-dd HH:mm:ss"
diff --git a/dolphinscheduler-worker/src/main/resources/bootstrap.yaml b/dolphinscheduler-worker/src/main/resources/bootstrap.yaml
new file mode 100644
index 0000000000..f89a64c473
--- /dev/null
+++ b/dolphinscheduler-worker/src/main/resources/bootstrap.yaml
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+spring:
+ application:
+ name: worker-server
+ cloud:
+ kubernetes:
+ reload:
+ enabled: true
+ mode: event
+ config:
+ sources:
+ - name: ${spring.application.name}
diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt
index b28957fd7a..57c4fb1c55 100755
--- a/tools/dependencies/known-dependencies.txt
+++ b/tools/dependencies/known-dependencies.txt
@@ -21,6 +21,8 @@ aws-java-sdk-emr-1.12.160.jar
aws-java-sdk-kms-1.12.160.jar
aws-java-sdk-s3-1.12.160.jar
aws-java-sdk-sagemaker-1.12.160.jar
+bcpkix-jdk15on-1.68.jar
+bcprov-jdk15on-1.68.jar
bonecp-0.8.0.RELEASE.jar
byte-buddy-1.9.16.jar
caffeine-2.9.3.jar
@@ -243,7 +245,13 @@ spring-boot-starter-json-2.7.3.jar
spring-boot-starter-logging-2.7.3.jar
spring-boot-starter-quartz-2.7.3.jar
spring-boot-starter-web-2.7.3.jar
+spring-cloud-commons-3.1.3.jar
+spring-cloud-context-3.1.3.jar
+spring-cloud-kubernetes-commons-2.1.3.jar
+spring-cloud-kubernetes-fabric8-autoconfig-2.1.3.jar
+spring-cloud-kubernetes-fabric8-config-2.1.3.jar
spring-cloud-starter-3.1.3.jar
+spring-cloud-starter-bootstrap-3.1.3.jar
spring-cloud-starter-kubernetes-fabric8-config-2.1.3.jar
spring-context-5.3.19.jar
spring-context-support-5.3.22.jar
@@ -254,6 +262,9 @@ spring-jdbc-5.3.19.jar
spring-ldap-1.1.2.jar
spring-plugin-core-2.0.0.RELEASE.jar
spring-plugin-metadata-2.0.0.RELEASE.jar
+spring-retry-1.3.3.jar
+spring-security-crypto-5.7.3.jar
+spring-security-rsa-1.0.10.RELEASE.jar
spring-tx-5.3.19.jar
spring-web-5.3.22.jar
spring-webmvc-5.3.22.jar