You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by li...@apache.org on 2021/02/14 09:42:14 UTC
[incubator-dolphinscheduler] branch dev updated:
[Improvement][Docker] Reduce docker image size, remove frontend/nginx,
remove stdout log,
improve compose/swarm deployment and fix random PSQLException (#4643)
This is an automated email from the ASF dual-hosted git repository.
lidongdai pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new cbada7c [Improvement][Docker] Reduce docker image size, remove frontend/nginx, remove stdout log, improve compose/swarm deployment and fix random PSQLException (#4643)
cbada7c is described below
commit cbada7c16de4c5b853aada6765bd352ff5262416
Author: Shiwen Cheng <ch...@gmail.com>
AuthorDate: Sun Feb 14 17:42:04 2021 +0800
[Improvement][Docker] Reduce docker image size, remove frontend/nginx, remove stdout log, improve compose/swarm deployment and fix random PSQLException (#4643)
* [Improvement][Docker] Reduce docker image size, remove frontend/nginx, remove stdout log and improve compose/swarm deployment
* [Improvement][Docker] Fix random PSQLException
---
.gitignore | 4 ++
docker/build/Dockerfile | 57 +++++----------
docker/build/README.md | 36 ++--------
docker/build/README_zh_CN.md | 35 ++-------
docker/build/checkpoint.sh | 0
.../dolphinscheduler/env/dolphinscheduler_env.sh | 12 +++-
.../dolphinscheduler/logback/logback-alert.xml | 9 ---
.../conf/dolphinscheduler/logback/logback-api.xml | 9 ---
.../dolphinscheduler/logback/logback-master.xml | 9 ---
.../dolphinscheduler/logback/logback-worker.xml | 9 ---
docker/build/conf/nginx/dolphinscheduler.conf | 51 -------------
docker/build/conf/zookeeper/zoo.cfg | 47 ------------
docker/build/hooks/build | 9 ++-
docker/build/hooks/push | 0
docker/build/startup-init-conf.sh | 18 ++---
docker/build/startup.sh | 55 ++++++--------
docker/docker-swarm/check | 2 +-
docker/docker-swarm/docker-compose.yml | 80 ++++++++++-----------
docker/docker-swarm/docker-stack.yml | 84 ++++++++++------------
docker/docker-swarm/dolphinscheduler_env.sh | 12 +++-
dolphinscheduler-ui/install-dolphinscheduler-ui.sh | 0
e2e/src/test/resources/config/config.properties | 2 +-
script/create-dolphinscheduler.sh | 0
script/dolphinscheduler-daemon.sh | 6 ++
script/env/dolphinscheduler_env.sh | 0
script/monitor-server.sh | 0
script/scp-hosts.sh | 0
script/start-all.sh | 0
script/status-all.sh | 0
script/stop-all.sh | 0
script/upgrade-dolphinscheduler.sh | 0
31 files changed, 170 insertions(+), 376 deletions(-)
diff --git a/.gitignore b/.gitignore
index 6717162..9011db1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,6 +7,10 @@
.target
.idea/
target/
+dist/
+all-dependencies.txt
+self-modules.txt
+third-party-dependencies.txt
.settings
.nbproject
.classpath
diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile
index 7b0f09c..a33cf49 100644
--- a/docker/build/Dockerfile
+++ b/docker/build/Dockerfile
@@ -15,55 +15,37 @@
# limitations under the License.
#
-FROM nginx:alpine
+FROM openjdk:8-jdk-alpine
ARG VERSION
ENV TZ Asia/Shanghai
ENV LANG C.UTF-8
-ENV DEBIAN_FRONTEND noninteractive
+ENV DOCKER true
-#1. install dos2unix shadow bash openrc python sudo vim wget iputils net-tools ssh pip tini kazoo.
-#If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example:
-#RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories
+# 1. install command/library/software
+# If install slowly, you can replcae alpine's mirror with aliyun's mirror, Example:
+# RUN sed -i "s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g" /etc/apk/repositories
+# RUN sed -i 's/dl-cdn.alpinelinux.org/mirror.tuna.tsinghua.edu.cn/g' /etc/apk/repositories
RUN apk update && \
- apk add --update --no-cache dos2unix shadow bash openrc python2 python3 sudo vim wget iputils net-tools openssh-server py-pip tini && \
- apk add --update --no-cache procps && \
- openrc boot && \
- pip install kazoo
+ apk add --no-cache tzdata dos2unix bash python2 python3 procps sudo shadow tini postgresql-client && \
+ cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
+ apk del tzdata && \
+ rm -rf /var/cache/apk/*
-#2. install jdk
-RUN apk add --update --no-cache openjdk8
-ENV JAVA_HOME /usr/lib/jvm/java-1.8-openjdk
-ENV PATH $JAVA_HOME/bin:$PATH
-
-#3. add dolphinscheduler
+# 2. add dolphinscheduler
ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/
-RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/
+RUN ln -s /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin /opt/dolphinscheduler
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
-#4. install database, if use mysql as your backend database, the `mysql-client` package should be installed
-RUN apk add --update --no-cache postgresql postgresql-contrib
-
-#5. modify nginx
-RUN echo "daemon off;" >> /etc/nginx/nginx.conf && \
- rm -rf /etc/nginx/conf.d/*
-COPY ./conf/nginx/dolphinscheduler.conf /etc/nginx/conf.d
-
-#6. add configuration and modify permissions and set soft links
+# 3. add configuration and modify permissions and set soft links
COPY ./checkpoint.sh /root/checkpoint.sh
COPY ./startup-init-conf.sh /root/startup-init-conf.sh
COPY ./startup.sh /root/startup.sh
COPY ./conf/dolphinscheduler/*.tpl /opt/dolphinscheduler/conf/
COPY ./conf/dolphinscheduler/logback/* /opt/dolphinscheduler/conf/
-COPY conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/
-RUN chmod +x /root/checkpoint.sh && \
- chmod +x /root/startup-init-conf.sh && \
- chmod +x /root/startup.sh && \
- chmod +x /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
- chmod +x /opt/dolphinscheduler/script/*.sh && \
- chmod +x /opt/dolphinscheduler/bin/*.sh && \
- dos2unix /root/checkpoint.sh && \
+COPY ./conf/dolphinscheduler/env/dolphinscheduler_env.sh /opt/dolphinscheduler/conf/env/
+RUN dos2unix /root/checkpoint.sh && \
dos2unix /root/startup-init-conf.sh && \
dos2unix /root/startup.sh && \
dos2unix /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh && \
@@ -71,13 +53,10 @@ RUN chmod +x /root/checkpoint.sh && \
dos2unix /opt/dolphinscheduler/bin/*.sh && \
rm -rf /bin/sh && \
ln -s /bin/bash /bin/sh && \
- mkdir -p /tmp/xls && \
- #7. remove apk index cache and disable coredup for sudo
- rm -rf /var/cache/apk/* && \
+ mkdir -p /var/mail /tmp/xls && \
echo "Set disable_coredump false" >> /etc/sudo.conf
-
-#8. expose port
-EXPOSE 2181 2888 3888 5432 5678 1234 12345 50051 8888
+# 4. expose port
+EXPOSE 5678 1234 12345 50051
ENTRYPOINT ["/sbin/tini", "--", "/root/startup.sh"]
diff --git a/docker/build/README.md b/docker/build/README.md
index 951f2d6..3a2a166 100644
--- a/docker/build/README.md
+++ b/docker/build/README.md
@@ -15,9 +15,9 @@ Official Website: https://dolphinscheduler.apache.org
#### You can start a dolphinscheduler instance
```
-$ docker run -dit --name dolphinscheduler \
+$ docker run -dit --name dolphinscheduler \
-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \
--p 8888:8888 \
+-p 12345:12345 \
dolphinscheduler all
```
@@ -33,7 +33,7 @@ You can specify **existing postgres service**. Example:
$ docker run -dit --name dolphinscheduler \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
--p 8888:8888 \
+-p 12345:12345 \
dolphinscheduler all
```
@@ -43,7 +43,7 @@ You can specify **existing zookeeper service**. Example:
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \
--p 8888:8888 \
+-p 12345:12345 \
dolphinscheduler all
```
@@ -90,14 +90,6 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler alert-server
```
-* Start a **frontend**, For example:
-
-```
-$ docker run -dit --name dolphinscheduler \
--e FRONTEND_API_SERVER_HOST="192.168.x.x" -e FRONTEND_API_SERVER_PORT="12345" \
--p 8888:8888 \
-dolphinscheduler frontend
-```
**Note**: You must be specify `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM` when start a standalone dolphinscheduler server.
@@ -146,7 +138,7 @@ This environment variable sets the host for database. The default value is `127.
This environment variable sets the port for database. The default value is `5432`.
-**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
+**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `master-server`, `worker-server`, `api-server`, `alert-server`.
**`DATABASE_USERNAME`**
@@ -306,18 +298,6 @@ This environment variable sets enterprise wechat agent id for `alert-server`. Th
This environment variable sets enterprise wechat users for `alert-server`. The default value is empty.
-**`FRONTEND_API_SERVER_HOST`**
-
-This environment variable sets api server host for `frontend`. The default value is `127.0.0.1`.
-
-**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `api-server`.
-
-**`FRONTEND_API_SERVER_PORT`**
-
-This environment variable sets api server port for `frontend`. The default value is `123451`.
-
-**Note**: You must be specify it when start a standalone dolphinscheduler server. Like `api-server`.
-
## Initialization scripts
If you would like to do additional initialization in an image derived from this one, add one or more environment variable under `/root/start-init-conf.sh`, and modify template files in `/opt/dolphinscheduler/conf/*.tpl`.
@@ -326,7 +306,7 @@ For example, to add an environment variable `API_SERVER_PORT` in `/root/start-in
```
export API_SERVER_PORT=5555
-```
+```
and to modify `/opt/dolphinscheduler/conf/application-api.properties.tpl` template file, add server port:
```
@@ -343,8 +323,4 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
EOF
" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
done
-
-echo "generate nginx config"
-sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf
-sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf
```
diff --git a/docker/build/README_zh_CN.md b/docker/build/README_zh_CN.md
index c4339a9..ffa0020 100644
--- a/docker/build/README_zh_CN.md
+++ b/docker/build/README_zh_CN.md
@@ -15,9 +15,9 @@ Official Website: https://dolphinscheduler.apache.org
#### 你可以运行一个dolphinscheduler实例
```
-$ docker run -dit --name dolphinscheduler \
+$ docker run -dit --name dolphinscheduler \
-e DATABASE_USERNAME=test -e DATABASE_PASSWORD=test -e DATABASE_DATABASE=dolphinscheduler \
--p 8888:8888 \
+-p 12345:12345 \
dolphinscheduler all
```
@@ -33,7 +33,7 @@ dolphinscheduler all
$ docker run -dit --name dolphinscheduler \
-e DATABASE_HOST="192.168.x.x" -e DATABASE_PORT="5432" -e DATABASE_DATABASE="dolphinscheduler" \
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" \
--p 8888:8888 \
+-p 12345:12345 \
dolphinscheduler all
```
@@ -43,7 +43,7 @@ dolphinscheduler all
$ docker run -dit --name dolphinscheduler \
-e ZOOKEEPER_QUORUM="l92.168.x.x:2181"
-e DATABASE_USERNAME="test" -e DATABASE_PASSWORD="test" -e DATABASE_DATABASE="dolphinscheduler" \
--p 8888:8888 \
+-p 12345:12345 \
dolphinscheduler all
```
@@ -90,15 +90,6 @@ $ docker run -dit --name dolphinscheduler \
dolphinscheduler alert-server
```
-* 启动一个 **frontend**, 如下:
-
-```
-$ docker run -dit --name dolphinscheduler \
--e FRONTEND_API_SERVER_HOST="192.168.x.x" -e FRONTEND_API_SERVER_PORT="12345" \
--p 8888:8888 \
-dolphinscheduler frontend
-```
-
**注意**: 当你运行dolphinscheduler中的部分服务时,你必须指定这些环境变量 `DATABASE_HOST` `DATABASE_PORT` `DATABASE_DATABASE` `DATABASE_USERNAME` `DATABASE_PASSWORD` `ZOOKEEPER_QUORUM`。
## 如何构建一个docker镜像
@@ -306,18 +297,6 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些
配置`alert-server`的邮件服务企业微信`USERS`,默认值 `空`。
-**`FRONTEND_API_SERVER_HOST`**
-
-配置`frontend`的连接`api-server`的地址,默认值 `127.0.0.1`。
-
-**Note**: 当单独运行`api-server`时,你应该指定`api-server`这个值。
-
-**`FRONTEND_API_SERVER_PORT`**
-
-配置`frontend`的连接`api-server`的端口,默认值 `12345`。
-
-**Note**: 当单独运行`api-server`时,你应该指定`api-server`这个值。
-
## 初始化脚本
如果你想在编译的时候或者运行的时候附加一些其它的操作及新增一些环境变量,你可以在`/root/start-init-conf.sh`文件中进行修改,同时如果涉及到配置文件的修改,请在`/opt/dolphinscheduler/conf/*.tpl`中修改相应的配置文件
@@ -326,7 +305,7 @@ Dolphin Scheduler映像使用了几个容易遗漏的环境变量。虽然这些
```
export API_SERVER_PORT=5555
-```
+```
当添加以上环境变量后,你应该在相应的模板文件`/opt/dolphinscheduler/conf/application-api.properties.tpl`中添加这个环境变量配置:
```
@@ -343,8 +322,4 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
EOF
" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
done
-
-echo "generate nginx config"
-sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf
-sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf
```
diff --git a/docker/build/checkpoint.sh b/docker/build/checkpoint.sh
old mode 100644
new mode 100755
diff --git a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh
old mode 100644
new mode 100755
index 78c8f98..7fd3933
--- a/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh
+++ b/docker/build/conf/dolphinscheduler/env/dolphinscheduler_env.sh
@@ -15,6 +15,14 @@
# limitations under the License.
#
-export PYTHON_HOME=/usr/bin/python2
+export HADOOP_HOME=/opt/soft/hadoop
+export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
+export SPARK_HOME1=/opt/soft/spark1
+export SPARK_HOME2=/opt/soft/spark2
+export PYTHON_HOME=/usr/bin/python
export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk
-export PATH=$PYTHON_HOME/bin:$JAVA_HOME/bin:$PATH
+export HIVE_HOME=/opt/soft/hive
+export FLINK_HOME=/opt/soft/flink
+export DATAX_HOME=/opt/soft/datax/bin/datax.py
+
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml
index 1718947..eec7838 100644
--- a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml
@@ -20,14 +20,6 @@
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
- </pattern>
- <charset>UTF-8</charset>
- </encoder>
- </appender>
<appender name="ALERTLOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${log.base}/dolphinscheduler-alert.log</file>
@@ -45,7 +37,6 @@
</appender>
<root level="INFO">
- <appender-ref ref="STDOUT"/>
<appender-ref ref="ALERTLOGFILE"/>
</root>
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-api.xml b/docker/build/conf/dolphinscheduler/logback/logback-api.xml
index 2df90d8..6d29f8a 100644
--- a/docker/build/conf/dolphinscheduler/logback/logback-api.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-api.xml
@@ -20,14 +20,6 @@
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
- </pattern>
- <charset>UTF-8</charset>
- </encoder>
- </appender>
<!-- api server logback config start -->
<appender name="APILOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
@@ -55,7 +47,6 @@
<root level="INFO">
- <appender-ref ref="STDOUT"/>
<appender-ref ref="APILOGFILE"/>
</root>
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-master.xml b/docker/build/conf/dolphinscheduler/logback/logback-master.xml
index 7410c01..d1bfb67 100644
--- a/docker/build/conf/dolphinscheduler/logback/logback-master.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-master.xml
@@ -20,14 +20,6 @@
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
- </pattern>
- <charset>UTF-8</charset>
- </encoder>
- </appender>
<conversionRule conversionWord="messsage"
converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
@@ -74,7 +66,6 @@
<!-- master server logback config end -->
<root level="INFO">
- <appender-ref ref="STDOUT"/>
<appender-ref ref="TASKLOGFILE"/>
<appender-ref ref="MASTERLOGFILE"/>
</root>
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml
index be1d0ac..b7e08dd 100644
--- a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml
@@ -20,14 +20,6 @@
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
<property name="log.base" value="logs"/>
- <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
- <encoder>
- <pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
- </pattern>
- <charset>UTF-8</charset>
- </encoder>
- </appender>
<!-- worker server logback config start -->
<conversionRule conversionWord="messsage"
@@ -75,7 +67,6 @@
<!-- worker server logback config end -->
<root level="INFO">
- <appender-ref ref="STDOUT"/>
<appender-ref ref="TASKLOGFILE"/>
<appender-ref ref="WORKERLOGFILE"/>
</root>
diff --git a/docker/build/conf/nginx/dolphinscheduler.conf b/docker/build/conf/nginx/dolphinscheduler.conf
deleted file mode 100644
index a594385..0000000
--- a/docker/build/conf/nginx/dolphinscheduler.conf
+++ /dev/null
@@ -1,51 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-server {
- listen 8888;
- server_name localhost;
- #charset koi8-r;
- #access_log /var/log/nginx/host.access.log main;
- location / {
- root /opt/dolphinscheduler/ui;
- index index.html index.html;
- }
- location /dolphinscheduler/ui{
- alias /opt/dolphinscheduler/ui;
- }
- location /dolphinscheduler {
- proxy_pass http://FRONTEND_API_SERVER_HOST:FRONTEND_API_SERVER_PORT;
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header x_real_ipP $remote_addr;
- proxy_set_header remote_addr $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_http_version 1.1;
- proxy_connect_timeout 300s;
- proxy_read_timeout 300s;
- proxy_send_timeout 300s;
- proxy_set_header Upgrade $http_upgrade;
- proxy_set_header Connection "upgrade";
- }
- #error_page 404 /404.html;
- # redirect server error pages to the static page /50x.html
- #
- error_page 500 502 503 504 /50x.html;
- location = /50x.html {
- root /usr/share/nginx/html;
- }
-}
diff --git a/docker/build/conf/zookeeper/zoo.cfg b/docker/build/conf/zookeeper/zoo.cfg
deleted file mode 100644
index 94f92d0..0000000
--- a/docker/build/conf/zookeeper/zoo.cfg
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# The number of milliseconds of each tick
-tickTime=2000
-# The number of ticks that the initial
-# synchronization phase can take
-initLimit=10
-# The number of ticks that can pass between
-# sending a request and getting an acknowledgement
-syncLimit=5
-# the directory where the snapshot is stored.
-# do not use /tmp for storage, /tmp here is just
-# example sakes.
-dataDir=/tmp/zookeeper
-# the port at which the clients will connect
-clientPort=2181
-# the maximum number of client connections.
-# increase this if you need to handle more clients
-#maxClientCnxns=60
-#
-# Be sure to read the maintenance section of the
-# administrator guide before turning on autopurge.
-#
-# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
-#
-# The number of snapshots to retain in dataDir
-#autopurge.snapRetainCount=3
-# Purge task interval in hours
-# Set to "0" to disable auto purge feature
-#autopurge.purgeInterval=1
-#Four Letter Words commands:stat,ruok,conf,isro
-4lw.commands.whitelist=*
diff --git a/docker/build/hooks/build b/docker/build/hooks/build
old mode 100644
new mode 100755
index 6362fdd..ce7362b
--- a/docker/build/hooks/build
+++ b/docker/build/hooks/build
@@ -48,7 +48,12 @@ echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubati
mv "$(pwd)"/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-"${VERSION}"-dolphinscheduler-bin.tar.gz $(pwd)/docker/build/
# docker build
-echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/\n"
-sudo docker build --build-arg VERSION="${VERSION}" -t $DOCKER_REPO:"${VERSION}" "$(pwd)/docker/build/"
+BUILD_COMMAND="docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/docker/build/"
+echo -e "$BUILD_COMMAND\n"
+if (docker info 2> /dev/null | grep -i "ERROR"); then
+ sudo $BUILD_COMMAND
+else
+ $BUILD_COMMAND
+fi
echo "------ dolphinscheduler end - build -------"
diff --git a/docker/build/hooks/push b/docker/build/hooks/push
old mode 100644
new mode 100755
diff --git a/docker/build/startup-init-conf.sh b/docker/build/startup-init-conf.sh
old mode 100644
new mode 100755
index d5cd86f..c3aadcd
--- a/docker/build/startup-init-conf.sh
+++ b/docker/build/startup-init-conf.sh
@@ -39,9 +39,9 @@ export DATABASE_PARAMS=${DATABASE_PARAMS:-"characterEncoding=utf8"}
export DOLPHINSCHEDULER_ENV_PATH=${DOLPHINSCHEDULER_ENV_PATH:-"/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"}
export DOLPHINSCHEDULER_DATA_BASEDIR_PATH=${DOLPHINSCHEDULER_DATA_BASEDIR_PATH:-"/tmp/dolphinscheduler"}
export DOLPHINSCHEDULER_OPTS=${DOLPHINSCHEDULER_OPTS:-""}
-export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"NONE"}
-export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/ds"}
-export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"s3a://xxxx"}
+export RESOURCE_STORAGE_TYPE=${RESOURCE_STORAGE_TYPE:-"HDFS"}
+export RESOURCE_UPLOAD_PATH=${RESOURCE_UPLOAD_PATH:-"/dolphinscheduler"}
+export FS_DEFAULT_FS=${FS_DEFAULT_FS:-"file:///"}
export FS_S3A_ENDPOINT=${FS_S3A_ENDPOINT:-"s3.xxx.amazonaws.com"}
export FS_S3A_ACCESS_KEY=${FS_S3A_ACCESS_KEY:-"xxxxxxx"}
export FS_S3A_SECRET_KEY=${FS_S3A_SECRET_KEY:-"xxxxxxx"}
@@ -81,7 +81,7 @@ export WORKER_WEIGHT=${WORKER_WEIGHT:-"100"}
#============================================================================
# alert plugin dir
export ALERT_PLUGIN_DIR=${ALERT_PLUGIN_DIR:-"/opt/dolphinscheduler"}
-# XLS FILE
+# xls file
export XLS_FILE_PATH=${XLS_FILE_PATH:-"/tmp/xls"}
# mail
export MAIL_SERVER_HOST=${MAIL_SERVER_HOST:-""}
@@ -99,12 +99,6 @@ export ENTERPRISE_WECHAT_SECRET=${ENTERPRISE_WECHAT_SECRET:-""}
export ENTERPRISE_WECHAT_AGENT_ID=${ENTERPRISE_WECHAT_AGENT_ID:-""}
export ENTERPRISE_WECHAT_USERS=${ENTERPRISE_WECHAT_USERS:-""}
-#============================================================================
-# Frontend
-#============================================================================
-export FRONTEND_API_SERVER_HOST=${FRONTEND_API_SERVER_HOST:-"127.0.0.1"}
-export FRONTEND_API_SERVER_PORT=${FRONTEND_API_SERVER_PORT:-"12345"}
-
echo "generate app config"
ls ${DOLPHINSCHEDULER_HOME}/conf/ | grep ".tpl" | while read line; do
eval "cat << EOF
@@ -112,7 +106,3 @@ $(cat ${DOLPHINSCHEDULER_HOME}/conf/${line})
EOF
" > ${DOLPHINSCHEDULER_HOME}/conf/${line%.*}
done
-
-echo "generate nginx config"
-sed -i "s/FRONTEND_API_SERVER_HOST/${FRONTEND_API_SERVER_HOST}/g" /etc/nginx/conf.d/dolphinscheduler.conf
-sed -i "s/FRONTEND_API_SERVER_PORT/${FRONTEND_API_SERVER_PORT}/g" /etc/nginx/conf.d/dolphinscheduler.conf
\ No newline at end of file
diff --git a/docker/build/startup.sh b/docker/build/startup.sh
old mode 100644
new mode 100755
index 0511788..8bd1895
--- a/docker/build/startup.sh
+++ b/docker/build/startup.sh
@@ -22,8 +22,8 @@ DOLPHINSCHEDULER_BIN=${DOLPHINSCHEDULER_HOME}/bin
DOLPHINSCHEDULER_SCRIPT=${DOLPHINSCHEDULER_HOME}/script
DOLPHINSCHEDULER_LOGS=${DOLPHINSCHEDULER_HOME}/logs
-# start database
-initDatabase() {
+# wait database
+waitDatabase() {
echo "test ${DATABASE_TYPE} service"
while ! nc -z ${DATABASE_HOST} ${DATABASE_PORT}; do
counter=$((counter+1))
@@ -43,19 +43,22 @@ initDatabase() {
exit 1
fi
else
- v=$(sudo -u postgres PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1")
+ v=$(PGPASSWORD=${DATABASE_PASSWORD} psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USERNAME} -d ${DATABASE_DATABASE} -tAc "select 1")
if [ "$(echo ${v} | grep 'FATAL' | wc -l)" -eq 1 ]; then
echo "Error: Can't connect to database...${v}"
exit 1
fi
fi
+}
+# init database
+initDatabase() {
echo "import sql data"
${DOLPHINSCHEDULER_SCRIPT}/create-dolphinscheduler.sh
}
-# start zk
-initZK() {
+# wait zk
+waitZK() {
echo "connect remote zookeeper"
echo "${ZOOKEEPER_QUORUM}" | awk -F ',' 'BEGIN{ i=1 }{ while( i <= NF ){ print $i; i++ } }' | while read line; do
while ! nc -z ${line%:*} ${line#*:}; do
@@ -70,12 +73,6 @@ initZK() {
done
}
-# start nginx
-initNginx() {
- echo "start nginx"
- nginx &
-}
-
# start master-server
initMasterServer() {
echo "start master-server"
@@ -115,59 +112,54 @@ initAlertServer() {
printUsage() {
echo -e "Dolphin Scheduler is a distributed and easy-to-expand visual DAG workflow scheduling system,"
echo -e "dedicated to solving the complex dependencies in data processing, making the scheduling system out of the box for data processing.\n"
- echo -e "Usage: [ all | master-server | worker-server | api-server | alert-server | frontend ]\n"
- printf "%-13s: %s\n" "all" "Run master-server, worker-server, api-server, alert-server and frontend."
+ echo -e "Usage: [ all | master-server | worker-server | api-server | alert-server ]\n"
+ printf "%-13s: %s\n" "all" "Run master-server, worker-server, api-server and alert-server"
printf "%-13s: %s\n" "master-server" "MasterServer is mainly responsible for DAG task split, task submission monitoring."
- printf "%-13s: %s\n" "worker-server" "WorkerServer is mainly responsible for task execution and providing log services.."
- printf "%-13s: %s\n" "api-server" "ApiServer is mainly responsible for processing requests from the front-end UI layer."
+ printf "%-13s: %s\n" "worker-server" "WorkerServer is mainly responsible for task execution and providing log services."
+ printf "%-13s: %s\n" "api-server" "ApiServer is mainly responsible for processing requests and providing the front-end UI layer."
printf "%-13s: %s\n" "alert-server" "AlertServer mainly include Alarms."
- printf "%-13s: %s\n" "frontend" "Frontend mainly provides various visual operation interfaces of the system."
}
# init config file
source /root/startup-init-conf.sh
-LOGFILE=/var/log/nginx/access.log
case "$1" in
(all)
- initZK
+ waitZK
+ waitDatabase
initDatabase
initMasterServer
initWorkerServer
initApiServer
initAlertServer
initLoggerServer
- initNginx
- LOGFILE=/var/log/nginx/access.log
+ LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log
;;
(master-server)
- initZK
- initDatabase
+ waitZK
+ waitDatabase
initMasterServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-master.log
;;
(worker-server)
- initZK
- initDatabase
+ waitZK
+ waitDatabase
initWorkerServer
initLoggerServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-worker.log
;;
(api-server)
- initZK
+ waitZK
+ waitDatabase
initDatabase
initApiServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-api-server.log
;;
(alert-server)
- initDatabase
+ waitDatabase
initAlertServer
LOGFILE=${DOLPHINSCHEDULER_LOGS}/dolphinscheduler-alert.log
;;
- (frontend)
- initNginx
- LOGFILE=/var/log/nginx/access.log
- ;;
(help)
printUsage
exit 1
@@ -179,8 +171,7 @@ case "$1" in
esac
# init directories and log files
-mkdir -p ${DOLPHINSCHEDULER_LOGS} && mkdir -p /var/log/nginx/ && cat /dev/null >> ${LOGFILE}
+mkdir -p ${DOLPHINSCHEDULER_LOGS} && cat /dev/null >> ${LOGFILE}
echo "tail begin"
exec bash -c "tail -n 1 -f ${LOGFILE}"
-
diff --git a/docker/docker-swarm/check b/docker/docker-swarm/check
old mode 100644
new mode 100755
index 59203c5..f409366
--- a/docker/docker-swarm/check
+++ b/docker/docker-swarm/check
@@ -25,7 +25,7 @@ else
echo "Server start failed "$server_num
exit 1
fi
-ready=`curl http://127.0.0.1:8888/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l`
+ready=`curl http://127.0.0.1:12345/dolphinscheduler/login -d 'userName=admin&userPassword=dolphinscheduler123' -v | grep "login success" | wc -l`
if [ $ready -eq 1 ]
then
echo "Servers is ready"
diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml
index 349b3ad..04a9bc5 100644
--- a/docker/docker-swarm/docker-compose.yml
+++ b/docker/docker-swarm/docker-compose.yml
@@ -31,6 +31,7 @@ services:
volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql
- dolphinscheduler-postgresql-initdb:/docker-entrypoint-initdb.d
+ restart: unless-stopped
networks:
- dolphinscheduler
@@ -45,13 +46,14 @@ services:
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper
+ restart: unless-stopped
networks:
- dolphinscheduler
dolphinscheduler-api:
image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-api
- command: ["api-server"]
+ command: api-server
ports:
- 12345:12345
environment:
@@ -62,6 +64,9 @@ services:
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+ RESOURCE_STORAGE_TYPE: HDFS
+ RESOURCE_UPLOAD_PATH: /dolphinscheduler
+ FS_DEFAULT_FS: file:///
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
interval: 30s
@@ -72,37 +77,16 @@ services:
- dolphinscheduler-postgresql
- dolphinscheduler-zookeeper
volumes:
- - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs
- networks:
- - dolphinscheduler
-
- dolphinscheduler-frontend:
- image: apache/dolphinscheduler:latest
- container_name: dolphinscheduler-frontend
- command: ["frontend"]
- ports:
- - 8888:8888
- environment:
- TZ: Asia/Shanghai
- FRONTEND_API_SERVER_HOST: dolphinscheduler-api
- FRONTEND_API_SERVER_PORT: 12345
- healthcheck:
- test: ["CMD", "nc", "-z", "localhost", "8888"]
- interval: 30s
- timeout: 5s
- retries: 3
- start_period: 30s
- depends_on:
- - dolphinscheduler-api
- volumes:
- - ./dolphinscheduler-logs:/var/log/nginx
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-resource-local:/dolphinscheduler
+ restart: unless-stopped
networks:
- dolphinscheduler
dolphinscheduler-alert:
image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-alert
- command: ["alert-server"]
+ command: alert-server
environment:
TZ: Asia/Shanghai
XLS_FILE_PATH: "/tmp/xls"
@@ -133,14 +117,15 @@ services:
depends_on:
- dolphinscheduler-postgresql
volumes:
- - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ restart: unless-stopped
networks:
- dolphinscheduler
dolphinscheduler-master:
image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-master
- command: ["master-server"]
+ command: master-server
ports:
- 5678:5678
environment:
@@ -168,14 +153,15 @@ services:
- dolphinscheduler-postgresql
- dolphinscheduler-zookeeper
volumes:
- - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ restart: unless-stopped
networks:
- dolphinscheduler
dolphinscheduler-worker:
image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-worker
- command: ["worker-server"]
+ command: worker-server
ports:
- 1234:1234
- 50051:50051
@@ -188,30 +174,40 @@ services:
WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUP: "default"
WORKER_WEIGHT: "100"
- DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
+ DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
+ XLS_FILE_PATH: "/tmp/xls"
+ MAIL_SERVER_HOST: ""
+ MAIL_SERVER_PORT: ""
+ MAIL_SENDER: ""
+ MAIL_USER: ""
+ MAIL_PASSWD: ""
+ MAIL_SMTP_STARTTLS_ENABLE: "false"
+ MAIL_SMTP_SSL_ENABLE: "false"
+ MAIL_SMTP_SSL_TRUST: ""
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+ RESOURCE_STORAGE_TYPE: HDFS
+ RESOURCE_UPLOAD_PATH: /dolphinscheduler
+ FS_DEFAULT_FS: file:///
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
interval: 30s
timeout: 5s
retries: 3
start_period: 30s
- depends_on:
+ depends_on:
- dolphinscheduler-postgresql
- dolphinscheduler-zookeeper
volumes:
- - type: bind
- source: ./dolphinscheduler_env.sh
- target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
- - type: volume
- source: dolphinscheduler-worker-data
- target: /tmp/dolphinscheduler
- - ./dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - ./dolphinscheduler_env.sh:/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
+ - dolphinscheduler-worker-data:/tmp/dolphinscheduler
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-resource-local:/dolphinscheduler
+ restart: unless-stopped
networks:
- dolphinscheduler
@@ -224,7 +220,5 @@ volumes:
dolphinscheduler-postgresql-initdb:
dolphinscheduler-zookeeper:
dolphinscheduler-worker-data:
-
-configs:
- dolphinscheduler-worker-task-env:
- file: ./dolphinscheduler_env.sh
\ No newline at end of file
+ dolphinscheduler-logs:
+ dolphinscheduler-resource-local:
\ No newline at end of file
diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml
index dff4a47..39c3634 100644
--- a/docker/docker-swarm/docker-stack.yml
+++ b/docker/docker-swarm/docker-stack.yml
@@ -20,13 +20,13 @@ services:
dolphinscheduler-postgresql:
image: bitnami/postgresql:latest
+ ports:
+ - 5432:5432
environment:
TZ: Asia/Shanghai
POSTGRESQL_USERNAME: root
POSTGRESQL_PASSWORD: root
POSTGRESQL_DATABASE: dolphinscheduler
- ports:
- - 5432:5432
volumes:
- dolphinscheduler-postgresql:/bitnami/postgresql
networks:
@@ -37,12 +37,12 @@ services:
dolphinscheduler-zookeeper:
image: bitnami/zookeeper:latest
+ ports:
+ - 2181:2181
environment:
TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes"
ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
- ports:
- - 2181:2181
volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper
networks:
@@ -53,7 +53,9 @@ services:
dolphinscheduler-api:
image: apache/dolphinscheduler:latest
- command: ["api-server"]
+ command: api-server
+ ports:
+ - 12345:12345
environment:
TZ: Asia/Shanghai
DATABASE_HOST: dolphinscheduler-postgresql
@@ -62,39 +64,17 @@ services:
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
- ports:
- - 12345:12345
+ RESOURCE_STORAGE_TYPE: HDFS
+ RESOURCE_UPLOAD_PATH: /dolphinscheduler
+ FS_DEFAULT_FS: file:///
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "ApiApplicationServer"]
- interval: 30
+ interval: 30s
timeout: 5s
retries: 3
start_period: 30s
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- networks:
- - dolphinscheduler
- deploy:
- mode: replicated
- replicas: 1
-
- dolphinscheduler-frontend:
- image: apache/dolphinscheduler:latest
- command: ["frontend"]
- ports:
- - 8888:8888
- environment:
- TZ: Asia/Shanghai
- FRONTEND_API_SERVER_HOST: dolphinscheduler-api
- FRONTEND_API_SERVER_PORT: 12345
- healthcheck:
- test: ["CMD", "nc", "-z", "localhost", "8888"]
- interval: 30
- timeout: 5s
- retries: 3
- start_period: 30s
- volumes:
- - dolphinscheduler-logs:/var/log/nginx
networks:
- dolphinscheduler
deploy:
@@ -103,7 +83,7 @@ services:
dolphinscheduler-alert:
image: apache/dolphinscheduler:latest
- command: ["alert-server"]
+ command: alert-server
environment:
TZ: Asia/Shanghai
XLS_FILE_PATH: "/tmp/xls"
@@ -127,13 +107,13 @@ services:
DATABASE_DATABASE: dolphinscheduler
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "AlertServer"]
- interval: 30
+ interval: 30s
timeout: 5s
retries: 3
start_period: 30s
volumes:
- dolphinscheduler-logs:/opt/dolphinscheduler/logs
- networks:
+ networks:
- dolphinscheduler
deploy:
mode: replicated
@@ -141,10 +121,10 @@ services:
dolphinscheduler-master:
image: apache/dolphinscheduler:latest
- command: ["master-server"]
- ports:
+ command: master-server
+ ports:
- 5678:5678
- environment:
+ environment:
TZ: Asia/Shanghai
MASTER_EXEC_THREADS: "100"
MASTER_EXEC_TASK_NUM: "20"
@@ -161,7 +141,7 @@ services:
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "MasterServer"]
- interval: 30
+ interval: 30s
timeout: 5s
retries: 3
start_period: 30s
@@ -175,11 +155,11 @@ services:
dolphinscheduler-worker:
image: apache/dolphinscheduler:latest
- command: ["worker-server"]
- ports:
+ command: worker-server
+ ports:
- 1234:1234
- 50051:50051
- environment:
+ environment:
TZ: Asia/Shanghai
WORKER_EXEC_THREADS: "100"
WORKER_HEARTBEAT_INTERVAL: "10"
@@ -188,25 +168,37 @@ services:
WORKER_RESERVED_MEMORY: "0.1"
WORKER_GROUP: "default"
WORKER_WEIGHT: "100"
- DOLPHINSCHEDULER_DATA_BASEDIR_PATH: "/tmp/dolphinscheduler"
+ DOLPHINSCHEDULER_DATA_BASEDIR_PATH: /tmp/dolphinscheduler
+ XLS_FILE_PATH: "/tmp/xls"
+ MAIL_SERVER_HOST: ""
+ MAIL_SERVER_PORT: ""
+ MAIL_SENDER: ""
+ MAIL_USER: ""
+ MAIL_PASSWD: ""
+ MAIL_SMTP_STARTTLS_ENABLE: "false"
+ MAIL_SMTP_SSL_ENABLE: "false"
+ MAIL_SMTP_SSL_TRUST: ""
DATABASE_HOST: dolphinscheduler-postgresql
DATABASE_PORT: 5432
DATABASE_USERNAME: root
DATABASE_PASSWORD: root
DATABASE_DATABASE: dolphinscheduler
ZOOKEEPER_QUORUM: dolphinscheduler-zookeeper:2181
+ RESOURCE_STORAGE_TYPE: HDFS
+ RESOURCE_UPLOAD_PATH: /dolphinscheduler
+ FS_DEFAULT_FS: file:///
healthcheck:
test: ["CMD", "/root/checkpoint.sh", "WorkerServer"]
- interval: 30
+ interval: 30s
timeout: 5s
retries: 3
start_period: 30s
- volumes:
- - dolphinscheduler-worker-data:/tmp/dolphinscheduler
- - dolphinscheduler-logs:/opt/dolphinscheduler/logs
configs:
- source: dolphinscheduler-worker-task-env
target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
+ volumes:
+ - dolphinscheduler-worker-data:/tmp/dolphinscheduler
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
deploy:
diff --git a/docker/docker-swarm/dolphinscheduler_env.sh b/docker/docker-swarm/dolphinscheduler_env.sh
old mode 100644
new mode 100755
index 654318c..7fd3933
--- a/docker/docker-swarm/dolphinscheduler_env.sh
+++ b/docker/docker-swarm/dolphinscheduler_env.sh
@@ -15,6 +15,14 @@
# limitations under the License.
#
-export PYTHON_HOME=/usr/bin/python2
+export HADOOP_HOME=/opt/soft/hadoop
+export HADOOP_CONF_DIR=/opt/soft/hadoop/etc/hadoop
+export SPARK_HOME1=/opt/soft/spark1
+export SPARK_HOME2=/opt/soft/spark2
+export PYTHON_HOME=/usr/bin/python
export JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk
-export PATH=$PYTHON_HOME:$JAVA_HOME/bin:$PATH
\ No newline at end of file
+export HIVE_HOME=/opt/soft/hive
+export FLINK_HOME=/opt/soft/flink
+export DATAX_HOME=/opt/soft/datax/bin/datax.py
+
+export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH:$FLINK_HOME/bin:$DATAX_HOME:$PATH
diff --git a/dolphinscheduler-ui/install-dolphinscheduler-ui.sh b/dolphinscheduler-ui/install-dolphinscheduler-ui.sh
old mode 100644
new mode 100755
diff --git a/e2e/src/test/resources/config/config.properties b/e2e/src/test/resources/config/config.properties
index 352d937..135af0b 100644
--- a/e2e/src/test/resources/config/config.properties
+++ b/e2e/src/test/resources/config/config.properties
@@ -17,7 +17,7 @@
############### project ##############
# login url
-LOGIN_URL=http://127.0.0.1:8888/dolphinscheduler/
+LOGIN_URL=http://127.0.0.1:12345/dolphinscheduler/
#login username
USER_NAME=admin
#login password
diff --git a/script/create-dolphinscheduler.sh b/script/create-dolphinscheduler.sh
old mode 100644
new mode 100755
diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh
old mode 100644
new mode 100755
index 5e76521..a87e62e
--- a/script/dolphinscheduler-daemon.sh
+++ b/script/dolphinscheduler-daemon.sh
@@ -54,6 +54,12 @@ fi
log=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command-$HOSTNAME.out
pid=$DOLPHINSCHEDULER_PID_DIR/dolphinscheduler-$command.pid
+# print logs to /dev/null in docker
+if [ "$DOCKER" = "true" ]; then
+ echo "start in docker"
+ log=/dev/null
+fi
+
cd $DOLPHINSCHEDULER_HOME
if [ "$command" = "api-server" ]; then
diff --git a/script/env/dolphinscheduler_env.sh b/script/env/dolphinscheduler_env.sh
old mode 100644
new mode 100755
diff --git a/script/monitor-server.sh b/script/monitor-server.sh
old mode 100644
new mode 100755
diff --git a/script/scp-hosts.sh b/script/scp-hosts.sh
old mode 100644
new mode 100755
diff --git a/script/start-all.sh b/script/start-all.sh
old mode 100644
new mode 100755
diff --git a/script/status-all.sh b/script/status-all.sh
old mode 100644
new mode 100755
diff --git a/script/stop-all.sh b/script/stop-all.sh
old mode 100644
new mode 100755
diff --git a/script/upgrade-dolphinscheduler.sh b/script/upgrade-dolphinscheduler.sh
old mode 100644
new mode 100755