You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by li...@apache.org on 2020/05/11 15:22:15 UTC
[incubator-dolphinscheduler] branch dev-1.3.0 updated: fix
zookeeper health check and merge volumes into logs volume (#2674)
This is an automated email from the ASF dual-hosted git repository.
liuxiaochun pushed a commit to branch dev-1.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev-1.3.0 by this push:
new 3c014c5 fix zookeeper health check and merge volumes into logs volume (#2674)
3c014c5 is described below
commit 3c014c5a570becf13c000c59222daca663fd1062
Author: liwenhe1993 <li...@apache.org>
AuthorDate: Mon May 11 23:22:04 2020 +0800
fix zookeeper health check and merge volumes into logs volume (#2674)
* fix zookeeper health check and merge volumes into logs volume
1. Fix zookeeper health check
2. Merge volumes into logs volume
3. Modify `dockerfile/hooks/build` script
* Modify ci_e2e and docker image registry
* Modify ci_e2e
* Modify ci_e2e
* Modify ci_e2e and move ./dockerfile/hooks/check to ./docker/docker-swarm/
---
.github/workflows/ci_e2e.yml | 11 +++----
{dockerfile/hooks => docker/docker-swarm}/check | 2 +-
docker/docker-swarm/docker-compose.yml | 41 ++++++++++++-------------
docker/docker-swarm/docker-stack.yml | 27 ++++++++--------
docker/kubernetes/dolphinscheduler/values.yaml | 6 ++--
dockerfile/Dockerfile | 4 +--
dockerfile/hooks/build | 10 +++---
7 files changed, 48 insertions(+), 53 deletions(-)
diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml
index 924ef11..c52f0e0 100644
--- a/.github/workflows/ci_e2e.yml
+++ b/.github/workflows/ci_e2e.yml
@@ -44,15 +44,14 @@ jobs:
${{ runner.os }}-maven-
- name: Build Image
run: |
- export VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT</version>" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'`
sh ./dockerfile/hooks/build
- name: Docker Run
run: |
- VERSION=`cat $(pwd)/pom.xml| grep "SNAPSHOT</version>" | awk -F "-SNAPSHOT" '{print $1}' | awk -F ">" '{print $2}'`
- mkdir -p /tmp/logs
- docker run -dit -e POSTGRESQL_USERNAME=test -e POSTGRESQL_PASSWORD=test -v /tmp/logs:/opt/dolphinscheduler/logs -p 8888:8888 dolphinscheduler:$VERSION all
+ export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
+ sed -i "s/apache\/dolphinscheduler:latest/apache\/dolphinscheduler:${VERSION}/g" $(pwd)/docker/docker-swarm/docker-compose.yml
+ docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml up -d
- name: Check Server Status
- run: sh ./dockerfile/hooks/check
+ run: sh $(pwd)/docker/docker-swarm/check
- name: Prepare e2e env
run: |
sudo apt-get install -y libxss1 libappindicator1 libindicator7 xvfb unzip libgbm1
@@ -70,6 +69,6 @@ jobs:
uses: actions/upload-artifact@v1
with:
name: dslogs
- path: /tmp/logs
+ path: /var/lib/docker/volumes/docker-swarm_dolphinscheduler-logs/_data
diff --git a/dockerfile/hooks/check b/docker/docker-swarm/check
similarity index 85%
rename from dockerfile/hooks/check
rename to docker/docker-swarm/check
index fdb1902..59203c5 100644
--- a/dockerfile/hooks/check
+++ b/docker/docker-swarm/check
@@ -17,7 +17,7 @@
#
echo "------ dolphinscheduler check - server - status -------"
sleep 60
-server_num=$(docker top `docker container list | grep '/sbin/tini' | awk '{print $1}'`| grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l)
+server_num=$(docker-compose -f $(pwd)/docker/docker-swarm/docker-compose.yml top | grep java | grep "dolphinscheduler" | awk -F 'classpath ' '{print $2}' | awk '{print $2}' | sort | uniq -c | wc -l)
if [ $server_num -eq 5 ]
then
echo "Server all start successfully"
diff --git a/docker/docker-swarm/docker-compose.yml b/docker/docker-swarm/docker-compose.yml
index ffa91a0..b564eda 100644
--- a/docker/docker-swarm/docker-compose.yml
+++ b/docker/docker-swarm/docker-compose.yml
@@ -41,13 +41,14 @@ services:
environment:
TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes"
+ ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper
networks:
- dolphinscheduler
dolphinscheduler-api:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+ image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-api
command: ["api-server"]
ports:
@@ -70,12 +71,12 @@ services:
- dolphinscheduler-postgresql
- dolphinscheduler-zookeeper
volumes:
- - dolphinscheduler-api:/opt/dolphinscheduler/logs
- networks:
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ networks:
- dolphinscheduler
dolphinscheduler-frontend:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+ image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-frontend
command: ["frontend"]
ports:
@@ -93,12 +94,12 @@ services:
depends_on:
- dolphinscheduler-api
volumes:
- - dolphinscheduler-frontend:/var/log/nginx
+ - dolphinscheduler-logs:/var/log/nginx
networks:
- dolphinscheduler
dolphinscheduler-alert:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+ image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-alert
command: ["alert-server"]
environment:
@@ -130,18 +131,18 @@ services:
start_period: 30s
depends_on:
- dolphinscheduler-postgresql
- volumes:
- - dolphinscheduler-alert:/opt/dolphinscheduler/logs
- networks:
+ volumes:
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
+ networks:
- dolphinscheduler
dolphinscheduler-master:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+ image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-master
command: ["master-server"]
- ports:
+ ports:
- 5678:5678
- environment:
+ environment:
TZ: Asia/Shanghai
MASTER_EXEC_THREADS: "100"
MASTER_EXEC_TASK_NUM: "20"
@@ -162,16 +163,16 @@ services:
timeout: 5s
retries: 3
start_period: 30s
- depends_on:
+ depends_on:
- dolphinscheduler-postgresql
- dolphinscheduler-zookeeper
- volumes:
- - dolphinscheduler-master:/opt/dolphinscheduler/logs
+ volumes:
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
dolphinscheduler-worker:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+ image: apache/dolphinscheduler:latest
container_name: dolphinscheduler-worker
command: ["worker-server"]
ports:
@@ -209,7 +210,7 @@ services:
source: dolphinscheduler-worker-data
target: /tmp/dolphinscheduler
- type: volume
- source: dolphinscheduler-worker-logs
+ source: dolphinscheduler-logs
target: /opt/dolphinscheduler/logs
networks:
- dolphinscheduler
@@ -221,12 +222,8 @@ networks:
volumes:
dolphinscheduler-postgresql:
dolphinscheduler-zookeeper:
- dolphinscheduler-api:
- dolphinscheduler-frontend:
- dolphinscheduler-alert:
- dolphinscheduler-master:
dolphinscheduler-worker-data:
- dolphinscheduler-worker-logs:
+ dolphinscheduler-logs:
configs:
dolphinscheduler-worker-task-env:
diff --git a/docker/docker-swarm/docker-stack.yml b/docker/docker-swarm/docker-stack.yml
index e628205..5c3873f 100644
--- a/docker/docker-swarm/docker-stack.yml
+++ b/docker/docker-swarm/docker-stack.yml
@@ -42,6 +42,7 @@ services:
environment:
TZ: Asia/Shanghai
ALLOW_ANONYMOUS_LOGIN: "yes"
+ ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
volumes:
- dolphinscheduler-zookeeper:/bitnami/zookeeper
networks:
@@ -51,7 +52,7 @@ services:
replicas: 1
dolphinscheduler-api:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+ image: apache/dolphinscheduler:latest
command: ["api-server"]
ports:
- 12345:12345
@@ -70,7 +71,7 @@ services:
retries: 3
start_period: 30s
volumes:
- - dolphinscheduler-api:/opt/dolphinscheduler/logs
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
deploy:
@@ -78,7 +79,7 @@ services:
replicas: 1
dolphinscheduler-frontend:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+ image: apache/dolphinscheduler:latest
command: ["frontend"]
ports:
- 8888:8888
@@ -93,7 +94,7 @@ services:
retries: 3
start_period: 30s
volumes:
- - dolphinscheduler-frontend:/var/log/nginx
+ - dolphinscheduler-logs:/var/log/nginx
networks:
- dolphinscheduler
deploy:
@@ -101,7 +102,7 @@ services:
replicas: 1
dolphinscheduler-alert:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+ image: apache/dolphinscheduler:latest
command: ["alert-server"]
environment:
TZ: Asia/Shanghai
@@ -131,7 +132,7 @@ services:
retries: 3
start_period: 30s
volumes:
- - dolphinscheduler-alert:/opt/dolphinscheduler/logs
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
deploy:
@@ -139,7 +140,7 @@ services:
replicas: 1
dolphinscheduler-master:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+ image: apache/dolphinscheduler:latest
command: ["master-server"]
ports:
- 5678:5678
@@ -165,7 +166,7 @@ services:
retries: 3
start_period: 30s
volumes:
- - dolphinscheduler-master:/opt/dolphinscheduler/logs
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
networks:
- dolphinscheduler
deploy:
@@ -173,7 +174,7 @@ services:
replicas: 1
dolphinscheduler-worker:
- image: registry.cn-qingdao.aliyuncs.com/sxyj/dolphinscheduler:dev
+ image: apache/dolphinscheduler:latest
command: ["worker-server"]
ports:
- 1234:1234
@@ -201,7 +202,7 @@ services:
start_period: 30s
volumes:
- dolphinscheduler-worker-data:/tmp/dolphinscheduler
- - dolphinscheduler-worker-logs:/opt/dolphinscheduler/logs
+ - dolphinscheduler-logs:/opt/dolphinscheduler/logs
configs:
- source: dolphinscheduler-worker-task-env
target: /opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh
@@ -218,12 +219,8 @@ networks:
volumes:
dolphinscheduler-postgresql:
dolphinscheduler-zookeeper:
- dolphinscheduler-api:
- dolphinscheduler-frontend:
- dolphinscheduler-alert:
- dolphinscheduler-master:
dolphinscheduler-worker-data:
- dolphinscheduler-worker-logs:
+ dolphinscheduler-logs:
configs:
dolphinscheduler-worker-task-env:
diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml
index 4f70afa..3cb35c1 100644
--- a/docker/kubernetes/dolphinscheduler/values.yaml
+++ b/docker/kubernetes/dolphinscheduler/values.yaml
@@ -25,9 +25,9 @@ fullnameOverride: ""
timezone: "Asia/Shanghai"
image:
- registry: "docker.io"
+ registry: "apache"
repository: "dolphinscheduler"
- tag: "1.3.0"
+ tag: "latest"
pullPolicy: "IfNotPresent"
imagePullSecrets: []
@@ -56,6 +56,8 @@ externalDatabase:
zookeeper:
enabled: true
taskQueue: "zookeeper"
+ config:
+ ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons
service:
port: "2181"
persistence:
diff --git a/dockerfile/Dockerfile b/dockerfile/Dockerfile
index bed8f65..53e362b 100644
--- a/dockerfile/Dockerfile
+++ b/dockerfile/Dockerfile
@@ -53,8 +53,8 @@ ENV PATH $ZK_HOME/bin:$PATH
RUN apk add postgresql postgresql-contrib
#5. add dolphinscheduler
-ADD ./apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz /opt/
-RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin/ /opt/dolphinscheduler/
+ADD ./apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz /opt/
+RUN mv /opt/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin/ /opt/dolphinscheduler/
ENV DOLPHINSCHEDULER_HOME /opt/dolphinscheduler
#6. modify nginx
diff --git a/dockerfile/hooks/build b/dockerfile/hooks/build
index 05fa09d..956881b 100644
--- a/dockerfile/hooks/build
+++ b/dockerfile/hooks/build
@@ -24,13 +24,13 @@ printenv
if [ -z "${VERSION}" ]
then
echo "set default environment variable [VERSION]"
- VERSION=$(cat $(pwd)/sql/soft_version)
+ export VERSION=$(cat $(pwd)/pom.xml | grep '<version>' -m 1 | awk '{print $1}' | sed 's/<version>//' | sed 's/<\/version>//')
fi
if [ "${DOCKER_REPO}x" = "x" ]
then
echo "set default environment variable [DOCKER_REPO]"
- DOCKER_REPO='dolphinscheduler'
+ export DOCKER_REPO='apache/dolphinscheduler'
fi
echo "Version: $VERSION"
@@ -43,11 +43,11 @@ echo -e "mvn -B clean compile package -Prelease -Dmaven.test.skip=true"
mvn -B clean compile package -Prelease -Dmaven.test.skip=true
# mv dolphinscheduler-bin.tar.gz file to dockerfile directory
-echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/\n"
-mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-SNAPSHOT-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/
+echo -e "mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/\n"
+mv $(pwd)/dolphinscheduler-dist/target/apache-dolphinscheduler-incubating-${VERSION}-dolphinscheduler-bin.tar.gz $(pwd)/dockerfile/
# docker build
echo -e "docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/\n"
-docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/
+sudo docker build --build-arg VERSION=${VERSION} -t $DOCKER_REPO:${VERSION} $(pwd)/dockerfile/
echo "------ dolphinscheduler end - build -------"