You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ke...@apache.org on 2023/04/08 09:20:16 UTC
[incubator-seatunnel] 01/01: Add Docker image for engine
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch docker/engine
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
commit 1f48f839d98502574c37e7b71a6d1db62889266b
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Sat Apr 8 17:19:57 2023 +0800
Add Docker image for engine
---
.github/workflows/publish-docker.yaml | 16 ++++------
.gitignore | 5 +--
Makefile | 59 +++++++++++++++++++++++++++++++++++
bin/install-plugin.sh | 14 ++++-----
config/log4j2.properties | 2 +-
docker/seatunnel-engine/Dockerfile | 30 ++++++++++++++++++
6 files changed, 106 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml
index e3b53331a..b9aba2136 100644
--- a/.github/workflows/publish-docker.yaml
+++ b/.github/workflows/publish-docker.yaml
@@ -20,7 +20,7 @@ on:
push:
branches:
- dev
- - docker
+ - docker/*
paths-ignore:
- 'docs/**'
- '**/*.md'
@@ -39,18 +39,12 @@ jobs:
timeout-minutes: 30
steps:
- uses: actions/checkout@v2
- - name: Cache local Maven repository
- uses: actions/cache@v2
- with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-
- name: Set up JDK 1.8
- uses: actions/setup-java@v2
+ uses: actions/setup-java@v3
with:
java-version: 8
distribution: 'adopt'
+ cache: 'maven'
- name: Log in to the Container registry
uses: docker/login-action@v1.10.0
with:
@@ -65,7 +59,7 @@ jobs:
env:
MAVEN_OPTS: -Xmx2G -Xms2G
run: |
- ./mvnw -B clean deploy \
+ ./mvnw -B clean install deploy \
-Dmaven.test.skip \
-Dmaven.javadoc.skip \
-Dlicense.skipAddThirdParty=true \
@@ -74,3 +68,5 @@ jobs:
-Ddocker.hub=${{ env.HUB }} \
-Pdocker \
--no-snapshot-updates
+ - name: Build and push engine images
+ run: make clean push.docker
diff --git a/.gitignore b/.gitignore
index 25977068e..6507089ce 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,7 +20,6 @@ logs.zip
# Intellij Idea files
.idea/
*.iml
-.idea/*
.DS_Store
@@ -48,4 +47,6 @@ test.conf
spark-warehouse
*.flattened-pom.xml
-seatunnel-examples
\ No newline at end of file
+seatunnel-examples
+
+dist
diff --git a/Makefile b/Makefile
new file mode 100644
index 000000000..1d3a468c6
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,59 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+SHELL := /bin/bash -o pipefail
+
+ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+
+HUB ?= apache
+IMAGE ?= seatunnel-engine
+TAG ?= $(shell git rev-parse HEAD)
+
+CONTEXT ?= $(ROOT)/dist/docker-build
+DIST_TAR ?= $(ROOT)/seatunnel-dist/target/apache-seatunnel-*-bin.tar.gz
+
+BUILD_ARGS ?=
+
+docker: PLATFORMS =
+docker: LOAD_OR_PUSH = --load
+push.docker: PLATFORMS = --platform linux/amd64,linux/arm64
+push.docker: LOAD_OR_PUSH = --push
+
+$(DIST_TAR):
+ ./mvnw -B clean install -Dmaven.test.skip -Prelease
+
+$(CONTEXT)/$(IMAGE): $(ROOT)/docker/seatunnel-engine/* $(DIST_TAR)
+ mkdir -p $(CONTEXT)/$(IMAGE)
+ cp -r $< $(CONTEXT)/$(IMAGE)
+ tar -zxf $(DIST_TAR) --strip-components=1 -C $@
+ $@/bin/install-plugin.sh $(shell ./mvnw help:evaluate -q -DforceStdout -D"expression=project.version")
+
+.PHONY: clean
+clean:
+ rm -rf $(CONTEXT)
+ docker buildx rm seatunnel_$(IMAGE) > /dev/null 2>&1 || true
+
+.PHONY: docker push.docker
+docker push.docker: $(CONTEXT)/$(IMAGE)
+ docker buildx create --driver docker-container --name seatunnel_$(IMAGE) > /dev/null 2>&1 || true
+ docker buildx build $(PLATFORMS) $(LOAD_OR_PUSH) \
+ --no-cache $(BUILD_ARGS) \
+ --builder seatunnel_$(IMAGE) \
+ -t $(HUB)/$(IMAGE):$(TAG) \
+ -t $(HUB)/$(IMAGE):latest \
+ $(CONTEXT)/$(IMAGE)
+ docker buildx rm seatunnel_$(IMAGE) || true
diff --git a/bin/install-plugin.sh b/bin/install-plugin.sh
index 35a4dbec2..d2a11db9f 100644
--- a/bin/install-plugin.sh
+++ b/bin/install-plugin.sh
@@ -16,7 +16,7 @@
# limitations under the License.
#
-#This script is used to download the connector plug-ins required during the running process.
+#This script is used to download the connector plug-ins required during the running process.
#All are downloaded by default. You can also choose what you need.
#You only need to configure the plug-in name in config/plugin_config.
@@ -32,7 +32,7 @@ fi
echo "Install hadoop shade jar, usage version is ${version}"
-${SEATUNNEL_HOME}/mvnw dependency:get -DgroupId=org.apache.seatunnel -Dclassifier=optional -DartifactId=seatunnel-hadoop3-3.1.4-uber -Dversion=${version} -Ddest=${SEATUNNEL_HOME}/lib
+${SEATUNNEL_HOME}/mvnw dependency:copy -Dartifact=org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber:${version}:jar:optional -DoutputDirectory=${SEATUNNEL_HOME}/lib
echo "Install SeaTunnel connectors plugins, usage version is ${version}"
@@ -40,21 +40,21 @@ echo "Install SeaTunnel connectors plugins, usage version is ${version}"
if [ ! -d ${SEATUNNEL_HOME}/connectors ];
then
mkdir ${SEATUNNEL_HOME}/connectors
- echo "create connectors directory"
+ echo "Create connectors directory"
fi
# create the seatunnel connectors directory (for v2)
if [ ! -d ${SEATUNNEL_HOME}/connectors/seatunnel ];
then
mkdir ${SEATUNNEL_HOME}/connectors/seatunnel
- echo "create seatunnel connectors directory"
+ echo "Create seatunnel connectors directory"
fi
while read line; do
if [ ${line:0:1} != "-" ] && [ ${line:0:1} != "#" ]
then
- echo "install connector : " $line
- ${SEATUNNEL_HOME}/mvnw dependency:get -DgroupId=org.apache.seatunnel -DartifactId=${line} -Dversion=${version} -Ddest=${SEATUNNEL_HOME}/connectors/seatunnel
+ echo "Install connector : " $line
+ ${SEATUNNEL_HOME}/mvnw dependency:copy -Dartifact=org.apache.seatunnel:${line}:${version}:jar -DoutputDirectory=${SEATUNNEL_HOME}/connectors/seatunnel
fi
-done < ${SEATUNNEL_HOME}/config/plugin_config
\ No newline at end of file
+done < ${SEATUNNEL_HOME}/config/plugin_config
diff --git a/config/log4j2.properties b/config/log4j2.properties
index fb1a07c6d..749d099ab 100644
--- a/config/log4j2.properties
+++ b/config/log4j2.properties
@@ -81,4 +81,4 @@ appender.file.strategy.action.condition.nested_condition.type = IfAny
appender.file.strategy.action.condition.nested_condition.lastModify.type = IfLastModified
appender.file.strategy.action.condition.nested_condition.lastModify.age = ${file_ttl}
appender.file.strategy.action.condition.nested_condition.fileCount.type = IfAccumulatedFileCount
-appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}
\ No newline at end of file
+appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}
diff --git a/docker/seatunnel-engine/Dockerfile b/docker/seatunnel-engine/Dockerfile
new file mode 100644
index 000000000..bf8351aa7
--- /dev/null
+++ b/docker/seatunnel-engine/Dockerfile
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ARG BASE_IMAGE='eclipse-temurin:8-jre'
+
+FROM $BASE_IMAGE
+
+ENV SEATUNNEL_HOME /opt/seatunnel
+
+WORKDIR $SEATUNNEL_HOME
+
+COPY . $SEATUNNEL_HOME
+
+EXPOSE 5801
+
+CMD [ "/bin/bash", "-c", "$SEATUNNEL_HOME/bin/seatunnel-cluster.sh" ]