You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ke...@apache.org on 2023/04/08 09:38:50 UTC
[incubator-seatunnel] branch docker/engine updated (1f48f839d -> 45410b266)
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a change to branch docker/engine
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
discard 1f48f839d Add Docker image for engine
new 45410b266 Add Docker image for engine
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (1f48f839d)
\
N -- N -- N refs/heads/docker/engine (45410b266)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/codeql.yaml | 12 +++---------
.github/workflows/docker.yml | 10 ++--------
.github/workflows/publish-docker.yaml | 2 +-
.github/workflows/schedule_backend.yml | 2 +-
4 files changed, 7 insertions(+), 19 deletions(-)
[incubator-seatunnel] 01/01: Add Docker image for engine
Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch docker/engine
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
commit 45410b266dfe90b861a4293420a5cc597b35c334
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Sat Apr 8 17:38:38 2023 +0800
Add Docker image for engine
---
.github/workflows/codeql.yaml | 12 ++-----
.github/workflows/docker.yml | 10 ++----
.github/workflows/publish-docker.yaml | 18 ++++-------
.github/workflows/schedule_backend.yml | 2 +-
.gitignore | 5 +--
Makefile | 59 ++++++++++++++++++++++++++++++++++
bin/install-plugin.sh | 14 ++++----
config/log4j2.properties | 2 +-
docker/seatunnel-engine/Dockerfile | 30 +++++++++++++++++
9 files changed, 113 insertions(+), 39 deletions(-)
diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml
index 86a0dc9dc..28fefc0af 100644
--- a/.github/workflows/codeql.yaml
+++ b/.github/workflows/codeql.yaml
@@ -17,7 +17,7 @@ name: "CodeQL"
on:
schedule:
- - cron: '0 0 12 * * ?'
+ - cron: '0 0 12 * *'
jobs:
analyze:
@@ -38,17 +38,11 @@ jobs:
with:
submodules: true
- name: Set up JDK 1.8
- uses: actions/setup-java@v2
+ uses: actions/setup-java@v3
with:
java-version: 8
distribution: 'adopt'
- - name: Cache local Maven repository
- uses: actions/cache@v2
- with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-
+ cache: maven
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
with:
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index b1354cb48..3e4865437 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -29,17 +29,11 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: Set up JDK 1.8
- uses: actions/setup-java@v2
+ uses: actions/setup-java@v3
with:
java-version: 8
distribution: 'adopt'
- - name: Cache local Maven repository
- uses: actions/cache@v2
- with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-
+ cache: maven
- name: Build and Package
run: |
./mvnw -B package \
diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml
index e3b53331a..ecc43480b 100644
--- a/.github/workflows/publish-docker.yaml
+++ b/.github/workflows/publish-docker.yaml
@@ -20,7 +20,7 @@ on:
push:
branches:
- dev
- - docker
+ - docker/*
paths-ignore:
- 'docs/**'
- '**/*.md'
@@ -39,18 +39,12 @@ jobs:
timeout-minutes: 30
steps:
- uses: actions/checkout@v2
- - name: Cache local Maven repository
- uses: actions/cache@v2
- with:
- path: ~/.m2/repository
- key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-maven-
- name: Set up JDK 1.8
- uses: actions/setup-java@v2
+ uses: actions/setup-java@v3
with:
java-version: 8
distribution: 'adopt'
+ cache: 'maven'
- name: Log in to the Container registry
uses: docker/login-action@v1.10.0
with:
@@ -65,12 +59,14 @@ jobs:
env:
MAVEN_OPTS: -Xmx2G -Xms2G
run: |
- ./mvnw -B clean deploy \
+ ./mvnw -B clean install deploy \
-Dmaven.test.skip \
-Dmaven.javadoc.skip \
-Dlicense.skipAddThirdParty=true \
-Dmaven.deploy.skip \
-Ddocker.tag=${{ github.sha }} \
-Ddocker.hub=${{ env.HUB }} \
- -Pdocker \
+ -Pdocker,release \
--no-snapshot-updates
+ - name: Build and push engine images
+ run: make clean push.docker
diff --git a/.github/workflows/schedule_backend.yml b/.github/workflows/schedule_backend.yml
index d78d2dc57..bb9bf6e82 100644
--- a/.github/workflows/schedule_backend.yml
+++ b/.github/workflows/schedule_backend.yml
@@ -18,7 +18,7 @@
name: Schedule Backend
on:
schedule:
- - cron: '0 0 03 * * ?'
+ - cron: '0 0 03 * *'
concurrency:
group: backend-${{ github.event.pull_request.number || github.ref }}
diff --git a/.gitignore b/.gitignore
index 25977068e..6507089ce 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,7 +20,6 @@ logs.zip
# Intellij Idea files
.idea/
*.iml
-.idea/*
.DS_Store
@@ -48,4 +47,6 @@ test.conf
spark-warehouse
*.flattened-pom.xml
-seatunnel-examples
\ No newline at end of file
+seatunnel-examples
+
+dist
diff --git a/Makefile b/Makefile
new file mode 100644
index 000000000..1d3a468c6
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,59 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+SHELL := /bin/bash -o pipefail
+
+ROOT := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+
+HUB ?= apache
+IMAGE ?= seatunnel-engine
+TAG ?= $(shell git rev-parse HEAD)
+
+CONTEXT ?= $(ROOT)/dist/docker-build
+DIST_TAR ?= $(ROOT)/seatunnel-dist/target/apache-seatunnel-*-bin.tar.gz
+
+BUILD_ARGS ?=
+
+docker: PLATFORMS =
+docker: LOAD_OR_PUSH = --load
+push.docker: PLATFORMS = --platform linux/amd64,linux/arm64
+push.docker: LOAD_OR_PUSH = --push
+
+$(DIST_TAR):
+ ./mvnw -B clean install -Dmaven.test.skip -Prelease
+
+$(CONTEXT)/$(IMAGE): $(ROOT)/docker/seatunnel-engine/* $(DIST_TAR)
+ mkdir -p $(CONTEXT)/$(IMAGE)
+ cp -r $< $(CONTEXT)/$(IMAGE)
+ tar -zxf $(DIST_TAR) --strip-components=1 -C $@
+ $@/bin/install-plugin.sh $(shell ./mvnw help:evaluate -q -DforceStdout -D"expression=project.version")
+
+.PHONY: clean
+clean:
+ rm -rf $(CONTEXT)
+ docker buildx rm seatunnel_$(IMAGE) > /dev/null 2>&1 || true
+
+.PHONY: docker push.docker
+docker push.docker: $(CONTEXT)/$(IMAGE)
+ docker buildx create --driver docker-container --name seatunnel_$(IMAGE) > /dev/null 2>&1 || true
+ docker buildx build $(PLATFORMS) $(LOAD_OR_PUSH) \
+ --no-cache $(BUILD_ARGS) \
+ --builder seatunnel_$(IMAGE) \
+ -t $(HUB)/$(IMAGE):$(TAG) \
+ -t $(HUB)/$(IMAGE):latest \
+ $(CONTEXT)/$(IMAGE)
+ docker buildx rm seatunnel_$(IMAGE) || true
diff --git a/bin/install-plugin.sh b/bin/install-plugin.sh
index 35a4dbec2..d2a11db9f 100644
--- a/bin/install-plugin.sh
+++ b/bin/install-plugin.sh
@@ -16,7 +16,7 @@
# limitations under the License.
#
-#This script is used to download the connector plug-ins required during the running process.
+#This script is used to download the connector plug-ins required during the running process.
#All are downloaded by default. You can also choose what you need.
#You only need to configure the plug-in name in config/plugin_config.
@@ -32,7 +32,7 @@ fi
echo "Install hadoop shade jar, usage version is ${version}"
-${SEATUNNEL_HOME}/mvnw dependency:get -DgroupId=org.apache.seatunnel -Dclassifier=optional -DartifactId=seatunnel-hadoop3-3.1.4-uber -Dversion=${version} -Ddest=${SEATUNNEL_HOME}/lib
+${SEATUNNEL_HOME}/mvnw dependency:copy -Dartifact=org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber:${version}:jar:optional -DoutputDirectory=${SEATUNNEL_HOME}/lib
echo "Install SeaTunnel connectors plugins, usage version is ${version}"
@@ -40,21 +40,21 @@ echo "Install SeaTunnel connectors plugins, usage version is ${version}"
if [ ! -d ${SEATUNNEL_HOME}/connectors ];
then
mkdir ${SEATUNNEL_HOME}/connectors
- echo "create connectors directory"
+ echo "Create connectors directory"
fi
# create the seatunnel connectors directory (for v2)
if [ ! -d ${SEATUNNEL_HOME}/connectors/seatunnel ];
then
mkdir ${SEATUNNEL_HOME}/connectors/seatunnel
- echo "create seatunnel connectors directory"
+ echo "Create seatunnel connectors directory"
fi
while read line; do
if [ ${line:0:1} != "-" ] && [ ${line:0:1} != "#" ]
then
- echo "install connector : " $line
- ${SEATUNNEL_HOME}/mvnw dependency:get -DgroupId=org.apache.seatunnel -DartifactId=${line} -Dversion=${version} -Ddest=${SEATUNNEL_HOME}/connectors/seatunnel
+ echo "Install connector : " $line
+ ${SEATUNNEL_HOME}/mvnw dependency:copy -Dartifact=org.apache.seatunnel:${line}:${version}:jar -DoutputDirectory=${SEATUNNEL_HOME}/connectors/seatunnel
fi
-done < ${SEATUNNEL_HOME}/config/plugin_config
\ No newline at end of file
+done < ${SEATUNNEL_HOME}/config/plugin_config
diff --git a/config/log4j2.properties b/config/log4j2.properties
index fb1a07c6d..749d099ab 100644
--- a/config/log4j2.properties
+++ b/config/log4j2.properties
@@ -81,4 +81,4 @@ appender.file.strategy.action.condition.nested_condition.type = IfAny
appender.file.strategy.action.condition.nested_condition.lastModify.type = IfLastModified
appender.file.strategy.action.condition.nested_condition.lastModify.age = ${file_ttl}
appender.file.strategy.action.condition.nested_condition.fileCount.type = IfAccumulatedFileCount
-appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}
\ No newline at end of file
+appender.file.strategy.action.condition.nested_condition.fileCount.exceeds = ${file_count}
diff --git a/docker/seatunnel-engine/Dockerfile b/docker/seatunnel-engine/Dockerfile
new file mode 100644
index 000000000..bf8351aa7
--- /dev/null
+++ b/docker/seatunnel-engine/Dockerfile
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ARG BASE_IMAGE='eclipse-temurin:8-jre'
+
+FROM $BASE_IMAGE
+
+ENV SEATUNNEL_HOME /opt/seatunnel
+
+WORKDIR $SEATUNNEL_HOME
+
+COPY . $SEATUNNEL_HOME
+
+EXPOSE 5801
+
+CMD [ "/bin/bash", "-c", "$SEATUNNEL_HOME/bin/seatunnel-cluster.sh" ]