You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by wa...@apache.org on 2024/08/23 14:45:49 UTC
(seatunnel) branch dev updated: [Feature] add dockerfile (#7346)
This is an automated email from the ASF dual-hosted git repository.
wanghailin pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 8a28290fde [Feature] add dockerfile (#7346)
8a28290fde is described below
commit 8a28290fde9a8f625ef3e900ff91819bf7a5417c
Author: Jarvis <li...@163.com>
AuthorDate: Fri Aug 23 22:45:43 2024 +0800
[Feature] add dockerfile (#7346)
---
.github/workflows/publish-docker.yaml | 43 +++++++------
docs/en/start-v2/docker/docker.md | 103 +++++++++++++++++++++++++++++-
pom.xml | 93 +++++++++++++++++++++++++++
seatunnel-dist/pom.xml | 21 +++++-
seatunnel-dist/src/main/docker/Dockerfile | 12 ++++
5 files changed, 250 insertions(+), 22 deletions(-)
diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml
index 96b0b01936..e1041bca2c 100644
--- a/.github/workflows/publish-docker.yaml
+++ b/.github/workflows/publish-docker.yaml
@@ -18,16 +18,17 @@ name: publish-docker
on:
push:
+ tags:
+ - '*'
branches:
- dev
- - docker
paths-ignore:
- 'docs/**'
- '**/*.md'
- - 'seatunnel-ui/**'
env:
- HUB: ghcr.io/${{ github.repository }}
+ DOCKER_USERNAME: ${{ secrets.DOCKERHUB_USER }}
+ DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
jobs:
build:
@@ -36,11 +37,16 @@ jobs:
permissions:
contents: read
packages: write
- timeout-minutes: 30
+ timeout-minutes: 60
steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v4
+ with:
+ submodules: true
+ - name: free disk space
+ run: tools/github/free_disk_space.sh
+ - uses: actions/checkout@v4
- name: Cache local Maven repository
- uses: actions/cache@v2
+ uses: actions/cache@v4
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
@@ -51,26 +57,27 @@ jobs:
with:
java-version: 8
distribution: 'adopt'
+
- name: Log in to the Container registry
- uses: docker/login-action@v1.10.0
+ uses: docker/login-action@v3
with:
- registry: ${{ env.HUB }}
- username: ${{ github.actor }}
- password: ${{ secrets.GITHUB_TOKEN }}
+ username: ${{ env.DOCKER_USERNAME }}
+ password: ${{ env.DOCKER_PASSWORD }}
- name: Set up QEMU
- uses: docker/setup-qemu-action@v1
+ uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v1
+ uses: docker/setup-buildx-action@v3
- name: Build and push docker images
env:
- MAVEN_OPTS: -Xmx2G -Xms2G
+ MAVEN_OPTS: -Xmx4096m
run: |
- ./mvnw -B clean deploy \
+ ./mvnw -B clean install \
-Dmaven.test.skip \
-Dmaven.javadoc.skip \
-Dlicense.skipAddThirdParty=true \
+ -D"docker.build.skip"=false \
+ -D"docker.verify.skip"=false \
+ -D"docker.push.skip"=false \
-Dmaven.deploy.skip \
- -Ddocker.tag=${{ github.sha }} \
- -Ddocker.hub=${{ env.HUB }} \
- -Pdocker \
- --no-snapshot-updates
+ --no-snapshot-updates \
+ -Pdocker,seatunnel
\ No newline at end of file
diff --git a/docs/en/start-v2/docker/docker.md b/docs/en/start-v2/docker/docker.md
index ad2e3b078a..5c2c842222 100644
--- a/docs/en/start-v2/docker/docker.md
+++ b/docs/en/start-v2/docker/docker.md
@@ -2,7 +2,104 @@
sidebar_position: 3
---
-# Set Up With Docker
+# Set Up with Docker in local mode
+
+## Zeta Engine
+
+### Download
+
+```shell
+docker pull apache/seatunnel:<version_tag>
+```
+
+How to submit job in local mode
+
+```shell
+docker run --rm -it apache/seatunnel bash ./bin/seatunnel.sh -e local -c <CONFIG_FILE>
+
+
+# eg: a fake source to console sink
+docker run --rm -it apache/seatunnel bash ./bin/seatunnel.sh -e local -c config/v2.batch.config.template
+
+```
+
+### Build Image By Yourself
+
+```Dockerfile
+FROM openjdk:8
+
+ARG VERSION
+# Build from Source Code And Copy it into image
+COPY ./target/apache-seatunnel-${VERSION}-bin.tar.gz /opt/
+
+# Download From Internet
+# Please Note this file only include fake/console connector, You'll need to download the other connectors manually
+# wget -P /opt https://dlcdn.apache.org/seatunnel/2.3.6/apache-seatunnel-${VERSION}-bin.tar.gz
+
+RUN cd /opt && \
+ tar -zxvf apache-seatunnel-${VERSION}-bin.tar.gz && \
+ mv apache-seatunnel-${VERSION} seatunnel && \
+ rm apache-seatunnel-${VERSION}-bin.tar.gz
+
+WORKDIR /opt/seatunnel
+```
+
+## Spark or Flink Engine
+
+### Download And Install the connectors you needed
+
+refer the step as Zeta Engine
+
+### Mount Spark/Flink library
+
+By default, Spark home is `/opt/spark`, Flink home is `/opt/flink`.
+If you need run with spark/flink, you can mount the related library to `/opt/spark` or `/opt/flink`.
+
+```shell
+docker run \
+ -v <SPARK_BINARY_PATH>:/opt/spark \
+ -v <FLINK_BINARY_PATH>:/opt/flink \
+ ...
+```
+
+Or you can change the `SPARK_HOME`, `FLINK_HOME` environment variable in Dockerfile and re-build your and mount the spark/flink to related path.
+
+```Dockerfile
+FROM apache/seatunnel
+
+ENV SPARK_HOME=<YOUR_CUSTOMIZATION_PATH>
+
+...
+
+```
+
+```shell
+docker run \
+ -v <SPARK_BINARY_PATH>:<YOUR_CUSTOMIZATION_PATH> \
+ ...
+```
+
+### Submit job
+
+The command is different for different engines and different versions of the same engine, please choose the correct command.
+
+- Spark
+
+```shell
+# spark2
+docker run --rm -it apache/seatunnel bash ./bin/start-seatunnel-spark-2-connector-v2.sh -c config/v2.batch.config.template
+
+# spark3
+docker run --rm -it apache/seatunnel bash ./bin/start-seatunnel-spark-3-connector-v2.sh -c config/v2.batch.config.template
+```
+
+- Flink
+ before you submit job, you need start flink cluster first.
+
+```shell
+# flink version between `1.12.x` and `1.14.x`
+docker run --rm -it apache/seatunnel bash -c '<YOUR_FLINK_HOME>/bin/start-cluster.sh && ./bin/start-seatunnel-flink-13-connector-v2.sh -c config/v2.streaming.conf.template'
+# flink version between `1.15.x` and `1.16.x`
+docker run --rm -it apache/seatunnel bash -c '<YOUR_FLINK_HOME>/bin/start-cluster.sh && ./bin/start-seatunnel-flink-15-connector-v2.sh -c config/v2.streaming.conf.template'
+```
-<!-- TODO -->
--->
diff --git a/pom.xml b/pom.xml
index 38860eb279..3815a5c393 100644
--- a/pom.xml
+++ b/pom.xml
@@ -113,7 +113,11 @@
<codec.version>1.13</codec.version>
<exec-maven-plugin.version>3.0.0</exec-maven-plugin.version>
<docker.hub>apache</docker.hub>
+ <docker.repo>seatunnel</docker.repo>
<docker.tag>${project.version}</docker.tag>
+ <docker.build.skip>true</docker.build.skip>
+ <docker.verify.skip>true</docker.verify.skip>
+ <docker.push.skip>true</docker.push.skip>
<jcommander.version>1.81</jcommander.version>
<junit4.version>4.13.2</junit4.version>
<junit5.version>5.9.0</junit5.version>
@@ -760,6 +764,95 @@
<artifactId>maven-dependency-plugin</artifactId>
<version>${maven-dependency-plugin.version}</version>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>${exec-maven-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>docker-build</id>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>package</phase>
+ <configuration>
+ <skip>${docker.build.skip}</skip>
+ <environmentVariables>
+ <DOCKER_BUILDKIT>1</DOCKER_BUILDKIT>
+ </environmentVariables>
+ <executable>docker</executable>
+ <workingDirectory>${project.basedir}</workingDirectory>
+ <arguments>
+ <argument>buildx</argument>
+ <argument>build</argument>
+ <argument>--load</argument>
+ <argument>--no-cache</argument>
+ <argument>-t</argument>
+ <argument>${docker.hub}/${docker.repo}:${docker.tag}</argument>
+ <argument>${project.basedir}</argument>
+ <argument>--build-arg</argument>
+ <argument>VERSION=${project.version}</argument>
+ <argument>--file=src/main/docker/Dockerfile</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>docker-verify</id>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>verify</phase>
+ <configuration>
+ <skip>${docker.verify.skip}</skip>
+ <environmentVariables>
+ <DOCKER_BUILDKIT>1</DOCKER_BUILDKIT>
+ </environmentVariables>
+ <executable>docker</executable>
+ <workingDirectory>${project.basedir}</workingDirectory>
+ <arguments>
+ <argument>run</argument>
+ <argument>--rm</argument>
+ <argument>${docker.hub}/${docker.repo}:${docker.tag}</argument>
+ <argument>bash</argument>
+ <argument>./bin/seatunnel.sh</argument>
+ <argument>-e</argument>
+ <argument>local</argument>
+ <argument>-c</argument>
+ <argument>config/v2.batch.config.template</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ <execution>
+ <id>docker-push</id>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>install</phase>
+ <configuration>
+ <skip>${docker.push.skip}</skip>
+ <environmentVariables>
+ <DOCKER_BUILDKIT>1</DOCKER_BUILDKIT>
+ </environmentVariables>
+ <executable>docker</executable>
+ <workingDirectory>${project.basedir}</workingDirectory>
+ <arguments>
+ <argument>buildx</argument>
+ <argument>build</argument>
+ <argument>--platform</argument>
+ <argument>linux/amd64,linux/arm64</argument>
+ <argument>--no-cache</argument>
+ <argument>--push</argument>
+ <argument>-t</argument>
+ <argument>${docker.hub}/${docker.repo}:${docker.tag}</argument>
+ <argument>${project.basedir}</argument>
+ <argument>--build-arg</argument>
+ <argument>VERSION=${revision}</argument>
+ <argument>--file=src/main/docker/Dockerfile</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</pluginManagement>
diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml
index e919293aab..fb6935f089 100644
--- a/seatunnel-dist/pom.xml
+++ b/seatunnel-dist/pom.xml
@@ -66,7 +66,6 @@
<appendAssemblyId>true</appendAssemblyId>
</configuration>
</execution>
-
</executions>
</plugin>
</plugins>
@@ -83,6 +82,9 @@
</property>
</activation>
<properties>
+ <docker.build.skip>false</docker.build.skip>
+ <docker.verify.skip>false</docker.verify.skip>
+ <docker.push.skip>false</docker.push.skip>
<mysql.version>8.0.27</mysql.version>
<postgresql.version>42.4.3</postgresql.version>
<postgis.jdbc.version>2.5.1</postgis.jdbc.version>
@@ -965,5 +967,22 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>docker</id>
+ <activation>
+ <property>
+ <name>release</name>
+ <value>false</value>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
diff --git a/seatunnel-dist/src/main/docker/Dockerfile b/seatunnel-dist/src/main/docker/Dockerfile
new file mode 100644
index 0000000000..7d040ee41a
--- /dev/null
+++ b/seatunnel-dist/src/main/docker/Dockerfile
@@ -0,0 +1,12 @@
+FROM openjdk:8
+
+ARG VERSION
+
+COPY ./target/apache-seatunnel-${VERSION}-bin.tar.gz /opt/
+
+RUN cd /opt && \
+ tar -zxvf apache-seatunnel-${VERSION}-bin.tar.gz && \
+ mv apache-seatunnel-${VERSION} seatunnel && \
+ rm apache-seatunnel-${VERSION}-bin.tar.gz
+
+WORKDIR /opt/seatunnel