You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by wa...@apache.org on 2024/08/23 14:45:49 UTC

(seatunnel) branch dev updated: [Feature] add dockerfile (#7346)

This is an automated email from the ASF dual-hosted git repository.

wanghailin pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git


The following commit(s) were added to refs/heads/dev by this push:
     new 8a28290fde [Feature] add dockerfile (#7346)
8a28290fde is described below

commit 8a28290fde9a8f625ef3e900ff91819bf7a5417c
Author: Jarvis <li...@163.com>
AuthorDate: Fri Aug 23 22:45:43 2024 +0800

    [Feature] add dockerfile (#7346)
---
 .github/workflows/publish-docker.yaml     |  43 +++++++------
 docs/en/start-v2/docker/docker.md         | 103 +++++++++++++++++++++++++++++-
 pom.xml                                   |  93 +++++++++++++++++++++++++++
 seatunnel-dist/pom.xml                    |  21 +++++-
 seatunnel-dist/src/main/docker/Dockerfile |  12 ++++
 5 files changed, 250 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml
index 96b0b01936..e1041bca2c 100644
--- a/.github/workflows/publish-docker.yaml
+++ b/.github/workflows/publish-docker.yaml
@@ -18,16 +18,17 @@ name: publish-docker
 
 on:
   push:
+    tags:
+      - '*'
     branches:
       - dev
-      - docker
     paths-ignore:
       - 'docs/**'
       - '**/*.md'
-      - 'seatunnel-ui/**'
 
 env:
-  HUB: ghcr.io/${{ github.repository }}
+  DOCKER_USERNAME: ${{ secrets.DOCKERHUB_USER }}
+  DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 
 jobs:
   build:
@@ -36,11 +37,16 @@ jobs:
     permissions:
       contents: read
       packages: write
-    timeout-minutes: 30
+    timeout-minutes: 60
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: free disk space
+        run: tools/github/free_disk_space.sh
+      - uses: actions/checkout@v4
       - name: Cache local Maven repository
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
@@ -51,26 +57,27 @@ jobs:
         with:
           java-version:  8
           distribution: 'adopt'
+
       - name: Log in to the Container registry
-        uses: docker/login-action@v1.10.0
+        uses: docker/login-action@v3
         with:
-          registry: ${{ env.HUB }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          username: ${{ env.DOCKER_USERNAME }}
+          password: ${{ env.DOCKER_PASSWORD }}
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
       - name: Build and push docker images
         env:
-          MAVEN_OPTS: -Xmx2G -Xms2G
+          MAVEN_OPTS: -Xmx4096m
         run: |
-          ./mvnw -B clean deploy \
+          ./mvnw -B clean install \
           -Dmaven.test.skip \
           -Dmaven.javadoc.skip \
           -Dlicense.skipAddThirdParty=true \
+          -D"docker.build.skip"=false \
+          -D"docker.verify.skip"=false \
+          -D"docker.push.skip"=false \
           -Dmaven.deploy.skip \
-          -Ddocker.tag=${{ github.sha }} \
-          -Ddocker.hub=${{ env.HUB }} \
-          -Pdocker \
-          --no-snapshot-updates
+          --no-snapshot-updates \
+          -Pdocker,seatunnel
\ No newline at end of file
diff --git a/docs/en/start-v2/docker/docker.md b/docs/en/start-v2/docker/docker.md
index ad2e3b078a..5c2c842222 100644
--- a/docs/en/start-v2/docker/docker.md
+++ b/docs/en/start-v2/docker/docker.md
@@ -2,7 +2,104 @@
 sidebar_position: 3
 ---
 
-# Set Up With Docker
+# Set Up with Docker in local mode
+
+## Zeta Engine
+
+### Download
+
+```shell
+docker pull apache/seatunnel:<version_tag>
+```
+
+How to submit job in local mode
+
+```shell
+docker run --rm -it apache/seatunnel bash ./bin/seatunnel.sh -e local -c <CONFIG_FILE>
+
+
+# eg: a fake source to console sink
+docker run --rm -it apache/seatunnel bash ./bin/seatunnel.sh -e local -c config/v2.batch.config.template
+
+```
+
+### Build Image By Yourself
+
+```Dockerfile
+FROM openjdk:8
+
+ARG VERSION
+# Build from Source Code And Copy it into image
+COPY ./target/apache-seatunnel-${VERSION}-bin.tar.gz /opt/
+
+# Download From Internet
+# Please Note this file only include fake/console connector, You'll need to download the other connectors manually
+# wget -P /opt https://dlcdn.apache.org/seatunnel/2.3.6/apache-seatunnel-${VERSION}-bin.tar.gz
+
+RUN cd /opt && \
+    tar -zxvf apache-seatunnel-${VERSION}-bin.tar.gz && \
+    mv apache-seatunnel-${VERSION} seatunnel && \
+    rm apache-seatunnel-${VERSION}-bin.tar.gz
+
+WORKDIR /opt/seatunnel
+```
+
+## Spark or Flink Engine
+
+### Download And Install the connectors you needed
+
+refer the step as Zeta Engine
+
+### Mount Spark/Flink library
+
+By default, Spark home is `/opt/spark`, Flink home is `/opt/flink`.
+If you need run with spark/flink, you can mount the related library to `/opt/spark` or `/opt/flink`.
+
+```shell
+docker run \ 
+ -v <SPARK_BINARY_PATH>:/opt/spark \
+ -v <FLINK_BINARY_PATH>:/opt/flink \
+  ...
+```
+
+Or you can change the `SPARK_HOME`, `FLINK_HOME` environment variable in Dockerfile and re-build your  and mount the spark/flink to related path.
+
+```Dockerfile
+FROM apache/seatunnel
+
+ENV SPARK_HOME=<YOUR_CUSTOMIZATION_PATH>
+
+...
+
+```
+
+```shell
+docker run \ 
+ -v <SPARK_BINARY_PATH>:<YOUR_CUSTOMIZATION_PATH> \
+  ...
+```
+
+### Submit job
+
+The command is different for different engines and different versions of the same engine, please choose the correct command.
+
+- Spark
+
+```shell
+# spark2
+docker run --rm -it apache/seatunnel bash ./bin/start-seatunnel-spark-2-connector-v2.sh -c config/v2.batch.config.template
+
+# spark3
+docker run --rm -it apache/seatunnel bash ./bin/start-seatunnel-spark-3-connector-v2.sh -c config/v2.batch.config.template
+```
+
+- Flink
+  before you submit job, you need start flink cluster first.
+
+```shell
+# flink version between `1.12.x` and `1.14.x`
+docker run --rm -it apache/seatunnel bash -c '<YOUR_FLINK_HOME>/bin/start-cluster.sh && ./bin/start-seatunnel-flink-13-connector-v2.sh -c config/v2.streaming.conf.template'
+# flink version between `1.15.x` and `1.16.x`
+docker run --rm -it apache/seatunnel bash -c '<YOUR_FLINK_HOME>/bin/start-cluster.sh && ./bin/start-seatunnel-flink-15-connector-v2.sh -c config/v2.streaming.conf.template'
+```
 
-<!-- TODO -->
--->
diff --git a/pom.xml b/pom.xml
index 38860eb279..3815a5c393 100644
--- a/pom.xml
+++ b/pom.xml
@@ -113,7 +113,11 @@
         <codec.version>1.13</codec.version>
         <exec-maven-plugin.version>3.0.0</exec-maven-plugin.version>
         <docker.hub>apache</docker.hub>
+        <docker.repo>seatunnel</docker.repo>
         <docker.tag>${project.version}</docker.tag>
+        <docker.build.skip>true</docker.build.skip>
+        <docker.verify.skip>true</docker.verify.skip>
+        <docker.push.skip>true</docker.push.skip>
         <jcommander.version>1.81</jcommander.version>
         <junit4.version>4.13.2</junit4.version>
         <junit5.version>5.9.0</junit5.version>
@@ -760,6 +764,95 @@
                     <artifactId>maven-dependency-plugin</artifactId>
                     <version>${maven-dependency-plugin.version}</version>
                 </plugin>
+                <plugin>
+                    <groupId>org.codehaus.mojo</groupId>
+                    <artifactId>exec-maven-plugin</artifactId>
+                    <version>${exec-maven-plugin.version}</version>
+                    <executions>
+                        <execution>
+                            <id>docker-build</id>
+                            <goals>
+                                <goal>exec</goal>
+                            </goals>
+                            <phase>package</phase>
+                            <configuration>
+                                <skip>${docker.build.skip}</skip>
+                                <environmentVariables>
+                                    <DOCKER_BUILDKIT>1</DOCKER_BUILDKIT>
+                                </environmentVariables>
+                                <executable>docker</executable>
+                                <workingDirectory>${project.basedir}</workingDirectory>
+                                <arguments>
+                                    <argument>buildx</argument>
+                                    <argument>build</argument>
+                                    <argument>--load</argument>
+                                    <argument>--no-cache</argument>
+                                    <argument>-t</argument>
+                                    <argument>${docker.hub}/${docker.repo}:${docker.tag}</argument>
+                                    <argument>${project.basedir}</argument>
+                                    <argument>--build-arg</argument>
+                                    <argument>VERSION=${project.version}</argument>
+                                    <argument>--file=src/main/docker/Dockerfile</argument>
+                                </arguments>
+                            </configuration>
+                        </execution>
+                        <execution>
+                            <id>docker-verify</id>
+                            <goals>
+                                <goal>exec</goal>
+                            </goals>
+                            <phase>verify</phase>
+                            <configuration>
+                                <skip>${docker.verify.skip}</skip>
+                                <environmentVariables>
+                                    <DOCKER_BUILDKIT>1</DOCKER_BUILDKIT>
+                                </environmentVariables>
+                                <executable>docker</executable>
+                                <workingDirectory>${project.basedir}</workingDirectory>
+                                <arguments>
+                                    <argument>run</argument>
+                                    <argument>--rm</argument>
+                                    <argument>${docker.hub}/${docker.repo}:${docker.tag}</argument>
+                                    <argument>bash</argument>
+                                    <argument>./bin/seatunnel.sh</argument>
+                                    <argument>-e</argument>
+                                    <argument>local</argument>
+                                    <argument>-c</argument>
+                                    <argument>config/v2.batch.config.template</argument>
+                                </arguments>
+                            </configuration>
+                        </execution>
+                        <execution>
+                            <id>docker-push</id>
+                            <goals>
+                                <goal>exec</goal>
+                            </goals>
+                            <phase>install</phase>
+                            <configuration>
+                                <skip>${docker.push.skip}</skip>
+                                <environmentVariables>
+                                    <DOCKER_BUILDKIT>1</DOCKER_BUILDKIT>
+                                </environmentVariables>
+                                <executable>docker</executable>
+                                <workingDirectory>${project.basedir}</workingDirectory>
+                                <arguments>
+                                    <argument>buildx</argument>
+                                    <argument>build</argument>
+                                    <argument>--platform</argument>
+                                    <argument>linux/amd64,linux/arm64</argument>
+                                    <argument>--no-cache</argument>
+                                    <argument>--push</argument>
+                                    <argument>-t</argument>
+                                    <argument>${docker.hub}/${docker.repo}:${docker.tag}</argument>
+                                    <argument>${project.basedir}</argument>
+                                    <argument>--build-arg</argument>
+                                    <argument>VERSION=${revision}</argument>
+                                    <argument>--file=src/main/docker/Dockerfile</argument>
+                                </arguments>
+                            </configuration>
+                        </execution>
+                    </executions>
+                </plugin>
             </plugins>
         </pluginManagement>
 
diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml
index e919293aab..fb6935f089 100644
--- a/seatunnel-dist/pom.xml
+++ b/seatunnel-dist/pom.xml
@@ -66,7 +66,6 @@
                             <appendAssemblyId>true</appendAssemblyId>
                         </configuration>
                     </execution>
-
                 </executions>
             </plugin>
         </plugins>
@@ -83,6 +82,9 @@
                 </property>
             </activation>
             <properties>
+                <docker.build.skip>false</docker.build.skip>
+                <docker.verify.skip>false</docker.verify.skip>
+                <docker.push.skip>false</docker.push.skip>
                 <mysql.version>8.0.27</mysql.version>
                 <postgresql.version>42.4.3</postgresql.version>
                 <postgis.jdbc.version>2.5.1</postgis.jdbc.version>
@@ -965,5 +967,22 @@
                 </plugins>
             </build>
         </profile>
+        <profile>
+            <id>docker</id>
+            <activation>
+                <property>
+                    <name>release</name>
+                    <value>false</value>
+                </property>
+            </activation>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>exec-maven-plugin</artifactId>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
     </profiles>
 </project>
diff --git a/seatunnel-dist/src/main/docker/Dockerfile b/seatunnel-dist/src/main/docker/Dockerfile
new file mode 100644
index 0000000000..7d040ee41a
--- /dev/null
+++ b/seatunnel-dist/src/main/docker/Dockerfile
@@ -0,0 +1,12 @@
+FROM openjdk:8
+
+ARG VERSION
+
+COPY ./target/apache-seatunnel-${VERSION}-bin.tar.gz /opt/
+
+RUN cd /opt && \
+    tar -zxvf apache-seatunnel-${VERSION}-bin.tar.gz && \
+    mv apache-seatunnel-${VERSION} seatunnel && \
+    rm apache-seatunnel-${VERSION}-bin.tar.gz
+
+WORKDIR /opt/seatunnel