You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ga...@apache.org on 2022/10/26 11:33:28 UTC
[incubator-seatunnel] branch dev updated: [Improve][CI] improve ci and only build the changed modules (#3179)
This is an automated email from the ASF dual-hosted git repository.
gaojun2048 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 0f2b6b881 [Improve][CI] improve ci and only build the changed modules (#3179)
0f2b6b881 is described below
commit 0f2b6b8817a68b124b3b81dd10b1a565faaeb01d
Author: Eric <ga...@gmail.com>
AuthorDate: Wed Oct 26 19:33:22 2022 +0800
[Improve][CI] improve ci and only build the changed modules (#3179)
* improve ci and only build by the update modules
* remove no use outputs
---
.github/workflows/backend.yml | 157 ++++++++++++++++++++-
pom.xml | 14 +-
.../e2e/flink/v2/file/FakeSourceToFileIT.java | 1 -
tools/update_modules_check/update_modules_check.py | 71 ++++++++++
4 files changed, 235 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 2b6bd8c86..8595cf049 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -135,10 +135,140 @@ jobs:
- name: Check Dependencies Licenses
run: tools/dependencies/checkLicense.sh
+ changes:
+ runs-on: ubuntu-latest
+ # To prevent error when there's no base branch
+ if: github.repository == 'apache/incubator-seatunnel'
+ timeout-minutes: 10
+ outputs:
+ api: ${{ steps.filter.outputs.api }}
+ ut-modules: ${{ steps.ut-modules.outputs.modules }}
+ it-modules: ${{ steps.it-modules.outputs.modules }}
+ steps:
+ - uses: actions/checkout@v3 # required for push event
+ - uses: actions/setup-python@v4
+ with:
+ python-version: '2.7'
+ - name: Check for file changes
+ uses: dorny/paths-filter@v2
+ id: filter
+ with:
+ token: ${{ github.token }}
+ list-files: json
+ # The following is a single composite pattern that allows next CI steps,
+ # the pattern is in form of [not (foo or bar)] to be safe.
+ # When new files come in, the CI will NOT ignore them unless listed,
+ # so remember to extend here if they do not serve functional purposes.
+ # NOTE: careful with using ** in expression, keep !**/{old, new things}.
+ filters: |
+ cv2:
+ - "seatunnel-connectors-v2/**"
+ cv2-flink-e2e:
+ - "seatunnel-e2e/seatunnel-flink-connector-v2-e2e/**"
+ cv2-spark-e2e:
+ - "seatunnel-e2e/seatunnel-spark-connector-v2-e2e/**"
+ cv2-e2e:
+ - "seatunnel-e2e/seatunnel-connector-v2-e2e/**"
+ api:
+ - "seatunnel-api"
+ - "seatunnel-apis"
+ - "seatunnel-common"
+ - "seatunnel-config"
+ - "seatunnel-connectors"
+ - "seatunnel-core"
+ - "seatunnel-dist"
+ - "seatunnel-e2e/seatunnel-e2e-common"
+ - "seatunnel-formats"
+ - "seatunnel-plugin-discovery"
+ - "seatunnel-transforms"
+ - "seatunnel-translation"
+ - "seatunnel-e2e/seatunnel-flink-e2e"
+ - "seatunnel-e2e/seatunnel-spark-e2e"
+ - "seatunnel-connectors"
+ - "plugin-mapping.properties"
+ - "**/tools/**"
+ - "pom.xml"
+ - "**/workflows/**"
+ engine:
+ - "seatunnel-engine"
+ engine-e2e:
+ - "seatunnel-e2e/seatunnel-engine-e2e"
+
+ - name: Check Connector V2 Update
+ id: cv2-modules
+ if: ${{ steps.filter.outputs.cv2 == 'true' }}
+ run: |
+ update_files='${{ steps.filter.outputs.cv2_files }}'
+ modules=`python tools/update_modules_check/update_modules_check.py cv2 $update_files`
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
+ - name: Check Flink Connector V2 E2E Update
+ id: cv2-flink-e2e-modules
+ if: ${{ steps.filter.outputs.cv2-flink-e2e == 'true' }}
+ run: |
+ update_files='${{ steps.filter.outputs.cv2-flink-e2e_files }}'
+ modules=`python tools/update_modules_check/update_modules_check.py cv2-flink-e2e $update_files`
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
+ - name: Check Spark Connector V2 E2E Update
+ id: cv2-spark-e2e-modules
+ if: ${{ steps.filter.outputs.cv2-spark-e2e == 'true' }}
+ run: |
+ update_files='${{ steps.filter.outputs.cv2-spark-e2e_files }}'
+ modules=`python tools/update_modules_check/update_modules_check.py cv2-spark-e2e $update_files`
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
+ - name: Check Connector V2 E2E Update
+ id: cv2-e2e-modules
+ if: ${{ steps.filter.outputs.cv2-e2e == 'true' }}
+ run: |
+ update_files='${{ steps.filter.outputs.cv2-e2e_files }}'
+ modules=`python tools/update_modules_check/update_modules_check.py cv2-e2e $update_files`
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
+ - name: Check Engine Update
+ id: engine-modules
+ if: ${{ steps.filter.outputs.engine == 'true' }}
+ run: |
+ update_files='${{ steps.filter.outputs.engine_files }}'
+ modules=`python tools/update_modules_check/update_modules_check.py engine $update_files`
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
+ - name: Check Engine E2E Update
+ id: engine-e2e-modules
+ if: ${{ steps.filter.outputs.engine-e2e == 'true' }}
+ run: |
+ update_files='${{ steps.filter.outputs.engine-e2e_files }}'
+ modules=`python tools/update_modules_check/update_modules_check.py engine-e2e $update_files`
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
+ - name: Make unit test modules
+ id: ut-modules
+ if: ${{ steps.filter.outputs.api == 'false' }}
+ run: |
+ modules='${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
+ modules=${modules: 1}
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
+ - name: Make integration test modules
+ id: it-modules
+ if: ${{ steps.filter.outputs.api == 'false' }}
+ run: |
+ modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ steps.cv2-flink-e2e-modules.outputs.modules }}${{ steps.cv2-spark-e2e-modules.outputs.modules }}${{ steps.engine-e2e-modules.outputs.modules }}${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
+ modules=${modules: 1}
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
unit-test:
- name: Unit Test
+ needs: [ changes, sanity-check ]
runs-on: ${{ matrix.os }}
- needs: [ sanity-check ]
strategy:
matrix:
java: [ '8', '11' ]
@@ -152,16 +282,23 @@ jobs:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: 'maven'
- - name: Run Unit tests
+ - name: run all modules unit test
+ if: needs.changes.outputs.api == 'true'
run: |
./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
env:
MAVEN_OPTS: -Xmx2048m
+ - name: run updated modules unit test
+ if: needs.changes.outputs.api == 'false'
+ run: |
+ ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.ut-modules}} -am -amd -Pno_dist
+ env:
+ MAVEN_OPTS: -Xmx2048m
+
integration-test:
- name: Integration Test
+ needs: [ changes, sanity-check ]
runs-on: ${{ matrix.os }}
- needs: [ sanity-check ]
strategy:
matrix:
java: [ '8', '11' ]
@@ -175,8 +312,16 @@ jobs:
java-version: ${{ matrix.java }}
distribution: 'temurin'
cache: 'maven'
- - name: Run Integration tests
+ - name: run all modules integration test
+ if: needs.changes.outputs.api == 'true'
run: |
./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates
env:
MAVEN_OPTS: -Xmx2048m
+
+ - name: run updated modules integration test
+ if: needs.changes.outputs.api == 'false'
+ run: |
+ ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"checkstyle.skip"=true -D"scalastyle.skip"=true -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -amd -Pno_dist
+ env:
+ MAVEN_OPTS: -Xmx2048m
diff --git a/pom.xml b/pom.xml
index a4e3f50db..fa633c0bc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -87,7 +87,6 @@
<module>seatunnel-translation</module>
<module>seatunnel-plugin-discovery</module>
<module>seatunnel-formats</module>
- <module>seatunnel-dist</module>
<module>seatunnel-engine</module>
</modules>
@@ -104,6 +103,7 @@
<modules>
<module>seatunnel-examples</module>
<module>seatunnel-e2e</module>
+ <module>seatunnel-dist</module>
</modules>
</profile>
<profile>
@@ -114,6 +114,18 @@
<value>true</value>
</property>
</activation>
+ <modules>
+ <module>seatunnel-dist</module>
+ </modules>
+ </profile>
+ <profile>
+ <id>no_dist</id>
+ <activation>
+ <property>
+ <name>release</name>
+ <value>false</value>
+ </property>
+ </activation>
</profile>
</profiles>
diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-file-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/file/FakeSourceToFileIT.java b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-file-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/file/FakeSourceToFileIT.java
index d6bfdefc5..43a0b3fdd 100644
--- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-file-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/file/FakeSourceToFileIT.java
+++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/connector-file-flink-e2e/src/test/java/org/apache/seatunnel/e2e/flink/v2/file/FakeSourceToFileIT.java
@@ -26,7 +26,6 @@ import org.testcontainers.containers.Container;
import java.io.IOException;
public class FakeSourceToFileIT extends FlinkContainer {
-
/**
* fake source -> local text file sink
*/
diff --git a/tools/update_modules_check/update_modules_check.py b/tools/update_modules_check/update_modules_check.py
new file mode 100644
index 000000000..2a887ae57
--- /dev/null
+++ b/tools/update_modules_check/update_modules_check.py
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/python
+import json
+import sys
+
+def get_cv2_modules(files):
+ get_modules(files, 1, "connector-", "seatunnel-connectors-v2")
+
+def get_cv2_flink_e2e_modules(files):
+ get_modules(files, 2, "connector-", "seatunnel-e2e/seatunnel-flink-connector-v2-e2e")
+
+def get_cv2_spark_e2e_modules(files):
+ get_modules(files, 2, "connector-", "seatunnel-e2e/seatunnel-spark-connector-v2-e2e")
+
+def get_cv2_e2e_modules(files):
+ get_modules(files, 2, "connector-", "seatunnel-e2e/seatunnel-connector-v2-e2e")
+
+def get_engine_modules(files):
+ get_modules(files, 1, "seatunnel-", "seatunnel-engine")
+
+def get_engine_e2e_modules(files):
+ get_modules(files, 2, "connector-", "seatunnel-e2e/seatunnel-engine-e2e")
+
+def get_modules(files, index, start_pre, root_module):
+ update_files = json.loads(files)
+ modules_name_set = set([])
+ for file in update_files:
+ module_name = file.split('/')[index]
+ if module_name.startswith(start_pre):
+ modules_name_set.add(root_module + "/" + module_name)
+
+ output_module = ""
+ if len(modules_name_set) > 0:
+ for module in modules_name_set:
+ output_module = output_module + "," + module
+
+ else:
+ output_module = root_module
+
+ print(output_module)
+
+def main(argv):
+ if argv[1] == "cv2":
+ get_cv2_modules(argv[2])
+ elif argv[1] == "cv2-e2e":
+ get_cv2_e2e_modules(argv[2])
+ elif argv[1] == "cv2-flink-e2e":
+ get_cv2_flink_e2e_modules(argv[2])
+ elif argv[1] == "cv2-spark-e2e":
+ get_cv2_spark_e2e_modules(argv[2])
+ elif argv[1] == "engine":
+ get_engine_modules(argv[2])
+ elif argv[1] == "engine-e2e":
+ get_engine_e2e_modules(argv[2])
+
+if __name__ == "__main__":
+ main(sys.argv)
\ No newline at end of file