You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ty...@apache.org on 2023/03/13 05:59:19 UTC
[incubator-seatunnel] branch dev updated: [Improve][CI] Improve ci steps (#4314)
This is an automated email from the ASF dual-hosted git repository.
tyrantlucifer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 560e00361 [Improve][CI] Improve ci steps (#4314)
560e00361 is described below
commit 560e0036157655c0256f3497092e6335728de59e
Author: Tyrantlucifer <Ty...@gmail.com>
AuthorDate: Mon Mar 13 13:59:11 2023 +0800
[Improve][CI] Improve ci steps (#4314)
* [Improve][CI] Improve ci steps
* [Improve][CI] Improve ci steps
* [Improve][CI] Improve ci steps
* [Improve][CI] Split the update integration modules
* [Improve][CI] Fix error ci step
* [Improve][CI] Avoid run all it cases
---
.github/workflows/backend.yml | 66 +++++++++++++++++-----
tools/update_modules_check/update_modules_check.py | 20 ++++++-
2 files changed, 69 insertions(+), 17 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 11031398e..84b818d7d 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -267,11 +267,13 @@ jobs:
java-version: '8'
cache: 'maven'
- name: Install
- run: >-
- ./mvnw -B -q install -DskipTests
- -D"maven.test.skip"=true
- -D"maven.javadoc.skip"=true
- -D"license.skipAddThirdParty"
+ uses: nick-fields/retry@v2
+ with:
+ timeout_minutes: 40
+ max_attempts: 3
+ retry_on: error
+ command: |
+ ./mvnw -B -q install -DskipTests -D"maven.test.skip"=true -D"maven.javadoc.skip"=true -D"license.skipAddThirdParty"
- name: Check Dependencies Licenses
run: tools/dependencies/checkLicense.sh
@@ -283,7 +285,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest', 'windows-latest' ]
- timeout-minutes: 30
+ timeout-minutes: 36
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -294,8 +296,13 @@ jobs:
cache: 'maven'
- name: run all modules unit test
if: needs.changes.outputs.api == 'true'
- run: |
- ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"license.skipAddThirdParty"=true --no-snapshot-updates
+ uses: nick-fields/retry@v2
+ with:
+ timeout_minutes: 36
+ max_attempts: 3
+ retry_on: error
+ command: |
+ ./mvnw -B -T 1C clean verify -D"maven.test.skip"=false -D"license.skipAddThirdParty"=true --no-snapshot-updates
env:
MAVEN_OPTS: -Xmx4096m
@@ -306,7 +313,32 @@ jobs:
env:
MAVEN_OPTS: -Xmx4096m
- updated-modules-integration-test:
+ updated-modules-integration-test-part-1:
+ needs: [ changes, sanity-check ]
+ if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ java: [ '8', '11' ]
+ os: [ 'ubuntu-latest' ]
+ timeout-minutes: 90
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up JDK ${{ matrix.java }}
+ uses: actions/setup-java@v3
+ with:
+ java-version: ${{ matrix.java }}
+ distribution: 'temurin'
+ cache: 'maven'
+ - name: run updated modules integration test
+ if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
+ run: |
+ sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 2 0`
+ ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci
+ env:
+ MAVEN_OPTS: -Xmx2048m
+
+ updated-modules-integration-test-part-2:
needs: [ changes, sanity-check ]
if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
runs-on: ${{ matrix.os }}
@@ -326,7 +358,8 @@ jobs:
- name: run updated modules integration test
if: needs.changes.outputs.api == 'false' && needs.changes.outputs.it-modules != ''
run: |
- ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl ${{needs.changes.outputs.it-modules}} -am -Pci
+ sub_modules=`python tools/update_modules_check/update_modules_check.py sub_update_it_module ${{needs.changes.outputs.it-modules}} 2 1`
+ [ ! -z $sub_modules ] && ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl $sub_modules -am -Pci
env:
MAVEN_OPTS: -Xmx2048m
@@ -338,7 +371,7 @@ jobs:
matrix:
java: [ '8', '11' ]
os: [ 'ubuntu-latest' ]
- timeout-minutes: 30
+ timeout-minutes: 45
steps:
- uses: actions/checkout@v2
- name: Set up JDK ${{ matrix.java }}
@@ -349,8 +382,13 @@ jobs:
cache: 'maven'
- name: run some modules integration test
if: needs.changes.outputs.api == 'true'
- run: |
- ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-seatunnel-e2e-base -am -Pci
+ uses: nick-fields/retry@v2
+ with:
+ timeout_minutes: 45
+ max_attempts: 3
+ retry_on: error
+ command: |
+ ./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :connector-seatunnel-e2e-base -am -Pci
env:
MAVEN_OPTS: -Xmx4096m
@@ -377,7 +415,6 @@ jobs:
with:
timeout_minutes: 30
max_attempts: 3
- continue_on_error: true
command: |
./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-1 -am -Pci
env:
@@ -406,7 +443,6 @@ jobs:
with:
timeout_minutes: 30
max_attempts: 3
- continue_on_error: true
command: |
./mvnw -T 1C -B verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-2 -am -Pci
env:
diff --git a/tools/update_modules_check/update_modules_check.py b/tools/update_modules_check/update_modules_check.py
index 4c21578a2..be1d9d06d 100644
--- a/tools/update_modules_check/update_modules_check.py
+++ b/tools/update_modules_check/update_modules_check.py
@@ -148,17 +148,31 @@ def get_deleted_modules(files):
output_module = output_module[1:len(output_module)]
print(output_module)
+
def get_sub_it_modules(modules, total_num, current_num):
modules_arr = modules.split(",")
modules_arr.remove("connector-jdbc-e2e")
output = ""
- for i,module in enumerate(modules_arr):
+ for i, module in enumerate(modules_arr):
if len(module) > 0 and i % int(total_num) == int(current_num):
output = output + ",:" + module
output = output[1:len(output)]
print(output)
+
+def get_sub_update_it_modules(modules, total_num, current_num):
+ final_modules = list()
+ # :connector-jdbc-e2e-common,:connector-jdbc-e2e-part-1 --> connector-jdbc-e2e-common,:connector-jdbc-e2e-part-1
+ modules = modules[1:]
+ # connector-jdbc-e2e-common,:connector-jdbc-e2e-part-1 --> [connector-jdbc-e2e-common, connector-jdbc-e2e-part-1]
+ module_list = modules.split(",:")
+ for i, module in enumerate(module_list):
+ if len(module) > 0 and i % int(total_num) == int(current_num):
+ final_modules.append(":" + module)
+ print(",".join(final_modules))
+
+
def main(argv):
if argv[1] == "cv2":
get_cv2_modules(argv[2])
@@ -188,7 +202,9 @@ def main(argv):
remove_deleted_modules(argv[2], argv[3])
elif argv[1] == "sub_it_module":
get_sub_it_modules(argv[2], argv[3], argv[4])
+ elif argv[1] == "sub_update_it_module":
+ get_sub_update_it_modules(argv[2], argv[3], argv[4])
if __name__ == "__main__":
- main(sys.argv)
\ No newline at end of file
+ main(sys.argv)