You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@seatunnel.apache.org by ki...@apache.org on 2022/10/26 14:33:51 UTC
[incubator-seatunnel] branch dev updated: [Improve][CI] Improve ci and remove build task and only api or engine changed need run license check (#3194)
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 033acd2f8 [Improve][CI] Improve ci and remove build task and only api or engine changed need run license check (#3194)
033acd2f8 is described below
commit 033acd2f88f99feeb053275d553f652374b693db
Author: Eric <ga...@gmail.com>
AuthorDate: Wed Oct 26 22:33:43 2022 +0800
[Improve][CI] Improve ci and remove build task and only api or engine changed need run license check (#3194)
* improve ci and only build by the update modules
* remove no use outputs
* improve ci and remove build test and connector update will not check license
---
.github/workflows/backend.yml | 156 ++++++++++++++++++++++++++++++++++--------
.github/workflows/codeql.yaml | 9 +--
2 files changed, 131 insertions(+), 34 deletions(-)
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 8595cf049..184da108e 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -79,40 +79,142 @@ jobs:
[[ ${{ needs.code-style.result }} == 'success' ]] || exit 1;
[[ ${{ needs.dead-link.result }} == 'success' ]] || exit 1;
- build:
+ changes:
+ runs-on: ubuntu-latest
+ # To prevent error when there's no base branch
if: github.repository == 'apache/incubator-seatunnel'
- name: Build
- needs: [ sanity-check ]
- strategy:
- matrix:
- java: [ '8', '11' ]
- os: [ 'ubuntu-latest', 'windows-latest' ]
- runs-on: ${{ matrix.os }}
- timeout-minutes: 80
+ timeout-minutes: 10
+ outputs:
+ api: ${{ steps.filter.outputs.api }}
+ engine: ${{ steps.filter.outputs.engine }}
+ ut-modules: ${{ steps.ut-modules.outputs.modules }}
+ it-modules: ${{ steps.it-modules.outputs.modules }}
steps:
- - uses: actions/checkout@v3
+ - uses: actions/checkout@v3 # required for push event
+ - uses: actions/setup-python@v4
with:
- submodules: true
- - uses: actions/setup-java@v3
+ python-version: '2.7'
+ - name: Check for file changes
+ uses: dorny/paths-filter@v2
+ id: filter
with:
- distribution: 'temurin'
- java-version: ${{ matrix.java }}
- cache: 'maven'
- - name: Build distribution tar
- run: >-
- ./mvnw -B install scalastyle:check
- -D"maven.test.skip"=true
- -D"checkstyle.skip"=true
- -D"license.skipAddThirdParty"=true
- -D"http.keepAlive"=false
- -D"maven.wagon.http.pool"=false
- -D"maven.wagon.http.retryHandler.count"=3
- -D"maven.wagon.httpconnectionManager.ttlSeconds"=120
+ token: ${{ github.token }}
+ list-files: json
+ # The following is a single composite pattern that allows next CI steps,
+ # the pattern is in form of [not (foo or bar)] to be safe.
+ # When new files come in, the CI will NOT ignore them unless listed,
+ # so remember to extend here if they do not serve functional purposes.
+ # NOTE: careful with using ** in expression, keep !**/{old, new things}.
+ filters: |
+ cv2:
+ - "seatunnel-connectors-v2/**"
+ cv2-flink-e2e:
+ - "seatunnel-e2e/seatunnel-flink-connector-v2-e2e/**"
+ cv2-spark-e2e:
+ - "seatunnel-e2e/seatunnel-spark-connector-v2-e2e/**"
+ cv2-e2e:
+ - "seatunnel-e2e/seatunnel-connector-v2-e2e/**"
+ api:
+ - "seatunnel-api"
+ - "seatunnel-apis"
+ - "seatunnel-common"
+ - "seatunnel-config"
+ - "seatunnel-connectors"
+ - "seatunnel-core"
+ - "seatunnel-dist"
+ - "seatunnel-e2e/seatunnel-e2e-common"
+ - "seatunnel-formats"
+ - "seatunnel-plugin-discovery"
+ - "seatunnel-transforms"
+ - "seatunnel-translation"
+ - "seatunnel-e2e/seatunnel-flink-e2e"
+ - "seatunnel-e2e/seatunnel-spark-e2e"
+ - "seatunnel-connectors"
+ - "plugin-mapping.properties"
+ - "**/tools/**"
+ - "pom.xml"
+ - "**/workflows/**"
+ engine:
+ - "seatunnel-engine"
+ engine-e2e:
+ - "seatunnel-e2e/seatunnel-engine-e2e"
+
+ - name: Check Connector V2 Update
+ id: cv2-modules
+ if: ${{ steps.filter.outputs.cv2 == 'true' }}
+ run: |
+ update_files='${{ steps.filter.outputs.cv2_files }}'
+ modules=`python tools/update_modules_check/update_modules_check.py cv2 $update_files`
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
+ - name: Check Flink Connector V2 E2E Update
+ id: cv2-flink-e2e-modules
+ if: ${{ steps.filter.outputs.cv2-flink-e2e == 'true' }}
+ run: |
+ update_files='${{ steps.filter.outputs.cv2-flink-e2e_files }}'
+ modules=`python tools/update_modules_check/update_modules_check.py cv2-flink-e2e $update_files`
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
+ - name: Check Spark Connector V2 E2E Update
+ id: cv2-spark-e2e-modules
+ if: ${{ steps.filter.outputs.cv2-spark-e2e == 'true' }}
+ run: |
+ update_files='${{ steps.filter.outputs.cv2-spark-e2e_files }}'
+ modules=`python tools/update_modules_check/update_modules_check.py cv2-spark-e2e $update_files`
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
+ - name: Check Connector V2 E2E Update
+ id: cv2-e2e-modules
+ if: ${{ steps.filter.outputs.cv2-e2e == 'true' }}
+ run: |
+ update_files='${{ steps.filter.outputs.cv2-e2e_files }}'
+ modules=`python tools/update_modules_check/update_modules_check.py cv2-e2e $update_files`
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
+ - name: Check Engine Update
+ id: engine-modules
+ if: ${{ steps.filter.outputs.engine == 'true' }}
+ run: |
+ update_files='${{ steps.filter.outputs.engine_files }}'
+ modules=`python tools/update_modules_check/update_modules_check.py engine $update_files`
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
+ - name: Check Engine E2E Update
+ id: engine-e2e-modules
+ if: ${{ steps.filter.outputs.engine-e2e == 'true' }}
+ run: |
+ update_files='${{ steps.filter.outputs.engine-e2e_files }}'
+ modules=`python tools/update_modules_check/update_modules_check.py engine-e2e $update_files`
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
+ - name: Make unit test modules
+ id: ut-modules
+ if: ${{ steps.filter.outputs.api == 'false' }}
+ run: |
+ modules='${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
+ modules=${modules: 1}
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
+
+ - name: Make integration test modules
+ id: it-modules
+ if: ${{ steps.filter.outputs.api == 'false' }}
+ run: |
+ modules='${{ steps.cv2-e2e-modules.outputs.modules }}${{ steps.cv2-flink-e2e-modules.outputs.modules }}${{ steps.cv2-spark-e2e-modules.outputs.modules }}${{ steps.engine-e2e-modules.outputs.modules }}${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'
+ modules=${modules: 1}
+ echo $modules
+ echo "modules=$modules" >> $GITHUB_OUTPUT
dependency-license:
- if: github.repository == 'apache/incubator-seatunnel'
+ if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true'
name: Dependency licenses
- needs: [ sanity-check ]
+ needs: [ changes, sanity-check ]
runs-on: ubuntu-latest
timeout-minutes: 40
steps:
diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml
index 237bebd08..8879d7a43 100644
--- a/.github/workflows/codeql.yaml
+++ b/.github/workflows/codeql.yaml
@@ -16,13 +16,8 @@
name: "CodeQL"
on:
- push:
- pull_request:
- branches: [dev]
- paths-ignore:
- - 'docs/**'
- - '**/*.md'
- - 'seatunnel-ui/**'
+ schedule:
+ - cron: '0 0 12 * * ?'
jobs:
analyze: