You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uniffle.apache.org by ck...@apache.org on 2023/02/15 13:35:11 UTC
[incubator-uniffle] branch master updated: [#580] chore: speed up CI workflows (#602)
This is an automated email from the ASF dual-hosted git repository.
ckj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 9359ef21 [#580] chore: speed up CI workflows (#602)
9359ef21 is described below
commit 9359ef21cd2054e4129be0ac0815ccf1eec09a8a
Author: Kaijie Chen <ck...@apache.org>
AuthorDate: Wed Feb 15 21:35:04 2023 +0800
[#580] chore: speed up CI workflows (#602)
### What changes were proposed in this pull request?
1. Split unit test and integration test
2. Enable maven parallel build (`-T1C`)
3. Do not use matrix in deploy/kubernetes workflow
4. Delay some secondary workflows
### Why are the changes needed?
Resolve #580
Make CI faster.
In case of failure, rerun the failed part only.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
CI.
---
.github/workflows/build.yml | 22 ++++++++++++++++-----
.github/workflows/parallel.yml | 2 +-
.github/workflows/sequential.yml | 6 +++---
.github/workflows/{deploy.yml => single.yml} | 29 ++++++++++++++--------------
4 files changed, 35 insertions(+), 24 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d7b9579a..76700b77 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -38,12 +38,14 @@ jobs:
summary: "grep -r '!?????' --include='rat.txt' | awk '{print $3}'"
spotbugs:
+ needs: [checkstyle, license] # delay execution
uses: ./.github/workflows/sequential.yml
with:
maven-args: test-compile spotbugs:check
cache-key: spotbugs
java-11:
+ needs: [spotbugs] # delay execution
name: 'compile'
uses: ./.github/workflows/sequential.yml
with:
@@ -52,6 +54,7 @@ jobs:
java-version: '11'
java-17:
+ needs: [spotbugs] # delay execution
name: 'compile'
uses: ./.github/workflows/sequential.yml
with:
@@ -59,13 +62,22 @@ jobs:
cache-key: package
java-version: '17'
- package:
+ unit:
uses: ./.github/workflows/parallel.yml
with:
- maven-args: package
+ maven-args: package -Dtest=!org.apache.uniffle.test.**
reports-path: "**/target/surefire-reports/*.txt"
- deploy:
- uses: ./.github/workflows/deploy.yml
+ integration:
+ uses: ./.github/workflows/parallel.yml
with:
- maven-args: package
+ maven-args: package -Dtest=org.apache.uniffle.test.**
+ reports-path: "**/target/surefire-reports/*.txt"
+
+ kubernetes:
+ needs: [checkstyle, license] # delay execution
+ uses: ./.github/workflows/single.yml
+ with:
+ maven-args: package -Pkubernetes -DskipUTs -DskipITs
+ cache-key: package
+ go-version: '1.17'
diff --git a/.github/workflows/parallel.yml b/.github/workflows/parallel.yml
index 67c06293..a6f115e8 100644
--- a/.github/workflows/parallel.yml
+++ b/.github/workflows/parallel.yml
@@ -79,7 +79,7 @@ jobs:
mvn-${{ inputs.java-version }}-package-${{ matrix.profile }}-
mvn-${{ inputs.java-version }}-package-
- name: Execute `mvn ${{ inputs.maven-args }} -P${{ matrix.profile }}`
- run: mvn -B -fae ${{ inputs.maven-args }} -P${{ matrix.profile }} | tee /tmp/maven.log
+ run: mvn -T1C -B -fae ${{ inputs.maven-args }} -P${{ matrix.profile }} | tee /tmp/maven.log
shell: bash
- name: Summary of failures
if: ${{ failure() && inputs.summary != '' }}
diff --git a/.github/workflows/sequential.yml b/.github/workflows/sequential.yml
index 66a766bf..e9b0927d 100644
--- a/.github/workflows/sequential.yml
+++ b/.github/workflows/sequential.yml
@@ -69,13 +69,13 @@ jobs:
restore-keys: |
mvn-${{ inputs.java-version }}-${{ inputs.cache-key }}-
- name: Execute `mvn ${{ inputs.maven-args }} -Pspark3`
- run: mvn -B -fae ${{ inputs.maven-args }} -Pspark3 | tee -a /tmp/maven.log;
+ run: mvn -T1C -B -fae ${{ inputs.maven-args }} -Pspark3 | tee -a /tmp/maven.log;
shell: bash
- name: Execute `mvn ${{ inputs.maven-args }} -Pspark2`
- run: mvn -B -fae ${{ inputs.maven-args }} -Pspark2 | tee -a /tmp/maven.log;
+ run: mvn -T1C -B -fae ${{ inputs.maven-args }} -Pspark2 | tee -a /tmp/maven.log;
shell: bash
- name: Execute `mvn ${{ inputs.maven-args }} -Pmr`
- run: mvn -B -fae ${{ inputs.maven-args }} -Pmr | tee -a /tmp/maven.log;
+ run: mvn -T1C -B -fae ${{ inputs.maven-args }} -Pmr | tee -a /tmp/maven.log;
shell: bash
- name: Summary of failures
if: ${{ failure() && inputs.summary != '' }}
diff --git a/.github/workflows/deploy.yml b/.github/workflows/single.yml
similarity index 78%
rename from .github/workflows/deploy.yml
rename to .github/workflows/single.yml
index 803de485..6d59fd76 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/single.yml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-name: All Deploy Profiles in Parallel
+name: Single Maven Profile
on:
workflow_call:
@@ -23,6 +23,10 @@ on:
maven-args:
required: true
type: string
+ cache-key:
+ default: ''
+ required: false
+ type: string
summary:
default: "grep -e '^\\[ERROR\\]' /tmp/maven.log || true"
required: false
@@ -44,19 +48,14 @@ on:
required: false
type: string
go-version:
- default: '1.17'
+ default: ''
required: false
type: string
jobs:
- package:
+ maven:
runs-on: ubuntu-20.04
- strategy:
- matrix:
- profile:
- - kubernetes
- fail-fast: false
- name: ${{ matrix.profile }}
+ name: java ${{ inputs.java-version }} + go ${{ inputs.go-version }}
steps:
- name: Checkout project
uses: actions/checkout@v3
@@ -66,20 +65,20 @@ jobs:
java-version: ${{ inputs.java-version }}
distribution: ${{ inputs.jdk-distro }}
- name: Cache local Maven repository
+ if: ${{ inputs.cache-key != '' }}
uses: actions/cache@v3
with:
path: ~/.m2/repository
- key: mvn-${{ inputs.java-version }}-package-${{ matrix.profile }}-${{ hashFiles('**/pom.xml') }}
+ key: mvn-${{ inputs.java-version }}-${{ inputs.cache-key }}-${{ hashFiles('**/pom.xml') }}
restore-keys: |
- mvn-${{ inputs.java-version }}-package-${{ matrix.profile }}-
- mvn-${{ inputs.java-version }}-package-
+ mvn-${{ inputs.java-version }}-${{ inputs.cache-key }}-
- name: Set up Go ${{ inputs.go-version }}
if: ${{ inputs.go-version != '' }}
uses: actions/setup-go@v3
with:
go-version: ${{ inputs.go-version }}
- - name: Execute `mvn ${{ inputs.maven-args }} -P${{ matrix.profile }}`
- run: mvn -B -fae ${{ inputs.maven-args }} -P${{ matrix.profile }} -DskipUTs -DskipITs | tee /tmp/maven.log
+ - name: Execute `mvn ${{ inputs.maven-args }}`
+ run: mvn -T1C -B -fae ${{ inputs.maven-args }} | tee /tmp/maven.log
shell: bash
- name: Summary of failures
if: ${{ failure() && inputs.summary != '' }}
@@ -89,7 +88,7 @@ jobs:
if: ${{ failure() && inputs.reports-path != '' }}
uses: actions/upload-artifact@v3
with:
- name: ${{ inputs.reports-name }}-${{ matrix.profile }}
+ name: ${{ inputs.reports-name }}
path: ${{ inputs.reports-path }}
continue-on-error: true
- name: Upload coverage to Codecov