[ozone] branch master updated: HDDS-6617. Skip compile(11) for non-java changes (#3324)

[ad...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git


The following commit(s) were added to refs/heads/master by this push:
     new 497c7a40b8 HDDS-6617. Skip compile(11) for non-java changes (#3324)
497c7a40b8 is described below

commit 497c7a40b856be665deb232a5304f4e87cf76481
Author: Doroszlai, Attila <64...@users.noreply.github.com>
AuthorDate: Mon Apr 25 10:21:35 2022 +0200

    HDDS-6617. Skip compile(11) for non-java changes (#3324)
---
 .github/workflows/post-commit.yml       | 45 ++++++++++++++++++++++++++++-----
 dev-support/ci/selective_ci_checks.bats | 22 ++++++++++++++++
 dev-support/ci/selective_ci_checks.sh   | 20 +++++++++++++++
 3 files changed, 81 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml
index 83982af1de..6f10c7728a 100644
--- a/.github/workflows/post-commit.yml
+++ b/.github/workflows/post-commit.yml
@@ -34,6 +34,7 @@ jobs:
       basic-checks: ${{ steps.selective-checks.outputs.basic-checks }}
       needs-basic-checks: ${{ steps.selective-checks.outputs.needs-basic-checks }}
       needs-build: ${{ steps.selective-checks.outputs.needs-build }}
+      needs-compile: ${{ steps.selective-checks.outputs.needs-compile }}
       needs-compose-tests: ${{ steps.selective-checks.outputs.needs-compose-tests }}
       needs-dependency-check: ${{ steps.selective-checks.outputs.needs-dependency-check }}
       needs-integration-tests: ${{ steps.selective-checks.outputs.needs-integration-tests }}
@@ -63,7 +64,7 @@ jobs:
             # Run all checks
             dev-support/ci/selective_ci_checks.sh
           fi
-  compile:
+  build:
     needs:
       - build-info
     runs-on: ubuntu-18.04
@@ -71,7 +72,7 @@ jobs:
     if: needs.build-info.outputs.needs-build == 'true'
     strategy:
       matrix:
-        java: [ 8, 11 ]
+        java: [ 8 ]
       fail-fast: false
     steps:
       - name: Checkout project
@@ -101,7 +102,6 @@ jobs:
         run: hadoop-ozone/dev-support/checks/build.sh -Pcoverage -Pdist
       - name: Store binaries for tests
         uses: actions/upload-artifact@v2
-        if: matrix.java == '8'
         with:
           name: ozone-bin
           path: hadoop-ozone/dist/target/ozone*.tar.gz
@@ -112,6 +112,39 @@ jobs:
           rm -rf ~/.m2/repository/org/apache/ozone/hdds*
           rm -rf ~/.m2/repository/org/apache/ozone/ozone*
         if: always()
+  compile:
+    needs:
+      - build-info
+    runs-on: ubuntu-18.04
+    timeout-minutes: 30
+    if: needs.build-info.outputs.needs-compile == 'true'
+    strategy:
+      matrix:
+        java: [ 11 ]
+      fail-fast: false
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v2
+      - name: Cache for maven dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/.m2/repository
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ matrix.java }}
+          restore-keys: |
+            maven-repo-${{ hashFiles('**/pom.xml') }}
+            maven-repo-
+      - name: Setup java
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+      - name: Compile Ozone using Java ${{ matrix.java }}
+        run: hadoop-ozone/dev-support/checks/build.sh -Dskip.npx -Dskip.installnpx
+      - name: Delete temporary build artifacts before caching
+        run: |
+          #Never cache local artifacts
+          rm -rf ~/.m2/repository/org/apache/ozone/hdds*
+          rm -rf ~/.m2/repository/org/apache/ozone/ozone*
+        if: always()
   basic:
     needs:
       - build-info
@@ -162,7 +195,7 @@ jobs:
   dependency:
     needs:
       - build-info
-      - compile
+      - build
     runs-on: ubuntu-18.04
     timeout-minutes: 5
     if: needs.build-info.outputs.needs-dependency-check == 'true'
@@ -191,7 +224,7 @@ jobs:
   acceptance:
     needs:
       - build-info
-      - compile
+      - build
     runs-on: ubuntu-18.04
     timeout-minutes: 150
     if: needs.build-info.outputs.needs-compose-tests == 'true'
@@ -237,7 +270,7 @@ jobs:
   kubernetes:
     needs:
       - build-info
-      - compile
+      - build
     runs-on: ubuntu-18.04
     timeout-minutes: 60
     if: needs.build-info.outputs.needs-kubernetes-tests == 'true'
diff --git a/dev-support/ci/selective_ci_checks.bats b/dev-support/ci/selective_ci_checks.bats
index 566430ac32..42e59d7bb3 100644
--- a/dev-support/ci/selective_ci_checks.bats
+++ b/dev-support/ci/selective_ci_checks.bats
@@ -38,6 +38,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","bats","checkstyle"]'
   assert_output -p needs-build=false
+  assert_output -p needs-compile=false
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=false
@@ -49,6 +50,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat"]'
   assert_output -p needs-build=true
+  assert_output -p needs-compile=false
   assert_output -p needs-compose-tests=true
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=false
@@ -60,6 +62,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","bats"]'
   assert_output -p needs-build=true
+  assert_output -p needs-compile=false
   assert_output -p needs-compose-tests=true
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=false
@@ -71,6 +74,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat"]'
   assert_output -p needs-build=true
+  assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
   assert_output -p needs-dependency-check=true
   assert_output -p needs-integration-tests=false
@@ -82,6 +86,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","bats"]'
   assert_output -p needs-build=true
+  assert_output -p needs-compile=false
   assert_output -p needs-compose-tests=true
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=false
@@ -93,6 +98,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
   assert_output -p needs-build=false
+  assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=true
@@ -104,6 +110,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","bats","unit"]'
   assert_output -p needs-build=false
+  assert_output -p needs-compile=false
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=true
@@ -115,6 +122,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
   assert_output -p needs-build=false
+  assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=false
@@ -126,6 +134,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
   assert_output -p needs-build=false
+  assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=true
@@ -137,6 +146,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs"]'
   assert_output -p needs-build=false
+  assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=true
@@ -148,6 +158,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","bats"]'
   assert_output -p needs-build=true
+  assert_output -p needs-compile=false
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=false
@@ -159,6 +170,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","docs"]'
   assert_output -p needs-build=false
+  assert_output -p needs-compile=false
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=false
@@ -170,6 +182,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
   assert_output -p needs-build=true
+  assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=true
@@ -181,6 +194,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
   assert_output -p needs-build=true
+  assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=true
@@ -192,6 +206,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","author","checkstyle","findbugs","unit"]'
   assert_output -p needs-build=true
+  assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=true
@@ -203,6 +218,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","author","checkstyle","docs","findbugs","unit"]'
   assert_output -p needs-build=true
+  assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=true
@@ -214,6 +230,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat","checkstyle","findbugs","unit"]'
   assert_output -p needs-build=true
+  assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
   assert_output -p needs-dependency-check=true
   assert_output -p needs-integration-tests=true
@@ -225,6 +242,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","rat","unit"]'
   assert_output -p needs-build=true
+  assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
   assert_output -p needs-dependency-check=true
   assert_output -p needs-integration-tests=true
@@ -236,6 +254,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["author","bats","checkstyle","docs","findbugs","rat","unit"]'
   assert_output -p needs-build=true
+  assert_output -p needs-compile=true
   assert_output -p needs-compose-tests=true
   assert_output -p needs-dependency-check=true
   assert_output -p needs-integration-tests=true
@@ -247,6 +266,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=[]'
   assert_output -p needs-build=false
+  assert_output -p needs-compile=false
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=false
@@ -258,6 +278,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=[]'
   assert_output -p needs-build=false
+  assert_output -p needs-compile=false
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=false
@@ -269,6 +290,7 @@ load bats-assert/load.bash
 
   assert_output -p 'basic-checks=["rat"]'
   assert_output -p needs-build=false
+  assert_output -p needs-compile=false
   assert_output -p needs-compose-tests=false
   assert_output -p needs-dependency-check=false
   assert_output -p needs-integration-tests=false
diff --git a/dev-support/ci/selective_ci_checks.sh b/dev-support/ci/selective_ci_checks.sh
index d7b4046fef..1347fcadeb 100755
--- a/dev-support/ci/selective_ci_checks.sh
+++ b/dev-support/ci/selective_ci_checks.sh
@@ -83,6 +83,7 @@ function get_changed_files() {
 
 function set_outputs_run_everything_and_exit() {
     BASIC_CHECKS="author bats checkstyle docs findbugs rat unit"
+    compile_needed=true
     compose_tests_needed=true
     dependency_check_needed=true
     integration_tests_needed=true
@@ -311,6 +312,23 @@ function check_needs_build() {
     start_end::group_end
 }
 
+function check_needs_compile() {
+    start_end::group_start "Check if compile is needed"
+    local pattern_array=(
+        "^hadoop-ozone/dev-support/checks/build.sh"
+        "src/..../java"
+        "src/..../proto"
+        "pom.xml"
+    )
+    filter_changed_files
+
+    if [[ ${match_count} != "0" ]]; then
+        compile_needed=true
+    fi
+
+    start_end::group_end
+}
+
 function check_needs_author() {
     start_end::group_start "Check if author is needed"
     local pattern_array=(
@@ -508,6 +526,7 @@ function set_outputs() {
     fi
 
     initialization::ga_output needs-build "${build_needed:-false}"
+    initialization::ga_output needs-compile "${compile_needed:-false}"
     initialization::ga_output needs-compose-tests "${compose_tests_needed}"
     initialization::ga_output needs-dependency-check "${dependency_check_needed}"
     initialization::ga_output needs-integration-tests "${integration_tests_needed}"
@@ -546,6 +565,7 @@ get_count_robot_files
 get_count_misc_files
 
 check_needs_build
+check_needs_compile
 
 # calculate basic checks to run
 BASIC_CHECKS="rat"


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org