[iceberg] branch master updated: Build: Create separate Spark test actions for 2.4 and 3.0 (#3298)

[bl...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/master by this push:
     new e6498f7  Build: Create separate Spark test actions for 2.4 and 3.0 (#3298)
e6498f7 is described below

commit e6498f78b5a1cede7e0ae9b8261a8952094c2acf
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Fri Oct 15 18:36:41 2021 -0700

    Build: Create separate Spark test actions for 2.4 and 3.0 (#3298)
    
    * Build: Create separate Spark test actions for 2.4 and 3.0.
    
    * Build: Run tests for all Spark modules.
    
    * Build: Fix matrix.
    
    * Build: Update to run check on each module.
    
    * Build: Minor fix.
---
 .github/workflows/java-ci.yml | 56 ++++++++++++++++++++++++++++++++++++++++---
 settings.gradle               |  3 ++-
 2 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/java-ci.yml b/.github/workflows/java-ci.yml
index e54471c..7ab028f 100644
--- a/.github/workflows/java-ci.yml
+++ b/.github/workflows/java-ci.yml
@@ -33,7 +33,7 @@ on:
     - 'site/**'
 
 jobs:
-  run-tests:
+  core-tests:
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -51,7 +51,7 @@ jobs:
         key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
         restore-keys: ${{ runner.os }}-gradle
     - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew check -Pquick=true -x javadoc
+    - run: ./gradlew check -DsparkVersions= -Pquick=true -x javadoc 
     - uses: actions/upload-artifact@v2
       if: failure()
       with:
@@ -59,7 +59,57 @@ jobs:
         path: |
           **/build/testlogs
 
-  extra-checks:
+  spark2-tests:
+    runs-on: ubuntu-latest
+    env:
+      SPARK_LOCAL_IP: localhost
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-java@v1
+      with:
+        java-version: 8
+    - uses: actions/cache@v2
+      with:
+        path: ~/.gradle/caches
+        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+        restore-keys: ${{ runner.os }}-gradle
+    - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+    - run: ./gradlew -DsparkVersions=2.4 :iceberg-spark:check :iceberg-spark:iceberg-spark2:check :iceberg-spark:iceberg-spark-runtime:check -Pquick=true -x javadoc
+    - uses: actions/upload-artifact@v2
+      if: failure()
+      with:
+        name: test logs
+        path: |
+          **/build/testlogs
+
+  spark3-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        jvm: [8, 11]
+        spark: ['3.0']
+    env:
+      SPARK_LOCAL_IP: localhost
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-java@v1
+      with:
+        java-version: ${{ matrix.jvm }}
+    - uses: actions/cache@v2
+      with:
+        path: ~/.gradle/caches
+        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
+        restore-keys: ${{ runner.os }}-gradle
+    - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+    - run: ./gradlew -DsparkVersions=${{ matrix.spark }} :iceberg-spark:check :iceberg-spark:iceberg-spark3:check :iceberg-spark:iceberg-spark3-extensions:check :iceberg-spark:iceberg-spark3-runtime:check -Pquick=true -x javadoc
+    - uses: actions/upload-artifact@v2
+      if: failure()
+      with:
+        name: test logs
+        path: |
+          **/build/testlogs
+
+  build-checks:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
diff --git a/settings.gradle b/settings.gradle
index 9ccf71f..931beed 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -57,7 +57,8 @@ project(':hive-metastore').name = 'iceberg-hive-metastore'
 project(':nessie').name = 'iceberg-nessie'
 
 List<String> knownSparkVersions = System.getProperty("knownSparkVersions").split(",")
-List<String> sparkVersions = (System.getProperty("sparkVersions") != null ? System.getProperty("sparkVersions") : System.getProperty("defaultSparkVersions")).split(",")
+String sparkVersionsString = System.getProperty("sparkVersions") != null ? System.getProperty("sparkVersions") : System.getProperty("defaultSparkVersions")
+List<String> sparkVersions = sparkVersionsString != null && !sparkVersionsString.isEmpty() ? sparkVersionsString.split(",") : []
 
 if (!knownSparkVersions.containsAll(sparkVersions)) {
   throw new GradleException("Found unsupported Spark versions: " + (sparkVersions - knownSparkVersions))