You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ho...@apache.org on 2025/04/10 15:47:10 UTC

(incubator-gluten) branch main updated: [CORE] Refine the test APIs for specified spark versions (#9274)

This is an automated email from the ASF dual-hosted git repository.

hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 118b114996 [CORE] Refine the test APIs for specified spark versions (#9274)
118b114996 is described below

commit 118b1149960e2f38c02fff5901704fc54a72465b
Author: Kaifei Yi <yi...@apache.org>
AuthorDate: Thu Apr 10 23:47:06 2025 +0800

    [CORE] Refine the test APIs for specified spark versions (#9274)
---
 ...ClickHouseIcebergMOREqualityDeletionSuite.scala |  2 +-
 .../execution/iceberg/ClickHouseIcebergSuite.scala | 17 ++++---
 .../GlutenClickHouseExcelFormatSuite.scala         |  4 +-
 .../execution/GlutenSQLCollectLimitExecSuite.scala | 20 +++-----
 .../execution/JsonFunctionsValidateSuite.scala     | 22 ++++-----
 .../gluten/execution/MiscOperatorSuite.scala       |  2 +-
 .../execution/ScalarFunctionsValidateSuite.scala   | 26 +++++------
 .../execution/VeloxAggregateFunctionsSuite.scala   |  8 ++--
 .../gluten/execution/VeloxColumnarCacheSuite.scala |  4 +-
 .../gluten/execution/VeloxHashJoinSuite.scala      |  2 +-
 .../execution/VeloxStringFunctionsSuite.scala      |  2 +-
 .../sql/execution/VeloxParquetReadSuite.scala      |  2 +-
 .../execution/VeloxParquetWriteForHiveSuite.scala  |  4 +-
 .../org/apache/gluten/execution/DeltaSuite.scala   | 20 ++++----
 .../org/apache/gluten/execution/HudiSuite.scala    |  8 ++--
 .../org/apache/gluten/execution/IcebergSuite.scala | 18 ++++----
 .../org/apache/spark/sql/GlutenQueryTest.scala     | 54 +++++++++++++++++-----
 .../expressions/GlutenExpressionMappingSuite.scala |  8 ++--
 18 files changed, 120 insertions(+), 103 deletions(-)

diff --git a/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergMOREqualityDeletionSuite.scala b/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergMOREqualityDeletionSuite.scala
index 52b69df542..fae32bf447 100644
--- a/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergMOREqualityDeletionSuite.scala
+++ b/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergMOREqualityDeletionSuite.scala
@@ -59,7 +59,7 @@ class ClickHouseIcebergMOREqualityDeletionSuite extends GlutenClickHouseWholeSta
   }
 
   testWithSpecifiedSparkVersion(
-    "iceberg read mor table with equality deletion", Array("3.3", "3.5")) {
+    "iceberg read mor table with equality deletion", "3.3", "3.5") {
     // The table 'test_upsert_query' was generated by Flink + Iceberg from the iceberg ut,
     // the root path must be the '/tmp/junit6640909127060857423/default'
     val testTableName = "local.db.test_upsert_query"
diff --git a/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala b/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala
index 3e44047b74..dafb5dada8 100644
--- a/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala
+++ b/backends-clickhouse/src-iceberg/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergSuite.scala
@@ -57,7 +57,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite
   }
 
   testWithSpecifiedSparkVersion(
-    "iceberg bucketed join", Array("3.3", "3.5")) {
+    "iceberg bucketed join", "3.3", "3.5") {
     val leftTable = "p_str_tb"
     val rightTable = "p_int_tb"
     withTable(leftTable, rightTable) {
@@ -140,7 +140,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite
   }
 
   testWithSpecifiedSparkVersion(
-    "iceberg bucketed join with partition", Array("3.3", "3.5")) {
+    "iceberg bucketed join with partition", "3.3", "3.5") {
     val leftTable = "p_str_tb"
     val rightTable = "p_int_tb"
     withTable(leftTable, rightTable) {
@@ -222,8 +222,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite
     }
   }
 
-  testWithSpecifiedSparkVersion("iceberg bucketed join partition value not exists",
-    Array("3.4", "3.5")) {
+  testWithMinSparkVersion("iceberg bucketed join partition value not exists", "3.5") {
     val leftTable = "p_str_tb"
     val rightTable = "p_int_tb"
     withTable(leftTable, rightTable) {
@@ -302,8 +301,8 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite
     }
   }
 
-  testWithSpecifiedSparkVersion(
-    "iceberg bucketed join partition value not exists partial cluster", Array("3.4", "3.5")) {
+  testWithMinSparkVersion(
+    "iceberg bucketed join partition value not exists partial cluster", "3.4") {
     val leftTable = "p_str_tb"
     val rightTable = "p_int_tb"
     withTable(leftTable, rightTable) {
@@ -383,7 +382,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite
   }
 
   testWithSpecifiedSparkVersion(
-    "iceberg bucketed join with partition filter", Array("3.3", "3.5")) {
+    "iceberg bucketed join with partition filter", "3.3", "3.5") {
     val leftTable = "p_str_tb"
     val rightTable = "p_int_tb"
     withTable(leftTable, rightTable) {
@@ -466,7 +465,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite
     }
   }
 
-  testWithSpecifiedSparkVersion("iceberg: time travel") {
+  test("iceberg: time travel") {
     withTable("iceberg_tm") {
       spark.sql(
         s"""
@@ -765,7 +764,7 @@ class ClickHouseIcebergSuite extends GlutenClickHouseWholeStageTransformerSuite
   // Spark configuration spark.sql.iceberg.handle-timestamp-without-timezone is not supported
   // in Spark 3.4
   // TODO: there is a bug when using timestamp type as the partition column
-  testWithSpecifiedSparkVersion("iceberg partition type - timestamp", Array("")) {
+  ignore("iceberg partition type - timestamp") {
     Seq("true", "false").foreach {
       flag =>
         withSQLConf(
diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseExcelFormatSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseExcelFormatSuite.scala
index cb7bcb430d..eb9e13e451 100644
--- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseExcelFormatSuite.scala
+++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickHouseExcelFormatSuite.scala
@@ -1485,9 +1485,7 @@ class GlutenClickHouseExcelFormatSuite
   }
 
   // TODO: pass spark configuration to FileFormatWriter in Spark 3.3 and 3.2
-  testWithSpecifiedSparkVersion(
-    "write succeed even if set wrong snappy compression codec level",
-    Some("3.5")) {
+  testWithMinSparkVersion("write succeed even if set wrong snappy compression codec level", "3.5") {
     // TODO: remove duplicated test codes
     val tablePath = s"$HDFS_URL_ENDPOINT/$SPARK_DIR_NAME/failed_test/"
     val format = "parquet"
diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/GlutenSQLCollectLimitExecSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/GlutenSQLCollectLimitExecSuite.scala
index 8af7b544e2..7ac89c33de 100644
--- a/backends-velox/src/test/scala/org/apache/gluten/execution/GlutenSQLCollectLimitExecSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/gluten/execution/GlutenSQLCollectLimitExecSuite.scala
@@ -56,9 +56,7 @@ class GlutenSQLCollectLimitExecSuite extends WholeStageTransformerSuite {
     assert(assertionCondition, assertionMessage)
   }
 
-  testWithSpecifiedSparkVersion(
-    "ColumnarCollectLimitExec - basic limit test",
-    Array("3.2", "3.3")) {
+  testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - basic limit test", "3.2", "3.3") {
     val df = spark.range(0, 1000, 1).toDF("id").limit(5)
     val expectedData = Seq(Row(0L), Row(1L), Row(2L), Row(3L), Row(4L))
 
@@ -67,7 +65,7 @@ class GlutenSQLCollectLimitExecSuite extends WholeStageTransformerSuite {
     assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](df, checkMatch = true)
   }
 
-  testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - with filter", Array("3.2", "3.3")) {
+  testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - with filter", "3.2", "3.3") {
     val df = spark
       .range(0, 20, 1)
       .toDF("id")
@@ -80,9 +78,7 @@ class GlutenSQLCollectLimitExecSuite extends WholeStageTransformerSuite {
     assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](df, checkMatch = true)
   }
 
-  testWithSpecifiedSparkVersion(
-    "ColumnarCollectLimitExec - range with repartition",
-    Array("3.2", "3.3")) {
+  testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - range with repartition", "3.2", "3.3") {
 
     val df = spark
       .range(0, 10, 1)
@@ -96,9 +92,7 @@ class GlutenSQLCollectLimitExecSuite extends WholeStageTransformerSuite {
     assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](df, checkMatch = true)
   }
 
-  testWithSpecifiedSparkVersion(
-    "ColumnarCollectLimitExec - with distinct values",
-    Array("3.2", "3.3")) {
+  testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - with distinct values", "3.2", "3.3") {
     val df = spark
       .range(0, 10, 1)
       .toDF("id")
@@ -112,7 +106,7 @@ class GlutenSQLCollectLimitExecSuite extends WholeStageTransformerSuite {
     assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](df, checkMatch = true)
   }
 
-  testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - chained limit", Array("3.2", "3.3")) {
+  testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - chained limit", "3.2", "3.3") {
     val df = spark
       .range(0, 10, 1)
       .toDF("id")
@@ -125,9 +119,7 @@ class GlutenSQLCollectLimitExecSuite extends WholeStageTransformerSuite {
     assertGlutenOperatorMatch[ColumnarCollectLimitBaseExec](df, checkMatch = true)
   }
 
-  testWithSpecifiedSparkVersion(
-    "ColumnarCollectLimitExec - limit after union",
-    Array("3.2", "3.3")) {
+  testWithSpecifiedSparkVersion("ColumnarCollectLimitExec - limit after union", "3.2", "3.3") {
     val df1 = spark.range(0, 5).toDF("id")
     val df2 = spark.range(5, 10).toDF("id")
     val unionDf = df1.union(df2).limit(3)
diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/JsonFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/JsonFunctionsValidateSuite.scala
index 3d7c255408..260fe58616 100644
--- a/backends-velox/src/test/scala/org/apache/gluten/execution/JsonFunctionsValidateSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/gluten/execution/JsonFunctionsValidateSuite.scala
@@ -79,7 +79,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("from_json function bool", Some("3.4")) {
+  testWithMinSparkVersion("from_json function bool", "3.4") {
     withTempPath {
       path =>
         Seq[(String)](
@@ -101,7 +101,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("from_json function small int", Some("3.4")) {
+  testWithMinSparkVersion("from_json function small int", "3.4") {
     withTempPath {
       path =>
         Seq[(String)](
@@ -123,7 +123,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("from_json function int", Some("3.4")) {
+  testWithMinSparkVersion("from_json function int", "3.4") {
     withTempPath {
       path =>
         Seq[(String)](
@@ -145,7 +145,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("from_json function big int", Some("3.4")) {
+  testWithMinSparkVersion("from_json function big int", "3.4") {
     withTempPath {
       path =>
         Seq[(String)](
@@ -167,7 +167,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("from_json function float", Some("3.4")) {
+  testWithMinSparkVersion("from_json function float", "3.4") {
     withTempPath {
       path =>
         Seq[(String)](
@@ -191,7 +191,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("from_json function double", Some("3.4")) {
+  testWithMinSparkVersion("from_json function double", "3.4") {
     withTempPath {
       path =>
         Seq[(String)](
@@ -215,7 +215,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("from_json function string", Some("3.4")) {
+  testWithMinSparkVersion("from_json function string", "3.4") {
     withTempPath {
       path =>
         Seq[(String)](
@@ -237,7 +237,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("from_json function array", Some("3.4")) {
+  testWithMinSparkVersion("from_json function array", "3.4") {
     withTempPath {
       path =>
         Seq[(String)](
@@ -257,7 +257,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("from_json function map", Some("3.4")) {
+  testWithMinSparkVersion("from_json function map", "3.4") {
     withTempPath {
       path =>
         Seq[(String)](
@@ -279,7 +279,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("from_json function row", Some("3.4")) {
+  testWithMinSparkVersion("from_json function row", "3.4") {
     withTempPath {
       path =>
         Seq[(String)](
@@ -319,7 +319,7 @@ class JsonFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("from_json function duplicate key", Some("3.4")) {
+  testWithMinSparkVersion("from_json function duplicate key", "3.4") {
     withTempPath {
       path =>
         Seq[(String)](
diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
index 43ae1cee0b..b7d7abe882 100644
--- a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
@@ -312,7 +312,7 @@ class MiscOperatorSuite extends VeloxWholeStageTransformerSuite with AdaptiveSpa
     checkLengthAndPlan(df, 5)
   }
 
-  testWithSpecifiedSparkVersion("coalesce validation", Some("3.4")) {
+  testWithMinSparkVersion("coalesce validation", "3.4") {
     withTempPath {
       path =>
         val data = "2019-09-09 01:02:03.456789"
diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
index c5cbfe7b83..8b1ae56893 100644
--- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala
@@ -123,7 +123,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("Test array_append function - INT", Some("3.4")) {
+  testWithMinSparkVersion("Test array_append function - INT", "3.4") {
     withTempPath {
       path =>
         Seq[(Array[Int], Int)](
@@ -148,7 +148,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("Test array_append function - STRING", Some("3.4")) {
+  testWithMinSparkVersion("Test array_append function - STRING", "3.4") {
     withTempPath {
       path =>
         Seq[(Array[String], String)](
@@ -186,7 +186,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("null input for array_size", Some("3.3")) {
+  testWithMinSparkVersion("null input for array_size", "3.3") {
     withTempPath {
       path =>
         Seq[(Array[Int])](
@@ -841,7 +841,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("Test width_bucket function", Some("3.4")) {
+  testWithMinSparkVersion("Test width_bucket function", "3.4") {
     withTempPath {
       path =>
         Seq[(Integer, Integer, Integer, Integer)](
@@ -859,7 +859,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("Test url_decode function", Some("3.4")) {
+  testWithMinSparkVersion("Test url_decode function", "3.4") {
     withTempPath {
       path =>
         Seq("https%3A%2F%2Fspark.apache.org")
@@ -874,7 +874,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("Test url_encode function", Some("3.4")) {
+  testWithMinSparkVersion("Test url_encode function", "3.4") {
     withTempPath {
       path =>
         Seq("https://spark.apache.org")
@@ -1005,7 +1005,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("mask", Some("3.4")) {
+  testWithMinSparkVersion("mask", "3.4") {
     runQueryAndCompare("SELECT mask(c_comment) FROM customer limit 50") {
       checkGlutenOperatorMatch[ProjectExecTransformer]
     }
@@ -1119,7 +1119,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("try_subtract", Some("3.3")) {
+  testWithMinSparkVersion("try_subtract", "3.3") {
     runQueryAndCompare(
       "select try_subtract(2147483647, cast(l_orderkey as int)), " +
         "try_subtract(-2147483648, cast(l_orderkey as int)) from lineitem") {
@@ -1135,7 +1135,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("try_multiply", Some("3.3")) {
+  testWithMinSparkVersion("try_multiply", "3.3") {
     runQueryAndCompare(
       "select try_multiply(2147483647, cast(l_orderkey as int)), " +
         "try_multiply(-2147483648, cast(l_orderkey as int)) from lineitem") {
@@ -1301,7 +1301,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("get", Some("3.4")) {
+  testWithMinSparkVersion("get", "3.4") {
     withTempPath {
       path =>
         Seq[Seq[Integer]](Seq(1, null, 5, 4), Seq(5, -1, 8, 9, -7, 2), Seq.empty, null)
@@ -1392,7 +1392,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("levenshtein with limit", Some("3.5")) {
+  testWithMinSparkVersion("levenshtein with limit", "3.5") {
     runQueryAndCompare("select levenshtein(c_comment, c_address, 3) from customer limit 50") {
       checkGlutenOperatorMatch[ProjectExecTransformer]
     }
@@ -1506,7 +1506,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("array insert", Some("3.4")) {
+  testWithMinSparkVersion("array insert", "3.4") {
     withTempPath {
       path =>
         Seq[Seq[Integer]](Seq(1, null, 5, 4), Seq(5, -1, 8, 9, -7, 2), Seq.empty, null)
@@ -1560,7 +1560,7 @@ abstract class ScalarFunctionsValidateSuite extends FunctionsValidateSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("Test try_cast", Some("3.4")) {
+  testWithMinSparkVersion("Test try_cast", "3.4") {
     withTempView("try_cast_table") {
       withTempPath {
         path =>
diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
index b3094ef235..c1700d487b 100644
--- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxAggregateFunctionsSuite.scala
@@ -390,7 +390,7 @@ abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSu
     }
   }
 
-  testWithSpecifiedSparkVersion("regr_r2", Some("3.3")) {
+  testWithMinSparkVersion("regr_r2", "3.3") {
     runQueryAndCompare("""
                          |select regr_r2(l_partkey, l_suppkey) from lineitem;
                          |""".stripMargin) {
@@ -409,7 +409,7 @@ abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSu
     }
   }
 
-  testWithSpecifiedSparkVersion("regr_slope", Some("3.4")) {
+  testWithMinSparkVersion("regr_slope", "3.4") {
     runQueryAndCompare("""
                          |select regr_slope(l_partkey, l_suppkey) from lineitem;
                          |""".stripMargin) {
@@ -428,7 +428,7 @@ abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSu
     }
   }
 
-  testWithSpecifiedSparkVersion("regr_intercept", Some("3.4")) {
+  testWithMinSparkVersion("regr_intercept", "3.4") {
     runQueryAndCompare("""
                          |select regr_intercept(l_partkey, l_suppkey) from lineitem;
                          |""".stripMargin) {
@@ -447,7 +447,7 @@ abstract class VeloxAggregateFunctionsSuite extends VeloxWholeStageTransformerSu
     }
   }
 
-  testWithSpecifiedSparkVersion("regr_sxy regr_sxx regr_syy", Some("3.4")) {
+  testWithMinSparkVersion("regr_sxy regr_sxx regr_syy", "3.4") {
     runQueryAndCompare("""
                          |select regr_sxy(l_quantity, l_tax) from lineitem;
                          |""".stripMargin) {
diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxColumnarCacheSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxColumnarCacheSuite.scala
index f595420793..e4f59051bd 100644
--- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxColumnarCacheSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxColumnarCacheSuite.scala
@@ -79,7 +79,7 @@ class VeloxColumnarCacheSuite extends VeloxWholeStageTransformerSuite with Adapt
     }
   }
 
-  testWithSpecifiedSparkVersion("input row", Some("3.2")) {
+  testWithMinSparkVersion("input row", "3.2") {
     withTable("t") {
       sql("CREATE TABLE t USING json AS SELECT * FROM values(1, 'a', (2, 'b'), (3, 'c'))")
       runQueryAndCompare("SELECT * FROM t", cache = true) {
@@ -102,7 +102,7 @@ class VeloxColumnarCacheSuite extends VeloxWholeStageTransformerSuite with Adapt
   }
 
   // See issue https://github.com/apache/incubator-gluten/issues/8497.
-  testWithSpecifiedSparkVersion("Input fallen back vanilla Spark columnar scan", Some("3.3")) {
+  testWithMinSparkVersion("Input fallen back vanilla Spark columnar scan", "3.3") {
     def withId(id: Int): Metadata =
       new MetadataBuilder().putLong("parquet.field.id", id).build()
 
diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxHashJoinSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxHashJoinSuite.scala
index 9ec2ec33a6..3388e7fc65 100644
--- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxHashJoinSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxHashJoinSuite.scala
@@ -73,7 +73,7 @@ class VeloxHashJoinSuite extends VeloxWholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("generate hash join plan - v2", Some("3.2")) {
+  testWithMinSparkVersion("generate hash join plan - v2", "3.2") {
     withSQLConf(
       ("spark.sql.autoBroadcastJoinThreshold", "-1"),
       ("spark.sql.adaptive.enabled", "false"),
diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxStringFunctionsSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxStringFunctionsSuite.scala
index ff2967b1c3..3cb8fa9173 100644
--- a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxStringFunctionsSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxStringFunctionsSuite.scala
@@ -532,7 +532,7 @@ class VeloxStringFunctionsSuite extends VeloxWholeStageTransformerSuite {
         s"from $LINEITEM_TABLE limit 5") { _ => }
   }
 
-  testWithSpecifiedSparkVersion("split", Some("3.4")) {
+  testWithMinSparkVersion("split", "3.4") {
     runQueryAndCompare(
       s"select l_orderkey, l_comment, split(l_comment, '') " +
         s"from $LINEITEM_TABLE limit 5") {
diff --git a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetReadSuite.scala b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetReadSuite.scala
index 552b173fbb..7cd2ba0880 100644
--- a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetReadSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetReadSuite.scala
@@ -32,7 +32,7 @@ class VeloxParquetReadSuite extends VeloxWholeStageTransformerSuite {
       .set(VeloxConfig.LOAD_QUANTUM.key, "256m")
   }
 
-  testWithSpecifiedSparkVersion("read example parquet files", Some("3.5"), Some("3.5")) {
+  testWithMinSparkVersion("read example parquet files", "3.5") {
     withTable("test_table") {
       val dir = new File(getClass.getResource(resourcePath).getFile)
       val files = dir.listFiles
diff --git a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala
index 44e36df09a..1315a3ef92 100644
--- a/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala
+++ b/backends-velox/src/test/scala/org/apache/spark/sql/execution/VeloxParquetWriteForHiveSuite.scala
@@ -331,9 +331,9 @@ class VeloxParquetWriteForHiveSuite
     }
   }
 
-  testWithSpecifiedSparkVersion(
+  testWithMaxSparkVersion(
     "Native writer should keep the same compression codec if `hive.exec.compress.output` is true",
-    maxSparkVersion = Some("3.3")) {
+    "3.3") {
     Seq(false, true).foreach {
       enableNativeWrite =>
         withSQLConf("spark.gluten.sql.native.writer.enabled" -> enableNativeWrite.toString) {
diff --git a/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala b/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
index 9046ba4140..fb135023eb 100644
--- a/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
+++ b/gluten-delta/src/test/scala/org/apache/gluten/execution/DeltaSuite.scala
@@ -43,7 +43,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
   }
 
   // IdMapping is supported in Delta 2.2 (related to Spark3.3.1)
-  testWithSpecifiedSparkVersion("column mapping mode = id", Some("3.3")) {
+  testWithMinSparkVersion("column mapping mode = id", "3.3") {
     withTable("delta_cm1") {
       spark.sql(s"""
                    |create table delta_cm1 (id int, name string) using delta
@@ -63,7 +63,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
   }
 
   // NameMapping is supported in Delta 2.0 (related to Spark3.2.0)
-  testWithSpecifiedSparkVersion("column mapping mode = name", Some("3.2")) {
+  testWithMinSparkVersion("column mapping mode = name", "3.2") {
     withTable("delta_cm2") {
       spark.sql(s"""
                    |create table delta_cm2 (id int, name string) using delta
@@ -82,7 +82,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("delta: time travel", Some("3.3")) {
+  testWithMinSparkVersion("delta: time travel", "3.3") {
     withTable("delta_tm") {
       spark.sql(s"""
                    |create table delta_tm (id int, name string) using delta
@@ -107,7 +107,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("delta: partition filters", Some("3.2")) {
+  testWithMinSparkVersion("delta: partition filters", "3.2") {
     withTable("delta_pf") {
       spark.sql(s"""
                    |create table delta_pf (id int, name string) using delta partitioned by (name)
@@ -126,7 +126,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("basic test with stats.skipping disabled", Some("3.2")) {
+  testWithMinSparkVersion("basic test with stats.skipping disabled", "3.2") {
     withTable("delta_test2") {
       withSQLConf("spark.databricks.delta.stats.skipping" -> "false") {
         spark.sql(s"""
@@ -146,7 +146,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("column mapping with complex type", Some("3.2")) {
+  testWithMinSparkVersion("column mapping with complex type", "3.2") {
     withTable("t1") {
       val simpleNestedSchema = new StructType()
         .add("a", StringType, true)
@@ -196,7 +196,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("deletion vector", Some("3.4")) {
+  testWithMinSparkVersion("deletion vector", "3.4") {
     withTempPath {
       p =>
         import testImplicits._
@@ -213,7 +213,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("delta: push down input_file_name expression", Some("3.2")) {
+  testWithMinSparkVersion("delta: push down input_file_name expression", "3.2") {
     withTable("source_table") {
       withTable("target_table") {
         spark.sql(s"""
@@ -251,9 +251,7 @@ abstract class DeltaSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion(
-    "delta: need to validate delta expression before execution",
-    Some("3.2")) {
+  testWithMinSparkVersion("delta: need to validate delta expression before execution", "3.2") {
     withTable("source_table") {
       withTable("target_table") {
         spark.sql(s"""
diff --git a/gluten-hudi/src/test/scala/org/apache/gluten/execution/HudiSuite.scala b/gluten-hudi/src/test/scala/org/apache/gluten/execution/HudiSuite.scala
index 97633fa064..471622f445 100644
--- a/gluten-hudi/src/test/scala/org/apache/gluten/execution/HudiSuite.scala
+++ b/gluten-hudi/src/test/scala/org/apache/gluten/execution/HudiSuite.scala
@@ -38,7 +38,7 @@ abstract class HudiSuite extends WholeStageTransformerSuite {
       .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
   }
 
-  testWithSpecifiedSparkVersion("hudi: time travel", Some("3.2")) {
+  testWithMinSparkVersion("hudi: time travel", "3.2") {
     withTable("hudi_tm") {
       spark.sql(s"""
                    |create table hudi_tm (id int, name string) using hudi
@@ -65,7 +65,7 @@ abstract class HudiSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("hudi: soft delete", Some("3.2")) {
+  testWithMinSparkVersion("hudi: soft delete", "3.2") {
     withTable("hudi_pf") {
       spark.sql(s"""
                    |create table hudi_pf (id int, name string) using hudi
@@ -85,7 +85,7 @@ abstract class HudiSuite extends WholeStageTransformerSuite {
   }
 
   // FIXME: flaky leaked file systems issue
-  ignore("hudi: mor", Some("3.2")) {
+  ignore("hudi: mor") {
     withTable("hudi_mor") {
       spark.sql(s"""
                    |create table hudi_mor (id int, name string, ts bigint)
@@ -111,7 +111,7 @@ abstract class HudiSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("hudi: partition filters", Some("3.2")) {
+  testWithMinSparkVersion("hudi: partition filters", "3.2") {
     withTable("hudi_pf") {
       spark.sql(s"""
                    |create table hudi_pf (id int, name string) using hudi partitioned by (name)
diff --git a/gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala b/gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala
index 6016001a06..ec071fff73 100644
--- a/gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala
+++ b/gluten-iceberg/src/test/scala/org/apache/gluten/execution/IcebergSuite.scala
@@ -57,7 +57,7 @@ abstract class IcebergSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("iceberg bucketed join", Some("3.4")) {
+  testWithMinSparkVersion("iceberg bucketed join", "3.4") {
     val leftTable = "p_str_tb"
     val rightTable = "p_int_tb"
     withTable(leftTable, rightTable) {
@@ -131,7 +131,7 @@ abstract class IcebergSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("iceberg bucketed join with partition", Some("3.4")) {
+  testWithMinSparkVersion("iceberg bucketed join with partition", "3.4") {
     val leftTable = "p_str_tb"
     val rightTable = "p_int_tb"
     withTable(leftTable, rightTable) {
@@ -205,9 +205,7 @@ abstract class IcebergSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion(
-    "iceberg bucketed join partition value not exists",
-    Array("3.4", "3.5")) {
+  testWithMinSparkVersion("iceberg bucketed join partition value not exists", "3.4") {
     val leftTable = "p_str_tb"
     val rightTable = "p_int_tb"
     withTable(leftTable, rightTable) {
@@ -282,9 +280,9 @@ abstract class IcebergSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion(
+  testWithMinSparkVersion(
     "iceberg bucketed join partition value not exists partial cluster",
-    Array("3.4", "3.5")) {
+    "3.4") {
     val leftTable = "p_str_tb"
     val rightTable = "p_int_tb"
     withTable(leftTable, rightTable) {
@@ -359,7 +357,7 @@ abstract class IcebergSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("iceberg bucketed join with partition filter", Some("3.4")) {
+  testWithMinSparkVersion("iceberg bucketed join with partition filter", "3.4") {
     val leftTable = "p_str_tb"
     val rightTable = "p_int_tb"
     withTable(leftTable, rightTable) {
@@ -435,7 +433,7 @@ abstract class IcebergSuite extends WholeStageTransformerSuite {
     }
   }
 
-  testWithSpecifiedSparkVersion("iceberg: time travel") {
+  test("iceberg: time travel") {
     withTable("iceberg_tm") {
       spark.sql(s"""
                    |create table iceberg_tm (id int, name string) using iceberg
@@ -593,7 +591,7 @@ abstract class IcebergSuite extends WholeStageTransformerSuite {
 
   // Spark configuration spark.sql.iceberg.handle-timestamp-without-timezone is not supported
   // in Spark 3.4
-  testWithSpecifiedSparkVersion("iceberg partition type - timestamp", Array("3.2", "3.3", "3.5")) {
+  testWithSpecifiedSparkVersion("iceberg partition type - timestamp", "3.2", "3.3", "3.5") {
     Seq("true", "false").foreach {
       flag =>
         withSQLConf(
diff --git a/gluten-substrait/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala b/gluten-substrait/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala
index 1d4a63bdbc..21185a3581 100644
--- a/gluten-substrait/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala
+++ b/gluten-substrait/src/test/scala/org/apache/spark/sql/GlutenQueryTest.scala
@@ -60,7 +60,21 @@ abstract class GlutenQueryTest extends PlanTest {
     maxVersion(0) > version(0) || maxVersion(0) == version(0) && maxVersion(1) >= version(1)
   }
 
-  def shouldRun(
+  /**
+   * Check if the current spark version is between the minVersion and maxVersion.
+   *
+   * If the maxVersion is not specified, it will check if the current spark version is greater than
+   * or equal to, and if the minVersion is not specified, it will check if the current spark version
+   * is less than or equal to.
+   *
+   * @param minSparkVersion
+   *   the minimum spark version
+   * @param maxSparkVersion
+   *   the maximum spark version
+   * @return
+   *   true if the current spark version is between the minVersion and maxVersion
+   */
+  def matchSparkVersion(
       minSparkVersion: Option[String] = None,
       maxSparkVersion: Option[String] = None): Boolean = {
     var shouldRun = true
@@ -77,31 +91,49 @@ abstract class GlutenQueryTest extends PlanTest {
     shouldRun
   }
 
-  def ignore(
+  /** Ignore the test if the current spark version is between the minVersion and maxVersion */
+  def ignoreWithSpecifiedSparkVersion(
       testName: String,
       minSparkVersion: Option[String] = None,
       maxSparkVersion: Option[String] = None)(testFun: => Any): Unit = {
-    if (shouldRun(minSparkVersion, maxSparkVersion)) {
+    if (matchSparkVersion(minSparkVersion, maxSparkVersion)) {
       ignore(testName) {
         testFun
       }
     }
   }
 
-  def testWithSpecifiedSparkVersion(
-      testName: String,
-      minSparkVersion: Option[String] = None,
-      maxSparkVersion: Option[String] = None)(testFun: => Any): Unit = {
-    if (shouldRun(minSparkVersion, maxSparkVersion)) {
+  /** Run the test if the current spark version is between the minVersion and maxVersion */
+  def testWithRangeSparkVersion(testName: String, minSparkVersion: String, maxSparkVersion: String)(
+      testFun: => Any): Unit = {
+    if (matchSparkVersion(Some(minSparkVersion), Some(maxSparkVersion))) {
       test(testName) {
         testFun
       }
     }
   }
 
-  def testWithSpecifiedSparkVersion(testName: String, versions: Array[String])(
-      testFun: => Any): Unit = {
-    if (versions.exists(v => shouldRun(Some(v), Some(v)))) {
+  /** Run the test if the current spark version less than the maxVersion */
+  def testWithMaxSparkVersion(testName: String, maxVersion: String)(testFun: => Any): Unit = {
+    if (matchSparkVersion(maxSparkVersion = Some(maxVersion))) {
+      test(testName) {
+        testFun
+      }
+    }
+  }
+
+  /** Run the test if the current spark version greater than the minVersion */
+  def testWithMinSparkVersion(testName: String, minVersion: String)(testFun: => Any): Unit = {
+    if (matchSparkVersion(Some(minVersion))) {
+      test(testName) {
+        testFun
+      }
+    }
+  }
+
+  /** Run the test on the specified spark version */
+  def testWithSpecifiedSparkVersion(testName: String, versions: String*)(testFun: => Any): Unit = {
+    if (versions.exists(v => matchSparkVersion(Some(v), Some(v)))) {
       test(testName) {
         testFun
       }
diff --git a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
index b2fbd11c0a..6841a87d13 100644
--- a/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
+++ b/gluten-ut/test/src/test/scala/org/apache/gluten/expressions/GlutenExpressionMappingSuite.scala
@@ -47,7 +47,7 @@ class GlutenExpressionMappingSuite
     conf
   }
 
-  testWithSpecifiedSparkVersion("test expression blacklist", Some("3.2")) {
+  testWithMinSparkVersion("test expression blacklist", "3.2") {
     val names = ExpressionMappings.expressionsMap.values.toSet
     assert(names.contains("regexp_replace"))
     assert(names.contains("regexp_extract"))
@@ -68,7 +68,7 @@ class GlutenExpressionMappingSuite
     }
   }
 
-  testWithSpecifiedSparkVersion("test blacklisting regexp expressions", Some("3.2")) {
+  testWithMinSparkVersion("test blacklisting regexp expressions", "3.2") {
     val names = ExpressionMappings.expressionsMap.values.toSet
     assert(names.contains("rlike"))
     assert(names.contains("regexp_replace"))
@@ -94,9 +94,9 @@ class GlutenExpressionMappingSuite
     }
   }
 
-  testWithSpecifiedSparkVersion(
+  testWithMinSparkVersion(
     "GLUTEN-7213: Check fallback reason with CheckOverflowInTableInsert",
-    Some("3.4")) {
+    "3.4") {
     withSQLConf(GlutenConfig.RAS_ENABLED.key -> "false") {
       withTable("t1", "t2") {
         sql("create table t1 (a float) using parquet")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org