You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2020/09/04 00:53:47 UTC

[spark] branch branch-3.0 updated: [SPARK-32786][SQL][TEST] Improve performance for some slow DPP tests

This is an automated email from the ASF dual-hosted git repository.

yamamuro pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-3.0 by this push:
     new 7320db2  [SPARK-32786][SQL][TEST] Improve performance for some slow DPP tests
7320db2 is described below

commit 7320db21bda01c162623839350de908042f4ebb8
Author: Zhenhua Wang <wz...@163.com>
AuthorDate: Fri Sep 4 09:33:20 2020 +0900

    [SPARK-32786][SQL][TEST] Improve performance for some slow DPP tests
    
    ### What changes were proposed in this pull request?
    
    The whole `DynamicPartitionPruningSuite` takes about 2 min on my laptop (either AE on or off). The slowest tests are `test("simple inner join triggers DPP with mock-up tables")` and `test("cleanup any DPP filter that isn't pushed down due to expression id clashes")`, which totally take about 1 min.
    
    We can reuse existing test tables or use smaller tables to reduce the cost. After that, the two tests takes only about 1 sec in total, leading to 2x speedup for the suite.
    
    ### Why are the changes needed?
    
    To speedup DPP test suites.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Modified two existing tests.
    
    Closes #29636 from wzhfy/improve_dpp_test.
    
    Authored-by: Zhenhua Wang <wz...@163.com>
    Signed-off-by: Takeshi Yamamuro <ya...@apache.org>
---
 .../spark/sql/DynamicPartitionPruningSuite.scala   | 30 ++++++----------------
 1 file changed, 8 insertions(+), 22 deletions(-)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index 47c5c9ea..8486d1d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -254,29 +254,15 @@ abstract class DynamicPartitionPruningSuiteBase
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
       SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
       SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
-      withTable("df1", "df2") {
-        spark.range(1000)
-          .select(col("id"), col("id").as("k"))
-          .write
-          .partitionBy("k")
-          .format(tableFormat)
-          .mode("overwrite")
-          .saveAsTable("df1")
-
-        spark.range(100)
-          .select(col("id"), col("id").as("k"))
-          .write
-          .partitionBy("k")
-          .format(tableFormat)
-          .mode("overwrite")
-          .saveAsTable("df2")
-
-        val df = sql("SELECT df1.id, df2.k FROM df1 JOIN df2 ON df1.k = df2.k AND df2.id < 2")
+      val df = sql(
+        """
+          |SELECT f.date_id, f.store_id FROM fact_sk f
+          |JOIN dim_store s ON f.store_id = s.store_id AND s.country = 'NL'
+        """.stripMargin)
 
-        checkPartitionPruningPredicate(df, true, false)
+      checkPartitionPruningPredicate(df, true, false)
 
-        checkAnswer(df, Row(0, 0) :: Row(1, 1) :: Nil)
-      }
+      checkAnswer(df, Row(1000, 1) :: Row(1010, 2) :: Row(1020, 2) :: Nil)
     }
   }
 
@@ -1157,7 +1143,7 @@ abstract class DynamicPartitionPruningSuiteBase
   test("cleanup any DPP filter that isn't pushed down due to expression id clashes") {
     withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
       withTable("fact", "dim") {
-        spark.range(1000).select($"id".as("A"), $"id".as("AA"))
+        spark.range(20).select($"id".as("A"), $"id".as("AA"))
           .write.partitionBy("A").format(tableFormat).mode("overwrite").saveAsTable("fact")
         spark.range(10).select($"id".as("B"), $"id".as("BB"))
           .write.format(tableFormat).mode("overwrite").saveAsTable("dim")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org