You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2020/09/04 00:53:47 UTC
[spark] branch branch-3.0 updated: [SPARK-32786][SQL][TEST] Improve
performance for some slow DPP tests
This is an automated email from the ASF dual-hosted git repository.
yamamuro pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 7320db2 [SPARK-32786][SQL][TEST] Improve performance for some slow DPP tests
7320db2 is described below
commit 7320db21bda01c162623839350de908042f4ebb8
Author: Zhenhua Wang <wz...@163.com>
AuthorDate: Fri Sep 4 09:33:20 2020 +0900
[SPARK-32786][SQL][TEST] Improve performance for some slow DPP tests
### What changes were proposed in this pull request?
The whole `DynamicPartitionPruningSuite` takes about 2 min on my laptop (either AE on or off). The slowest tests are `test("simple inner join triggers DPP with mock-up tables")` and `test("cleanup any DPP filter that isn't pushed down due to expression id clashes")`, which totally take about 1 min.
We can reuse existing test tables or use smaller tables to reduce the cost. After that, the two tests takes only about 1 sec in total, leading to 2x speedup for the suite.
### Why are the changes needed?
To speedup DPP test suites.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Modified two existing tests.
Closes #29636 from wzhfy/improve_dpp_test.
Authored-by: Zhenhua Wang <wz...@163.com>
Signed-off-by: Takeshi Yamamuro <ya...@apache.org>
---
.../spark/sql/DynamicPartitionPruningSuite.scala | 30 ++++++----------------
1 file changed, 8 insertions(+), 22 deletions(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index 47c5c9ea..8486d1d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -254,29 +254,15 @@ abstract class DynamicPartitionPruningSuiteBase
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true",
SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "false",
SQLConf.EXCHANGE_REUSE_ENABLED.key -> "false") {
- withTable("df1", "df2") {
- spark.range(1000)
- .select(col("id"), col("id").as("k"))
- .write
- .partitionBy("k")
- .format(tableFormat)
- .mode("overwrite")
- .saveAsTable("df1")
-
- spark.range(100)
- .select(col("id"), col("id").as("k"))
- .write
- .partitionBy("k")
- .format(tableFormat)
- .mode("overwrite")
- .saveAsTable("df2")
-
- val df = sql("SELECT df1.id, df2.k FROM df1 JOIN df2 ON df1.k = df2.k AND df2.id < 2")
+ val df = sql(
+ """
+ |SELECT f.date_id, f.store_id FROM fact_sk f
+ |JOIN dim_store s ON f.store_id = s.store_id AND s.country = 'NL'
+ """.stripMargin)
- checkPartitionPruningPredicate(df, true, false)
+ checkPartitionPruningPredicate(df, true, false)
- checkAnswer(df, Row(0, 0) :: Row(1, 1) :: Nil)
- }
+ checkAnswer(df, Row(1000, 1) :: Row(1010, 2) :: Row(1020, 2) :: Nil)
}
}
@@ -1157,7 +1143,7 @@ abstract class DynamicPartitionPruningSuiteBase
test("cleanup any DPP filter that isn't pushed down due to expression id clashes") {
withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_REUSE_BROADCAST_ONLY.key -> "true") {
withTable("fact", "dim") {
- spark.range(1000).select($"id".as("A"), $"id".as("AA"))
+ spark.range(20).select($"id".as("A"), $"id".as("AA"))
.write.partitionBy("A").format(tableFormat).mode("overwrite").saveAsTable("fact")
spark.range(10).select($"id".as("B"), $"id".as("BB"))
.write.format(tableFormat).mode("overwrite").saveAsTable("dim")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org