You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2020/04/09 12:18:27 UTC
[spark] branch branch-3.0 updated: [SPARK-31291][SQL][TEST]
SQLQueryTestSuite: Sharing test data and test tables among multiple test
cases
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 886d712 [SPARK-31291][SQL][TEST] SQLQueryTestSuite: Sharing test data and test tables among multiple test cases
886d712 is described below
commit 886d71290b77c2e2ad5ac0076339f91aad162193
Author: gengjiaan <ge...@360.cn>
AuthorDate: Thu Apr 9 12:16:43 2020 +0000
[SPARK-31291][SQL][TEST] SQLQueryTestSuite: Sharing test data and test tables among multiple test cases
### What changes were proposed in this pull request?
`SQLQueryTestSuite` spend 35 minutes time to test.
I've listed the 10 test cases that took the longest time in the `SQL` module below.
Class | Spend time ↑ | Failure | Skip | Pass | Total test case
-- | -- | -- | -- | -- | --
SQLQueryTestSuite | 35 minutes | 0 | 1 | 230 | 231
TPCDSQuerySuite | 3 minutes 8 seconds | 0 | 0 | 156 | 156
SQLQuerySuite | 2 minutes 52 seconds | 0 | 0 | 185 | 185
DynamicPartitionPruningSuiteAEOff | 1 minutes 52 seconds | 0 | 0 | 22 | 22
DataFrameFunctionsSuite | 1 minutes 37 seconds | 0 | 0 | 102 | 102
DynamicPartitionPruningSuiteAEOn | 1 minutes 24 seconds | 0 | 0 | 22 | 22
DataFrameSuite | 1 minutes 14 seconds | 0 | 2 | 157 | 159
SubquerySuite | 1 minutes 12 seconds | 0 | 1 | 70 | 71
SingleLevelAggregateHashMapSuite | 1 minutes 1 seconds | 0 | 0 | 50 | 50
DataFrameAggregateSuite | 59 seconds | 0 | 0 | 50 | 50
I checked the code of `SQLQueryTestSuite` and found `SQLQueryTestSuite` load test data repeatedly.
This PR will improve the performance of `SQLQueryTestSuite`.
The total time run `SQLQueryTestSuite` before and after this PR show below.
Before
No | Time
-- | --
1 | 20 minutes, 22 seconds
2 | 23 minutes, 21 seconds
3 | 21 minutes, 19 seconds
4 | 22 minutes, 26 seconds
5 | 20 minutes, 8 seconds
After
No | Time
-- | --
1 | 20 minutes, 52 seconds
2 | 20 minutes, 47 seconds
3 | 20 minutes, 7 seconds
4 | 21 minutes, 10 seconds
5 | 20 minutes, 4 seconds
### Why are the changes needed?
Improve the performance of `SQLQueryTestSuite`.
### Does this PR introduce any user-facing change?
'No'.
### How was this patch tested?
Jenkins test
Closes #28060 from beliefer/avoid-load-test-data-repeatedly.
Lead-authored-by: gengjiaan <ge...@360.cn>
Co-authored-by: beliefer <be...@163.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 014d33570ba1518529a080a1ab79d33c536a0ee3)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../test/resources/sql-tests/results/limit.sql.out | 2 +-
.../sql-tests/results/show-tables.sql.out | 12 -------
.../resources/sql-tests/results/show-views.sql.out | 24 -------------
.../org/apache/spark/sql/SQLQueryTestSuite.scala | 40 +++++++++++++++++-----
4 files changed, 33 insertions(+), 45 deletions(-)
diff --git a/sql/core/src/test/resources/sql-tests/results/limit.sql.out b/sql/core/src/test/resources/sql-tests/results/limit.sql.out
index 281326e..074e7a6 100644
--- a/sql/core/src/test/resources/sql-tests/results/limit.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/limit.sql.out
@@ -88,7 +88,7 @@ SELECT * FROM testdata LIMIT key > 3
struct<>
-- !query output
org.apache.spark.sql.AnalysisException
-The limit expression must evaluate to a constant value, but got (testdata.`key` > 3);
+The limit expression must evaluate to a constant value, but got (spark_catalog.default.testdata.`key` > 3);
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
index dd2e7b1..a95b02c 100644
--- a/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show-tables.sql.out
@@ -63,15 +63,9 @@ SHOW TABLES
-- !query schema
struct<database:string,tableName:string,isTemporary:boolean>
-- !query output
-aggtest
-arraydata
-mapdata
-onek
show_t1
show_t2
show_t3
-tenk1
-testdata
-- !query
@@ -79,15 +73,9 @@ SHOW TABLES IN showdb
-- !query schema
struct<database:string,tableName:string,isTemporary:boolean>
-- !query output
-aggtest
-arraydata
-mapdata
-onek
show_t1
show_t2
show_t3
-tenk1
-testdata
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/show-views.sql.out b/sql/core/src/test/resources/sql-tests/results/show-views.sql.out
index ee8a747..d88790d 100644
--- a/sql/core/src/test/resources/sql-tests/results/show-views.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/show-views.sql.out
@@ -63,12 +63,6 @@ SHOW VIEWS
-- !query schema
struct<namespace:string,viewName:string,isTemporary:boolean>
-- !query output
-aggtest
-arraydata
-mapdata
-onek
-tenk1
-testdata
view_1
view_2
view_4
@@ -79,12 +73,6 @@ SHOW VIEWS FROM showdb
-- !query schema
struct<namespace:string,viewName:string,isTemporary:boolean>
-- !query output
-aggtest
-arraydata
-mapdata
-onek
-tenk1
-testdata
view_1
view_2
view_4
@@ -95,12 +83,6 @@ SHOW VIEWS IN showdb
-- !query schema
struct<namespace:string,viewName:string,isTemporary:boolean>
-- !query output
-aggtest
-arraydata
-mapdata
-onek
-tenk1
-testdata
view_1
view_2
view_4
@@ -111,12 +93,6 @@ SHOW VIEWS IN global_temp
-- !query schema
struct<namespace:string,viewName:string,isTemporary:boolean>
-- !query output
-aggtest
-arraydata
-mapdata
-onek
-tenk1
-testdata
view_3
view_4
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index 848966a..4a39d95 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -361,7 +361,6 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
// Create a local SparkSession to have stronger isolation between different test cases.
// This does not isolate catalog changes.
val localSparkSession = spark.newSession()
- loadTestData(localSparkSession)
testCase match {
case udfTestCase: UDFTest =>
@@ -570,14 +569,20 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
}
/** Load built-in test tables into the SparkSession. */
- private def loadTestData(session: SparkSession): Unit = {
+ private def createTestTables(session: SparkSession): Unit = {
import session.implicits._
- (1 to 100).map(i => (i, i.toString)).toDF("key", "value").createOrReplaceTempView("testdata")
+ (1 to 100).map(i => (i, i.toString)).toDF("key", "value")
+ .repartition(1)
+ .write
+ .format("parquet")
+ .saveAsTable("testdata")
((Seq(1, 2, 3), Seq(Seq(1, 2, 3))) :: (Seq(2, 3, 4), Seq(Seq(2, 3, 4))) :: Nil)
.toDF("arraycol", "nestedarraycol")
- .createOrReplaceTempView("arraydata")
+ .write
+ .format("parquet")
+ .saveAsTable("arraydata")
(Tuple1(Map(1 -> "a1", 2 -> "b1", 3 -> "c1", 4 -> "d1", 5 -> "e1")) ::
Tuple1(Map(1 -> "a2", 2 -> "b2", 3 -> "c2", 4 -> "d2")) ::
@@ -585,7 +590,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
Tuple1(Map(1 -> "a4", 2 -> "b4")) ::
Tuple1(Map(1 -> "a5")) :: Nil)
.toDF("mapcol")
- .createOrReplaceTempView("mapdata")
+ .write
+ .format("parquet")
+ .saveAsTable("mapdata")
session
.read
@@ -593,7 +600,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
.options(Map("delimiter" -> "\t", "header" -> "false"))
.schema("a int, b float")
.load(testFile("test-data/postgresql/agg.data"))
- .createOrReplaceTempView("aggtest")
+ .write
+ .format("parquet")
+ .saveAsTable("aggtest")
session
.read
@@ -619,7 +628,9 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
|string4 string
""".stripMargin)
.load(testFile("test-data/postgresql/onek.data"))
- .createOrReplaceTempView("onek")
+ .write
+ .format("parquet")
+ .saveAsTable("onek")
session
.read
@@ -645,7 +656,18 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
|string4 string
""".stripMargin)
.load(testFile("test-data/postgresql/tenk.data"))
- .createOrReplaceTempView("tenk1")
+ .write
+ .format("parquet")
+ .saveAsTable("tenk1")
+ }
+
+ private def removeTestTables(session: SparkSession): Unit = {
+ session.sql("DROP TABLE IF EXISTS testdata")
+ session.sql("DROP TABLE IF EXISTS arraydata")
+ session.sql("DROP TABLE IF EXISTS mapdata")
+ session.sql("DROP TABLE IF EXISTS aggtest")
+ session.sql("DROP TABLE IF EXISTS onek")
+ session.sql("DROP TABLE IF EXISTS tenk1")
}
private val originalTimeZone = TimeZone.getDefault
@@ -653,6 +675,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
override def beforeAll(): Unit = {
super.beforeAll()
+ createTestTables(spark)
// Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
// Add Locale setting
@@ -664,6 +687,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
try {
TimeZone.setDefault(originalTimeZone)
Locale.setDefault(originalLocale)
+ removeTestTables(spark)
// For debugging dump some statistics about how much time was spent in various optimizer rules
logWarning(RuleExecutor.dumpTimeSpent())
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org