You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/01/22 12:33:02 UTC
spark git commit: [MINOR][SQL][TEST] Test case cleanups for recent PRs
Repository: spark
Updated Branches:
refs/heads/master 78801881c -> 896e45af5
[MINOR][SQL][TEST] Test case cleanups for recent PRs
## What changes were proposed in this pull request?
Revert the unneeded test case changes we made in SPARK-23000
Also fixes the test suites that do not call `super.afterAll()` in the local `afterAll`. The `afterAll()` of `TestHiveSingleton` actually reset the environments.
## How was this patch tested?
N/A
Author: gatorsmile <ga...@gmail.com>
Closes #20341 from gatorsmile/testRelated.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/896e45af
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/896e45af
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/896e45af
Branch: refs/heads/master
Commit: 896e45af5fea264683b1d7d20a1711f33908a06f
Parents: 7880188
Author: gatorsmile <ga...@gmail.com>
Authored: Mon Jan 22 04:32:59 2018 -0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Mon Jan 22 04:32:59 2018 -0800
----------------------------------------------------------------------
.../apache/spark/sql/DataFrameJoinSuite.scala | 21 ++++++------
.../apache/spark/sql/hive/test/TestHive.scala | 3 +-
.../sql/hive/HiveMetastoreCatalogSuite.scala | 26 +++++++--------
.../sql/hive/execution/HiveUDAFSuite.scala | 8 +++--
.../sql/hive/execution/Hive_2_1_DDLSuite.scala | 6 +++-
.../execution/ObjectHashAggregateSuite.scala | 6 +++-
.../apache/spark/sql/hive/parquetSuites.scala | 35 ++++++++++++--------
7 files changed, 60 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/896e45af/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
index 1656f29..0d9eeab 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
@@ -21,6 +21,7 @@ import org.apache.spark.sql.catalyst.plans.{Inner, LeftOuter, RightOuter}
import org.apache.spark.sql.catalyst.plans.logical.Join
import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
import org.apache.spark.sql.functions._
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSQLContext
class DataFrameJoinSuite extends QueryTest with SharedSQLContext {
@@ -276,16 +277,14 @@ class DataFrameJoinSuite extends QueryTest with SharedSQLContext {
test("SPARK-23087: don't throw Analysis Exception in CheckCartesianProduct when join condition " +
"is false or null") {
- val df = spark.range(10)
- val dfNull = spark.range(10).select(lit(null).as("b"))
- val planNull = df.join(dfNull, $"id" === $"b", "left").queryExecution.analyzed
-
- spark.sessionState.executePlan(planNull).optimizedPlan
-
- val dfOne = df.select(lit(1).as("a"))
- val dfTwo = spark.range(10).select(lit(2).as("b"))
- val planFalse = dfOne.join(dfTwo, $"a" === $"b", "left").queryExecution.analyzed
-
- spark.sessionState.executePlan(planFalse).optimizedPlan
+ withSQLConf(SQLConf.CROSS_JOINS_ENABLED.key -> "false") {
+ val df = spark.range(10)
+ val dfNull = spark.range(10).select(lit(null).as("b"))
+ df.join(dfNull, $"id" === $"b", "left").queryExecution.optimizedPlan
+
+ val dfOne = df.select(lit(1).as("a"))
+ val dfTwo = spark.range(10).select(lit(2).as("b"))
+ dfOne.join(dfTwo, $"a" === $"b", "left").queryExecution.optimizedPlan
+ }
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/896e45af/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
index c84131f..7287e20 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/test/TestHive.scala
@@ -492,8 +492,7 @@ private[hive] class TestHiveSparkSession(
protected val originalUDFs: JavaSet[String] = FunctionRegistry.getFunctionNames
/**
- * Resets the test instance by deleting any tables that have been created.
- * TODO: also clear out UDFs, views, etc.
+ * Resets the test instance by deleting any table, view, temp view, and UDF that have been created
*/
def reset() {
try {
http://git-wip-us.apache.org/repos/asf/spark/blob/896e45af/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
index 83b4c86..ba9b944 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
@@ -166,13 +166,13 @@ class DataSourceWithHiveMetastoreCatalogSuite
))
).foreach { case (provider, (inputFormat, outputFormat, serde)) =>
test(s"Persist non-partitioned $provider relation into metastore as managed table") {
- withTable("default.t") {
+ withTable("t") {
withSQLConf(SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key -> "true") {
testDF
.write
.mode(SaveMode.Overwrite)
.format(provider)
- .saveAsTable("default.t")
+ .saveAsTable("t")
}
val hiveTable = sessionState.catalog.getTableMetadata(TableIdentifier("t", Some("default")))
@@ -187,15 +187,14 @@ class DataSourceWithHiveMetastoreCatalogSuite
assert(columns.map(_.name) === Seq("d1", "d2"))
assert(columns.map(_.dataType) === Seq(DecimalType(10, 3), StringType))
- checkAnswer(table("default.t"), testDF)
- assert(sparkSession.metadataHive.runSqlHive("SELECT * FROM default.t") ===
- Seq("1.1\t1", "2.1\t2"))
+ checkAnswer(table("t"), testDF)
+ assert(sparkSession.metadataHive.runSqlHive("SELECT * FROM t") === Seq("1.1\t1", "2.1\t2"))
}
}
test(s"Persist non-partitioned $provider relation into metastore as external table") {
withTempPath { dir =>
- withTable("default.t") {
+ withTable("t") {
val path = dir.getCanonicalFile
withSQLConf(SQLConf.PARQUET_WRITE_LEGACY_FORMAT.key -> "true") {
@@ -204,7 +203,7 @@ class DataSourceWithHiveMetastoreCatalogSuite
.mode(SaveMode.Overwrite)
.format(provider)
.option("path", path.toString)
- .saveAsTable("default.t")
+ .saveAsTable("t")
}
val hiveTable =
@@ -220,8 +219,8 @@ class DataSourceWithHiveMetastoreCatalogSuite
assert(columns.map(_.name) === Seq("d1", "d2"))
assert(columns.map(_.dataType) === Seq(DecimalType(10, 3), StringType))
- checkAnswer(table("default.t"), testDF)
- assert(sparkSession.metadataHive.runSqlHive("SELECT * FROM default.t") ===
+ checkAnswer(table("t"), testDF)
+ assert(sparkSession.metadataHive.runSqlHive("SELECT * FROM t") ===
Seq("1.1\t1", "2.1\t2"))
}
}
@@ -229,9 +228,9 @@ class DataSourceWithHiveMetastoreCatalogSuite
test(s"Persist non-partitioned $provider relation into metastore as managed table using CTAS") {
withTempPath { dir =>
- withTable("default.t") {
+ withTable("t") {
sql(
- s"""CREATE TABLE default.t USING $provider
+ s"""CREATE TABLE t USING $provider
|OPTIONS (path '${dir.toURI}')
|AS SELECT 1 AS d1, "val_1" AS d2
""".stripMargin)
@@ -249,9 +248,8 @@ class DataSourceWithHiveMetastoreCatalogSuite
assert(columns.map(_.name) === Seq("d1", "d2"))
assert(columns.map(_.dataType) === Seq(IntegerType, StringType))
- checkAnswer(table("default.t"), Row(1, "val_1"))
- assert(sparkSession.metadataHive.runSqlHive("SELECT * FROM default.t") ===
- Seq("1\tval_1"))
+ checkAnswer(table("t"), Row(1, "val_1"))
+ assert(sparkSession.metadataHive.runSqlHive("SELECT * FROM t") === Seq("1\tval_1"))
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/896e45af/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
index 8986fb5..7402c96 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala
@@ -49,8 +49,12 @@ class HiveUDAFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils {
}
protected override def afterAll(): Unit = {
- sql(s"DROP TEMPORARY FUNCTION IF EXISTS mock")
- sql(s"DROP TEMPORARY FUNCTION IF EXISTS hive_max")
+ try {
+ sql(s"DROP TEMPORARY FUNCTION IF EXISTS mock")
+ sql(s"DROP TEMPORARY FUNCTION IF EXISTS hive_max")
+ } finally {
+ super.afterAll()
+ }
}
test("built-in Hive UDAF") {
http://git-wip-us.apache.org/repos/asf/spark/blob/896e45af/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala
index bc82887..eaedac1 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/Hive_2_1_DDLSuite.scala
@@ -74,7 +74,11 @@ class Hive_2_1_DDLSuite extends SparkFunSuite with TestHiveSingleton with Before
}
override def afterAll(): Unit = {
- catalog = null
+ try {
+ catalog = null
+ } finally {
+ super.afterAll()
+ }
}
test("SPARK-21617: ALTER TABLE for non-compatible DataSource tables") {
http://git-wip-us.apache.org/repos/asf/spark/blob/896e45af/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ObjectHashAggregateSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ObjectHashAggregateSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ObjectHashAggregateSuite.scala
index 9eaf44c..8dbcd24 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ObjectHashAggregateSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/ObjectHashAggregateSuite.scala
@@ -47,7 +47,11 @@ class ObjectHashAggregateSuite
}
protected override def afterAll(): Unit = {
- sql(s"DROP TEMPORARY FUNCTION IF EXISTS hive_max")
+ try {
+ sql(s"DROP TEMPORARY FUNCTION IF EXISTS hive_max")
+ } finally {
+ super.afterAll()
+ }
}
test("typed_count without grouping keys") {
http://git-wip-us.apache.org/repos/asf/spark/blob/896e45af/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala
index 740e083..2327d83 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/parquetSuites.scala
@@ -180,15 +180,18 @@ class ParquetMetastoreSuite extends ParquetPartitioningTest {
}
override def afterAll(): Unit = {
- dropTables("partitioned_parquet",
- "partitioned_parquet_with_key",
- "partitioned_parquet_with_complextypes",
- "partitioned_parquet_with_key_and_complextypes",
- "normal_parquet",
- "jt",
- "jt_array",
- "test_parquet")
- super.afterAll()
+ try {
+ dropTables("partitioned_parquet",
+ "partitioned_parquet_with_key",
+ "partitioned_parquet_with_complextypes",
+ "partitioned_parquet_with_key_and_complextypes",
+ "normal_parquet",
+ "jt",
+ "jt_array",
+ "test_parquet")
+ } finally {
+ super.afterAll()
+ }
}
test(s"conversion is working") {
@@ -931,11 +934,15 @@ abstract class ParquetPartitioningTest extends QueryTest with SQLTestUtils with
}
override protected def afterAll(): Unit = {
- partitionedTableDir.delete()
- normalTableDir.delete()
- partitionedTableDirWithKey.delete()
- partitionedTableDirWithComplexTypes.delete()
- partitionedTableDirWithKeyAndComplexTypes.delete()
+ try {
+ partitionedTableDir.delete()
+ normalTableDir.delete()
+ partitionedTableDirWithKey.delete()
+ partitionedTableDirWithComplexTypes.delete()
+ partitionedTableDirWithKeyAndComplexTypes.delete()
+ } finally {
+ super.afterAll()
+ }
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org