You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/10/08 20:05:57 UTC
spark git commit: [SPARK-25630][TEST] Reduce test time of
HadoopFsRelationTest
Repository: spark
Updated Branches:
refs/heads/master 6353425af -> 6a60fb0aa
[SPARK-25630][TEST] Reduce test time of HadoopFsRelationTest
## What changes were proposed in this pull request?
There was 5 suites extends `HadoopFsRelationTest`, for testing "orc"/"parquet"/"text"/"json" data sources.
This PR refactor the base trait `HadoopFsRelationTest`:
1. Rename unnecessary loop for setting parquet conf
2. The test case `SPARK-8406: Avoids name collision while writing files` takes about 14 to 20 seconds. As now all the file format data source are using common code, for creating result files, we can test one data source(Parquet) only to reduce test time.
To run related 5 suites:
```
./build/sbt "hive/testOnly *HadoopFsRelationSuite"
```
The total test run time is reduced from 5 minutes 40 seconds to 3 minutes 50 seconds.
## How was this patch tested?
Unit test
Closes #22643 from gengliangwang/refactorHadoopFsRelationTest.
Authored-by: Gengliang Wang <ge...@databricks.com>
Signed-off-by: gatorsmile <ga...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6a60fb0a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6a60fb0a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6a60fb0a
Branch: refs/heads/master
Commit: 6a60fb0aad62e98c8a0e1c365819f31b1fc0132e
Parents: 6353425
Author: Gengliang Wang <ge...@databricks.com>
Authored: Mon Oct 8 13:05:53 2018 -0700
Committer: gatorsmile <ga...@gmail.com>
Committed: Mon Oct 8 13:05:53 2018 -0700
----------------------------------------------------------------------
.../sql/sources/HadoopFsRelationTest.scala | 50 ++++++++------------
.../sources/ParquetHadoopFsRelationSuite.scala | 31 +++++++++++-
2 files changed, 49 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6a60fb0a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
index b9ec940..6bd59fd 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
@@ -38,6 +38,10 @@ abstract class HadoopFsRelationTest extends QueryTest with SQLTestUtils with Tes
val dataSourceName: String
+ protected val parquetDataSourceName: String = "parquet"
+
+ private def isParquetDataSource: Boolean = dataSourceName == parquetDataSourceName
+
protected def supportsDataType(dataType: DataType): Boolean = true
val dataSchema =
@@ -114,10 +118,21 @@ abstract class HadoopFsRelationTest extends QueryTest with SQLTestUtils with Tes
new UDT.MyDenseVectorUDT()
).filter(supportsDataType)
- for (dataType <- supportedDataTypes) {
- for (parquetDictionaryEncodingEnabled <- Seq(true, false)) {
- test(s"test all data types - $dataType with parquet.enable.dictionary = " +
- s"$parquetDictionaryEncodingEnabled") {
+ test(s"test all data types") {
+ val parquetDictionaryEncodingEnabledConfs = if (isParquetDataSource) {
+ // Run with/without Parquet dictionary encoding enabled for Parquet data source.
+ Seq(true, false)
+ } else {
+ Seq(false)
+ }
+ for (dataType <- supportedDataTypes) {
+ for (parquetDictionaryEncodingEnabled <- parquetDictionaryEncodingEnabledConfs) {
+ val extraMessage = if (isParquetDataSource) {
+ s" with parquet.enable.dictionary = $parquetDictionaryEncodingEnabled"
+ } else {
+ ""
+ }
+ logInfo(s"Testing $dataType data type$extraMessage")
val extraOptions = Map[String, String](
"parquet.enable.dictionary" -> parquetDictionaryEncodingEnabled.toString
@@ -754,33 +769,6 @@ abstract class HadoopFsRelationTest extends QueryTest with SQLTestUtils with Tes
}
}
- // NOTE: This test suite is not super deterministic. On nodes with only relatively few cores
- // (4 or even 1), it's hard to reproduce the data loss issue. But on nodes with for example 8 or
- // more cores, the issue can be reproduced steadily. Fortunately our Jenkins builder meets this
- // requirement. We probably want to move this test case to spark-integration-tests or spark-perf
- // later.
- test("SPARK-8406: Avoids name collision while writing files") {
- withTempPath { dir =>
- val path = dir.getCanonicalPath
- spark
- .range(10000)
- .repartition(250)
- .write
- .mode(SaveMode.Overwrite)
- .format(dataSourceName)
- .save(path)
-
- assertResult(10000) {
- spark
- .read
- .format(dataSourceName)
- .option("dataSchema", StructType(StructField("id", LongType) :: Nil).json)
- .load(path)
- .count()
- }
- }
- }
-
test("SPARK-8887: Explicitly define which data types can be used as dynamic partition columns") {
val df = Seq(
(1, "v1", Array(1, 2, 3), Map("k1" -> "v1"), Tuple2(1, "4")),
http://git-wip-us.apache.org/repos/asf/spark/blob/6a60fb0a/sql/hive/src/test/scala/org/apache/spark/sql/sources/ParquetHadoopFsRelationSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/ParquetHadoopFsRelationSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/ParquetHadoopFsRelationSuite.scala
index 6858bbc..6ebc1d1 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/ParquetHadoopFsRelationSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/ParquetHadoopFsRelationSuite.scala
@@ -33,7 +33,7 @@ import org.apache.spark.sql.types._
class ParquetHadoopFsRelationSuite extends HadoopFsRelationTest {
import testImplicits._
- override val dataSourceName: String = "parquet"
+ override val dataSourceName: String = parquetDataSourceName
// Parquet does not play well with NullType.
override protected def supportsDataType(dataType: DataType): Boolean = dataType match {
@@ -232,4 +232,33 @@ class ParquetHadoopFsRelationSuite extends HadoopFsRelationTest {
}
}
}
+
+ // NOTE: This test suite is not super deterministic. On nodes with only relatively few cores
+ // (4 or even 1), it's hard to reproduce the data loss issue. But on nodes with for example 8 or
+ // more cores, the issue can be reproduced steadily. Fortunately our Jenkins builder meets this
+ // requirement. We probably want to move this test case to spark-integration-tests or spark-perf
+ // later.
+ // Also, this test is slow. As now all the file format data source are using common code
+ // for creating result files, we can test Parquet only to reduce test time.
+ test("SPARK-8406: Avoids name collision while writing files") {
+ withTempPath { dir =>
+ val path = dir.getCanonicalPath
+ spark
+ .range(10000)
+ .repartition(250)
+ .write
+ .mode(SaveMode.Overwrite)
+ .format(dataSourceName)
+ .save(path)
+
+ assertResult(10000) {
+ spark
+ .read
+ .format(dataSourceName)
+ .option("dataSchema", StructType(StructField("id", LongType) :: Nil).json)
+ .load(path)
+ .count()
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org