You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2018/10/16 22:49:44 UTC
spark git commit: [SPARK-25631][SPARK-25632][SQL][TEST] Improve the
test runtime of KafkaRDDSuite
Repository: spark
Updated Branches:
refs/heads/master bd2c44713 -> 9d4dd7992
[SPARK-25631][SPARK-25632][SQL][TEST] Improve the test runtime of KafkaRDDSuite
## What changes were proposed in this pull request?
Set a reasonable poll timeout thats used while consuming topics/partitions from kafka. In the
absence of it, a default of 2 minute is used as the timeout values. And all the negative tests take a minimum of 2 minute to execute.
After this change, we save about 4 minutes in this suite.
## How was this patch tested?
Test fix.
Closes #22670 from dilipbiswal/SPARK-25631.
Authored-by: Dilip Biswal <db...@us.ibm.com>
Signed-off-by: Sean Owen <se...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9d4dd799
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9d4dd799
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9d4dd799
Branch: refs/heads/master
Commit: 9d4dd7992bb8c3bd2ae3081734e6a5ed8531fddb
Parents: bd2c447
Author: Dilip Biswal <db...@us.ibm.com>
Authored: Tue Oct 16 17:49:40 2018 -0500
Committer: Sean Owen <se...@databricks.com>
Committed: Tue Oct 16 17:49:40 2018 -0500
----------------------------------------------------------------------
.../apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala | 3 +++
.../org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala | 4 ++++
2 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/9d4dd799/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
----------------------------------------------------------------------
diff --git a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
index 1974bb1..93d0d2f 100644
--- a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
+++ b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
@@ -52,6 +52,9 @@ class DirectKafkaStreamSuite
val sparkConf = new SparkConf()
.setMaster("local[4]")
.setAppName(this.getClass.getSimpleName)
+ // Set a timeout of 10 seconds that's going to be used to fetch topics/partitions from kafka.
+ // Othewise the poll timeout defaults to 2 minutes and causes test cases to run longer.
+ .set("spark.streaming.kafka.consumer.poll.ms", "10000")
private var ssc: StreamingContext = _
private var testDir: File = _
http://git-wip-us.apache.org/repos/asf/spark/blob/9d4dd799/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
----------------------------------------------------------------------
diff --git a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
index 561bca5..47bc8fe 100644
--- a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
+++ b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala
@@ -41,6 +41,10 @@ class KafkaRDDSuite extends SparkFunSuite with BeforeAndAfterAll {
private val sparkConf = new SparkConf().setMaster("local[4]")
.setAppName(this.getClass.getSimpleName)
+ // Set a timeout of 10 seconds that's going to be used to fetch topics/partitions from kafka.
+ // Othewise the poll timeout defaults to 2 minutes and causes test cases to run longer.
+ .set("spark.streaming.kafka.consumer.poll.ms", "10000")
+
private var sc: SparkContext = _
override def beforeAll {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org