You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2016/09/16 09:20:52 UTC
spark git commit: [SPARK-17534][TESTS] Increase timeouts for
DirectKafkaStreamSuite tests
Repository: spark
Updated Branches:
refs/heads/master b2e272624 -> fc1efb720
[SPARK-17534][TESTS] Increase timeouts for DirectKafkaStreamSuite tests
**## What changes were proposed in this pull request?**
There are two tests in this suite that are particularly flaky on the following hardware:
2x Intel(R) Xeon(R) CPU E5-2697 v2 2.70GHz and 16 GB of RAM, 1 TB HDD
This simple PR increases the timeout times and batch duration so they can reliably pass
**## How was this patch tested?**
Existing unit tests with the two core box where I was seeing the failures often
Author: Adam Roberts <ar...@uk.ibm.com>
Closes #15094 from a-roberts/patch-6.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fc1efb72
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fc1efb72
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fc1efb72
Branch: refs/heads/master
Commit: fc1efb720c9c0033077c3c20ee144d0f757e6bcd
Parents: b2e2726
Author: Adam Roberts <ar...@uk.ibm.com>
Authored: Fri Sep 16 10:20:50 2016 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Fri Sep 16 10:20:50 2016 +0100
----------------------------------------------------------------------
.../spark/streaming/kafka010/DirectKafkaStreamSuite.scala | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/fc1efb72/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
----------------------------------------------------------------------
diff --git a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
index b1d90b8..e04f35e 100644
--- a/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
+++ b/external/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/DirectKafkaStreamSuite.scala
@@ -108,7 +108,7 @@ class DirectKafkaStreamSuite
val expectedTotal = (data.values.sum * topics.size) - 2
val kafkaParams = getKafkaParams("auto.offset.reset" -> "earliest")
- ssc = new StreamingContext(sparkConf, Milliseconds(200))
+ ssc = new StreamingContext(sparkConf, Milliseconds(1000))
val stream = withClue("Error creating direct stream") {
KafkaUtils.createDirectStream[String, String](
ssc,
@@ -150,7 +150,7 @@ class DirectKafkaStreamSuite
allReceived.addAll(Arrays.asList(rdd.map(r => (r.key, r.value)).collect(): _*))
}
ssc.start()
- eventually(timeout(20000.milliseconds), interval(200.milliseconds)) {
+ eventually(timeout(100000.milliseconds), interval(1000.milliseconds)) {
assert(allReceived.size === expectedTotal,
"didn't get expected number of messages, messages:\n" +
allReceived.asScala.mkString("\n"))
@@ -172,7 +172,7 @@ class DirectKafkaStreamSuite
val expectedTotal = (data.values.sum * 2) - 3
val kafkaParams = getKafkaParams("auto.offset.reset" -> "earliest")
- ssc = new StreamingContext(sparkConf, Milliseconds(200))
+ ssc = new StreamingContext(sparkConf, Milliseconds(1000))
val stream = withClue("Error creating direct stream") {
KafkaUtils.createDirectStream[String, String](
ssc,
@@ -214,7 +214,7 @@ class DirectKafkaStreamSuite
allReceived.addAll(Arrays.asList(rdd.map(r => (r.key, r.value)).collect(): _*))
}
ssc.start()
- eventually(timeout(20000.milliseconds), interval(200.milliseconds)) {
+ eventually(timeout(100000.milliseconds), interval(1000.milliseconds)) {
assert(allReceived.size === expectedTotal,
"didn't get expected number of messages, messages:\n" +
allReceived.asScala.mkString("\n"))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org