You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by zs...@apache.org on 2016/03/07 22:45:49 UTC
spark git commit: [SPARK-13655] Improve isolation between tests in
KinesisBackedBlockRDDSuite
Repository: spark
Updated Branches:
refs/heads/master b6071a700 -> e9e67b39a
[SPARK-13655] Improve isolation between tests in KinesisBackedBlockRDDSuite
This patch modifies `KinesisBackedBlockRDDTests` to increase the isolation between tests in order to fix a bug which causes the tests to hang.
See #11558 for more details.
/cc zsxwing srowen
Author: Josh Rosen <jo...@databricks.com>
Closes #11564 from JoshRosen/SPARK-13655.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e9e67b39
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e9e67b39
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e9e67b39
Branch: refs/heads/master
Commit: e9e67b39abb23a88d8be2d0fea5b5fd93184a25b
Parents: b6071a7
Author: Josh Rosen <jo...@databricks.com>
Authored: Mon Mar 7 13:45:45 2016 -0800
Committer: Shixiong Zhu <sh...@databricks.com>
Committed: Mon Mar 7 13:45:45 2016 -0800
----------------------------------------------------------------------
.../kinesis/KinesisBackedBlockRDDSuite.scala | 30 +++++++++++---------
1 file changed, 17 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e9e67b39/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala
----------------------------------------------------------------------
diff --git a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala b/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala
index e916f1e..2555332 100644
--- a/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala
+++ b/extras/kinesis-asl/src/test/scala/org/apache/spark/streaming/kinesis/KinesisBackedBlockRDDSuite.scala
@@ -17,13 +17,13 @@
package org.apache.spark.streaming.kinesis
-import org.scalatest.BeforeAndAfterAll
+import org.scalatest.BeforeAndAfterEach
-import org.apache.spark.{SparkConf, SparkContext, SparkException}
+import org.apache.spark.{LocalSparkContext, SparkConf, SparkContext, SparkException}
import org.apache.spark.storage.{BlockId, BlockManager, StorageLevel, StreamBlockId}
abstract class KinesisBackedBlockRDDTests(aggregateTestData: Boolean)
- extends KinesisFunSuite with BeforeAndAfterAll {
+ extends KinesisFunSuite with BeforeAndAfterEach with LocalSparkContext {
private val testData = 1 to 8
@@ -35,10 +35,10 @@ abstract class KinesisBackedBlockRDDTests(aggregateTestData: Boolean)
private var shardIdToRange: Map[String, SequenceNumberRange] = null
private var allRanges: Seq[SequenceNumberRange] = null
- private var sc: SparkContext = null
private var blockManager: BlockManager = null
override def beforeAll(): Unit = {
+ super.beforeAll()
runIfTestsEnabled("Prepare KinesisTestUtils") {
testUtils = new KPLBasedKinesisTestUtils()
testUtils.createStream()
@@ -55,19 +55,23 @@ abstract class KinesisBackedBlockRDDTests(aggregateTestData: Boolean)
(shardId, seqNumRange)
}
allRanges = shardIdToRange.values.toSeq
-
- val conf = new SparkConf().setMaster("local[4]").setAppName("KinesisBackedBlockRDDSuite")
- sc = new SparkContext(conf)
- blockManager = sc.env.blockManager
}
}
+ override def beforeEach(): Unit = {
+ super.beforeEach()
+ val conf = new SparkConf().setMaster("local[4]").setAppName("KinesisBackedBlockRDDSuite")
+ sc = new SparkContext(conf)
+ blockManager = sc.env.blockManager
+ }
+
override def afterAll(): Unit = {
- if (testUtils != null) {
- testUtils.deleteStream()
- }
- if (sc != null) {
- sc.stop()
+ try {
+ if (testUtils != null) {
+ testUtils.deleteStream()
+ }
+ } finally {
+ super.afterAll()
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org