You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2015/05/05 16:04:20 UTC
spark git commit: [SPARK-5074] [CORE] [TESTS] Fix the flakey test
'run shuffle with map stage failure' in DAGSchedulerSuite
Repository: spark
Updated Branches:
refs/heads/master b83091ae4 -> 5ffc73e68
[SPARK-5074] [CORE] [TESTS] Fix the flakey test 'run shuffle with map stage failure' in DAGSchedulerSuite
Test failure: https://amplab.cs.berkeley.edu/jenkins/job/Spark-Master-SBT/AMPLAB_JENKINS_BUILD_PROFILE=hadoop2.2,label=centos/2240/testReport/junit/org.apache.spark.scheduler/DAGSchedulerSuite/run_shuffle_with_map_stage_failure/
This is because many tests share the same `JobListener`. Because after each test, `scheduler` isn't stopped. So actually it's still running. When running the test `run shuffle with map stage failure`, some previous test may trigger [ResubmitFailedStages](https://github.com/apache/spark/blob/ebc25a4ddfe07a67668217cec59893bc3b8cf730/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala#L1120) logic, and report `jobFailed` and override the global `failure` variable.
This PR uses `after` to call `scheduler.stop()` for each test.
Author: zsxwing <zs...@gmail.com>
Closes #5903 from zsxwing/SPARK-5074 and squashes the following commits:
1e6f13e [zsxwing] Fix the flakey test 'run shuffle with map stage failure' in DAGSchedulerSuite
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5ffc73e6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5ffc73e6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5ffc73e6
Branch: refs/heads/master
Commit: 5ffc73e68b3a6ea30c25931e9e0495a4c7e5654c
Parents: b83091a
Author: zsxwing <zs...@gmail.com>
Authored: Tue May 5 15:04:14 2015 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Tue May 5 15:04:14 2015 +0100
----------------------------------------------------------------------
.../scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5ffc73e6/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
index 2482603..6a8ae29 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/DAGSchedulerSuite.scala
@@ -174,6 +174,10 @@ class DAGSchedulerSuite
dagEventProcessLoopTester = new DAGSchedulerEventProcessLoopTester(scheduler)
}
+ after {
+ scheduler.stop()
+ }
+
override def afterAll() {
super.afterAll()
}
@@ -261,8 +265,9 @@ class DAGSchedulerSuite
override def taskSucceeded(partition: Int, value: Any) = numResults += 1
override def jobFailed(exception: Exception) = throw exception
}
- submit(new MyRDD(sc, 0, Nil), Array(), listener = fakeListener)
+ val jobId = submit(new MyRDD(sc, 0, Nil), Array(), listener = fakeListener)
assert(numResults === 0)
+ cancel(jobId)
}
test("run trivial job") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org