You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2013/12/01 21:47:05 UTC
[2/3] git commit: Log exception in scheduler in addition to passing
it to the caller. Code Styling changes.
Log exception in scheduler in addition to passing it to the caller.
Code Styling changes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/be3ea239
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/be3ea239
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/be3ea239
Branch: refs/heads/master
Commit: be3ea2394fa2e626fb6b5f2cd46e7156016c9b3f
Parents: 4d53830
Author: Sundeep Narravula <su...@yahoo-inc.com>
Authored: Sun Dec 1 00:50:34 2013 -0800
Committer: Sundeep Narravula <su...@yahoo-inc.com>
Committed: Sun Dec 1 00:50:34 2013 -0800
----------------------------------------------------------------------
.../main/scala/org/apache/spark/scheduler/DAGScheduler.scala | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/be3ea239/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
index f6a4482..9159186 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
@@ -384,12 +384,14 @@ class DAGScheduler(
private[scheduler] def processEvent(event: DAGSchedulerEvent): Boolean = {
event match {
case JobSubmitted(jobId, rdd, func, partitions, allowLocal, callSite, listener, properties) =>
- var finalStage:Stage = null
+ var finalStage: Stage = null
try {
+ // New stage creation at times and if its not protected, the scheduler thread is killed.
+ // e.g. it can fail when jobs are run on HadoopRDD whose underlying hdfs files have been deleted
finalStage = newStage(rdd, partitions.size, None, jobId, Some(callSite))
} catch {
case e: Exception =>
- logWarning("Creating new stage failed due to exception - job: " + jobId )
+ logWarning("Creating new stage failed due to exception - job: " + jobId, e)
listener.jobFailed(e)
return false
}