You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2014/11/26 00:51:11 UTC
spark git commit: Fix SPARK-4471: blockManagerIdFromJson function
throws exception while B...
Repository: spark
Updated Branches:
refs/heads/master 9afcbe494 -> 9bdf5da59
Fix SPARK-4471: blockManagerIdFromJson function throws exception while B...
Fix [SPARK-4471](https://issues.apache.org/jira/browse/SPARK-4471): blockManagerIdFromJson function throws exception while BlockManagerId be null in MetadataFetchFailedException
Author: hushan[胡珊] <hu...@xiaomi.com>
Closes #3340 from suyanNone/fix-blockmanagerId-jnothing-2 and squashes the following commits:
159f9a3 [hushan[胡珊]] Refine test code for blockmanager is null
4380d73 [hushan[胡珊]] remove useless blank line
3ccf651 [hushan[胡珊]] Fix SPARK-4471: blockManagerIdFromJson function throws exception while metadata fetch failed
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9bdf5da5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9bdf5da5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9bdf5da5
Branch: refs/heads/master
Commit: 9bdf5da59036c0b052df756fc4a28d64677072e7
Parents: 9afcbe4
Author: hushan[胡珊] <hu...@xiaomi.com>
Authored: Tue Nov 25 15:51:08 2014 -0800
Committer: Andrew Or <an...@databricks.com>
Committed: Tue Nov 25 15:51:08 2014 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/util/JsonProtocol.scala | 4 ++++
.../org/apache/spark/util/JsonProtocolSuite.scala | 15 ++++++++++++---
2 files changed, 16 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/9bdf5da5/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
index 7b5db1e..e7b80e8 100644
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@@ -688,6 +688,10 @@ private[spark] object JsonProtocol {
}
def blockManagerIdFromJson(json: JValue): BlockManagerId = {
+ // On metadata fetch fail, block manager ID can be null (SPARK-4471)
+ if (json == JNothing) {
+ return null
+ }
val executorId = (json \ "Executor ID").extract[String]
val host = (json \ "Host").extract[String]
val port = (json \ "Port").extract[Int]
http://git-wip-us.apache.org/repos/asf/spark/blob/9bdf5da5/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index 0bc9492..593d6dd 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -19,6 +19,8 @@ package org.apache.spark.util
import java.util.Properties
+import org.apache.spark.shuffle.MetadataFetchFailedException
+
import scala.collection.Map
import org.json4s.jackson.JsonMethods._
@@ -116,10 +118,13 @@ class JsonProtocolSuite extends FunSuite {
// TaskEndReason
val fetchFailed = FetchFailed(BlockManagerId("With or", "without you", 15), 17, 18, 19,
"Some exception")
+ val fetchMetadataFailed = new MetadataFetchFailedException(17,
+ 19, "metadata Fetch failed exception").toTaskEndReason
val exceptionFailure = new ExceptionFailure(exception, None)
testTaskEndReason(Success)
testTaskEndReason(Resubmitted)
testTaskEndReason(fetchFailed)
+ testTaskEndReason(fetchMetadataFailed)
testTaskEndReason(exceptionFailure)
testTaskEndReason(TaskResultLost)
testTaskEndReason(TaskKilled)
@@ -431,9 +436,13 @@ class JsonProtocolSuite extends FunSuite {
}
private def assertEquals(bm1: BlockManagerId, bm2: BlockManagerId) {
- assert(bm1.executorId === bm2.executorId)
- assert(bm1.host === bm2.host)
- assert(bm1.port === bm2.port)
+ if (bm1 == null || bm2 == null) {
+ assert(bm1 === bm2)
+ } else {
+ assert(bm1.executorId === bm2.executorId)
+ assert(bm1.host === bm2.host)
+ assert(bm1.port === bm2.port)
+ }
}
private def assertEquals(result1: JobResult, result2: JobResult) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org