You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2015/05/23 08:07:59 UTC
spark git commit: [HOTFIX] Add tests for
SparkListenerApplicationStart with Driver Logs.
Repository: spark
Updated Branches:
refs/heads/master baa89838c -> 368b8c2b5
[HOTFIX] Add tests for SparkListenerApplicationStart with Driver Logs.
#6166 added the driver logs to `SparkListenerApplicationStart`. This adds tests in `JsonProtocolSuite` to ensure we don't regress.
Author: Hari Shreedharan <hs...@apache.org>
Closes #6368 from harishreedharan/jsonprotocol-test and squashes the following commits:
dc9eafc [Hari Shreedharan] [HOTFIX] Add tests for SparkListenerApplicationStart with Driver Logs.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/368b8c2b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/368b8c2b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/368b8c2b
Branch: refs/heads/master
Commit: 368b8c2b5ed8b06b00ac87059f75915b13ba3b8d
Parents: baa8983
Author: Hari Shreedharan <hs...@apache.org>
Authored: Fri May 22 23:07:56 2015 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Fri May 22 23:07:56 2015 -0700
----------------------------------------------------------------------
.../apache/spark/util/JsonProtocolSuite.scala | 25 ++++++++++++++++++--
1 file changed, 23 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/368b8c2b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index 0c5221d..0d9126f 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -75,10 +75,12 @@ class JsonProtocolSuite extends FunSuite {
val blockManagerRemoved = SparkListenerBlockManagerRemoved(2L,
BlockManagerId("Scarce", "to be counted...", 100))
val unpersistRdd = SparkListenerUnpersistRDD(12345)
+ val logUrlMap = Map("stderr" -> "mystderr", "stdout" -> "mystdout").toMap
val applicationStart = SparkListenerApplicationStart("The winner of all", Some("appId"),
42L, "Garfield", Some("appAttempt"))
+ val applicationStartWithLogs = SparkListenerApplicationStart("The winner of all", Some("appId"),
+ 42L, "Garfield", Some("appAttempt"), Some(logUrlMap))
val applicationEnd = SparkListenerApplicationEnd(42L)
- val logUrlMap = Map("stderr" -> "mystderr", "stdout" -> "mystdout").toMap
val executorAdded = SparkListenerExecutorAdded(executorAddedTime, "exec1",
new ExecutorInfo("Hostee.awesome.com", 11, logUrlMap))
val executorRemoved = SparkListenerExecutorRemoved(executorRemovedTime, "exec2", "test reason")
@@ -97,6 +99,7 @@ class JsonProtocolSuite extends FunSuite {
testEvent(blockManagerRemoved, blockManagerRemovedJsonString)
testEvent(unpersistRdd, unpersistRDDJsonString)
testEvent(applicationStart, applicationStartJsonString)
+ testEvent(applicationStartWithLogs, applicationStartJsonWithLogUrlsString)
testEvent(applicationEnd, applicationEndJsonString)
testEvent(executorAdded, executorAddedJsonString)
testEvent(executorRemoved, executorRemovedJsonString)
@@ -277,10 +280,12 @@ class JsonProtocolSuite extends FunSuite {
test("SparkListenerApplicationStart backwards compatibility") {
// SparkListenerApplicationStart in Spark 1.0.0 do not have an "appId" property.
// SparkListenerApplicationStart pre-Spark 1.4 does not have "appAttemptId".
- val applicationStart = SparkListenerApplicationStart("test", None, 1L, "user", None)
+ // SparkListenerApplicationStart pre-Spark 1.5 does not have "driverLogs
+ val applicationStart = SparkListenerApplicationStart("test", None, 1L, "user", None, None)
val oldEvent = JsonProtocol.applicationStartToJson(applicationStart)
.removeField({ _._1 == "App ID" })
.removeField({ _._1 == "App Attempt ID" })
+ .removeField({ _._1 == "Driver Logs"})
assert(applicationStart === JsonProtocol.applicationStartFromJson(oldEvent))
}
@@ -1544,6 +1549,22 @@ class JsonProtocolSuite extends FunSuite {
|}
"""
+ private val applicationStartJsonWithLogUrlsString =
+ """
+ |{
+ | "Event": "SparkListenerApplicationStart",
+ | "App Name": "The winner of all",
+ | "App ID": "appId",
+ | "Timestamp": 42,
+ | "User": "Garfield",
+ | "App Attempt ID": "appAttempt",
+ | "Driver Logs" : {
+ | "stderr" : "mystderr",
+ | "stdout" : "mystdout"
+ | }
+ |}
+ """
+
private val applicationEndJsonString =
"""
|{
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org