You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ir...@apache.org on 2017/01/19 15:08:39 UTC
[4/4] spark git commit: [SPARK-16654][CORE] Add UI coverage for
Application Level Blacklisting
[SPARK-16654][CORE] Add UI coverage for Application Level Blacklisting
Builds on top of work in SPARK-8425 to update Application Level Blacklisting in the scheduler.
## What changes were proposed in this pull request?
Adds a UI to these patches by:
- defining new listener events for blacklisting and unblacklisting, nodes and executors;
- sending said events at the relevant points in BlacklistTracker;
- adding JSON (de)serialization code for these events;
- augmenting the Executors UI page to show which, and how many, executors are blacklisted;
- adding a unit test to make sure events are being fired;
- adding HistoryServerSuite coverage to verify that the SHS reads these events correctly.
- updates the Executor UI to show Blacklisted/Active/Dead as a tri-state in Executors Status
Updates .rat-excludes to pass tests.
username squito
## How was this patch tested?
./dev/run-tests
testOnly org.apache.spark.util.JsonProtocolSuite
testOnly org.apache.spark.scheduler.BlacklistTrackerSuite
testOnly org.apache.spark.deploy.history.HistoryServerSuite
https://github.com/jsoltren/jose-utils/blob/master/blacklist/test-blacklist.sh
Author: Jos� Hiram Soltren <jo...@cloudera.com>
Closes #16346 from jsoltren/SPARK-16654-submit.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/640f9423
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/640f9423
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/640f9423
Branch: refs/heads/master
Commit: 640f942337e1ce87075195998bd051e19c4b50b9
Parents: 064fadd
Author: Jos� Hiram Soltren <jo...@cloudera.com>
Authored: Thu Jan 19 09:08:18 2017 -0600
Committer: Imran Rashid <ir...@cloudera.com>
Committed: Thu Jan 19 09:08:18 2017 -0600
----------------------------------------------------------------------
.../org/apache/spark/SparkFirehoseListener.java | 20 ++++
.../spark/ui/static/executorspage-template.html | 5 +
.../org/apache/spark/ui/static/executorspage.js | 39 ++++--
.../spark/scheduler/BlacklistTracker.scala | 16 ++-
.../spark/scheduler/EventLoggingListener.scala | 16 +++
.../apache/spark/scheduler/SparkListener.scala | 54 +++++++++
.../spark/scheduler/SparkListenerBus.scala | 8 ++
.../spark/scheduler/TaskSchedulerImpl.scala | 10 +-
.../org/apache/spark/status/api/v1/api.scala | 1 +
.../scala/org/apache/spark/ui/ToolTips.scala | 3 +
.../apache/spark/ui/exec/ExecutorsPage.scala | 2 +
.../org/apache/spark/ui/exec/ExecutorsTab.scala | 57 ++++++++-
.../apache/spark/ui/jobs/ExecutorTable.scala | 6 +
.../scala/org/apache/spark/ui/jobs/UIData.scala | 2 +
.../application_list_json_expectation.json | 44 +++++--
.../completed_app_list_json_expectation.json | 44 +++++--
.../executor_list_json_expectation.json | 1 +
.../executor_node_blacklisting_expectation.json | 118 +++++++++++++++++++
...blacklisting_unblacklisting_expectation.json | 118 +++++++++++++++++++
.../limit_app_list_json_expectation.json | 64 ++++------
.../maxDate2_app_list_json_expectation.json | 2 +-
.../maxDate_app_list_json_expectation.json | 4 +-
.../minDate_app_list_json_expectation.json | 40 ++++++-
.../one_app_json_expectation.json | 2 +-
.../one_app_multi_attempt_json_expectation.json | 4 +-
.../one_stage_attempt_json_expectation.json | 80 ++++++-------
.../one_stage_json_expectation.json | 80 ++++++-------
...summary_w__custom_quantiles_expectation.json | 2 +-
...stage_with_accumulable_json_expectation.json | 100 ++++++++--------
.../spark-events/app-20161115172038-0000 | 75 ++++++++++++
.../spark-events/app-20161116163331-0000 | 68 +++++++++++
.../deploy/history/HistoryServerSuite.scala | 4 +-
.../spark/scheduler/BlacklistTrackerSuite.scala | 42 ++++++-
.../spark/scheduler/TaskSetManagerSuite.scala | 3 +-
.../apache/spark/util/JsonProtocolSuite.scala | 49 ++++++++
dev/.rat-excludes | 2 +
36 files changed, 950 insertions(+), 235 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/java/org/apache/spark/SparkFirehoseListener.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/spark/SparkFirehoseListener.java b/core/src/main/java/org/apache/spark/SparkFirehoseListener.java
index 97eed61..9fe97b4 100644
--- a/core/src/main/java/org/apache/spark/SparkFirehoseListener.java
+++ b/core/src/main/java/org/apache/spark/SparkFirehoseListener.java
@@ -114,6 +114,26 @@ public class SparkFirehoseListener implements SparkListenerInterface {
}
@Override
+ public final void onExecutorBlacklisted(SparkListenerExecutorBlacklisted executorBlacklisted) {
+ onEvent(executorBlacklisted);
+ }
+
+ @Override
+ public final void onExecutorUnblacklisted(SparkListenerExecutorUnblacklisted executorUnblacklisted) {
+ onEvent(executorUnblacklisted);
+ }
+
+ @Override
+ public final void onNodeBlacklisted(SparkListenerNodeBlacklisted nodeBlacklisted) {
+ onEvent(nodeBlacklisted);
+ }
+
+ @Override
+ public final void onNodeUnblacklisted(SparkListenerNodeUnblacklisted nodeUnblacklisted) {
+ onEvent(nodeUnblacklisted);
+ }
+
+ @Override
public void onBlockUpdated(SparkListenerBlockUpdated blockUpdated) {
onEvent(blockUpdated);
}
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
index 64ea719..4e83d6d 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
+++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage-template.html
@@ -45,6 +45,11 @@ limitations under the License.
title="Bytes and records written to disk in order to be read by a shuffle in a future stage.">
Shuffle Write</span>
</th>
+ <th>
+ <span data-toggle="tooltip" data-placement="left"
+ title="Number of executors blacklisted by the scheduler due to task failures.">
+ Blacklisted</span>
+ </th>
</thead>
<tbody>
</tbody>
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
index fe5db6a..7dbfe32 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
@@ -182,7 +182,7 @@ $(document).ready(function () {
executorsSummary = $("#active-executors");
getStandAloneppId(function (appId) {
-
+
var endPoint = createRESTEndPoint(appId);
$.getJSON(endPoint, function (response, status, jqXHR) {
var summary = [];
@@ -202,7 +202,8 @@ $(document).ready(function () {
var allTotalInputBytes = 0;
var allTotalShuffleRead = 0;
var allTotalShuffleWrite = 0;
-
+ var allTotalBlacklisted = 0;
+
var activeExecCnt = 0;
var activeRDDBlocks = 0;
var activeMemoryUsed = 0;
@@ -219,7 +220,8 @@ $(document).ready(function () {
var activeTotalInputBytes = 0;
var activeTotalShuffleRead = 0;
var activeTotalShuffleWrite = 0;
-
+ var activeTotalBlacklisted = 0;
+
var deadExecCnt = 0;
var deadRDDBlocks = 0;
var deadMemoryUsed = 0;
@@ -236,7 +238,8 @@ $(document).ready(function () {
var deadTotalInputBytes = 0;
var deadTotalShuffleRead = 0;
var deadTotalShuffleWrite = 0;
-
+ var deadTotalBlacklisted = 0;
+
response.forEach(function (exec) {
allExecCnt += 1;
allRDDBlocks += exec.rddBlocks;
@@ -254,6 +257,7 @@ $(document).ready(function () {
allTotalInputBytes += exec.totalInputBytes;
allTotalShuffleRead += exec.totalShuffleRead;
allTotalShuffleWrite += exec.totalShuffleWrite;
+ allTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
if (exec.isActive) {
activeExecCnt += 1;
activeRDDBlocks += exec.rddBlocks;
@@ -271,6 +275,7 @@ $(document).ready(function () {
activeTotalInputBytes += exec.totalInputBytes;
activeTotalShuffleRead += exec.totalShuffleRead;
activeTotalShuffleWrite += exec.totalShuffleWrite;
+ activeTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
} else {
deadExecCnt += 1;
deadRDDBlocks += exec.rddBlocks;
@@ -288,9 +293,10 @@ $(document).ready(function () {
deadTotalInputBytes += exec.totalInputBytes;
deadTotalShuffleRead += exec.totalShuffleRead;
deadTotalShuffleWrite += exec.totalShuffleWrite;
+ deadTotalBlacklisted += exec.isBlacklisted ? 1 : 0;
}
});
-
+
var totalSummary = {
"execCnt": ( "Total(" + allExecCnt + ")"),
"allRDDBlocks": allRDDBlocks,
@@ -307,7 +313,8 @@ $(document).ready(function () {
"allTotalGCTime": allTotalGCTime,
"allTotalInputBytes": allTotalInputBytes,
"allTotalShuffleRead": allTotalShuffleRead,
- "allTotalShuffleWrite": allTotalShuffleWrite
+ "allTotalShuffleWrite": allTotalShuffleWrite,
+ "allTotalBlacklisted": allTotalBlacklisted
};
var activeSummary = {
"execCnt": ( "Active(" + activeExecCnt + ")"),
@@ -325,7 +332,8 @@ $(document).ready(function () {
"allTotalGCTime": activeTotalGCTime,
"allTotalInputBytes": activeTotalInputBytes,
"allTotalShuffleRead": activeTotalShuffleRead,
- "allTotalShuffleWrite": activeTotalShuffleWrite
+ "allTotalShuffleWrite": activeTotalShuffleWrite,
+ "allTotalBlacklisted": activeTotalBlacklisted
};
var deadSummary = {
"execCnt": ( "Dead(" + deadExecCnt + ")" ),
@@ -343,12 +351,13 @@ $(document).ready(function () {
"allTotalGCTime": deadTotalGCTime,
"allTotalInputBytes": deadTotalInputBytes,
"allTotalShuffleRead": deadTotalShuffleRead,
- "allTotalShuffleWrite": deadTotalShuffleWrite
+ "allTotalShuffleWrite": deadTotalShuffleWrite,
+ "allTotalBlacklisted": deadTotalBlacklisted
};
-
+
var data = {executors: response, "execSummary": [activeSummary, deadSummary, totalSummary]};
$.get(createTemplateURI(appId), function (template) {
-
+
executorsSummary.append(Mustache.render($(template).filter("#executors-summary-template").html(), data));
var selector = "#active-executors-table";
var conf = {
@@ -360,7 +369,12 @@ $(document).ready(function () {
}
},
{data: 'hostPort'},
- {data: 'isActive', render: formatStatus},
+ {data: 'isActive', render: function (data, type, row) {
+ if (type !== 'display') return data;
+ if (row.isBlacklisted) return "Blacklisted";
+ else return formatStatus (data, type);
+ }
+ },
{data: 'rddBlocks'},
{
data: function (row, type) {
@@ -474,7 +488,8 @@ $(document).ready(function () {
},
{data: 'allTotalInputBytes', render: formatBytes},
{data: 'allTotalShuffleRead', render: formatBytes},
- {data: 'allTotalShuffleWrite', render: formatBytes}
+ {data: 'allTotalShuffleWrite', render: formatBytes},
+ {data: 'allTotalBlacklisted'}
],
"paging": false,
"searching": false,
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala b/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
index bf7a62e..77d5c97 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala
@@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicReference
import scala.collection.mutable.{ArrayBuffer, HashMap, HashSet}
-import org.apache.spark.SparkConf
+import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.internal.Logging
import org.apache.spark.internal.config
import org.apache.spark.util.{Clock, SystemClock, Utils}
@@ -48,9 +48,14 @@ import org.apache.spark.util.{Clock, SystemClock, Utils}
* one exception is [[nodeBlacklist()]], which can be called without holding a lock.
*/
private[scheduler] class BlacklistTracker (
+ private val listenerBus: LiveListenerBus,
conf: SparkConf,
clock: Clock = new SystemClock()) extends Logging {
+ def this(sc: SparkContext) = {
+ this(sc.listenerBus, sc.conf)
+ }
+
BlacklistTracker.validateBlacklistConfs(conf)
private val MAX_FAILURES_PER_EXEC = conf.get(config.MAX_FAILURES_PER_EXEC)
private val MAX_FAILED_EXEC_PER_NODE = conf.get(config.MAX_FAILED_EXEC_PER_NODE)
@@ -103,6 +108,7 @@ private[scheduler] class BlacklistTracker (
execsToUnblacklist.foreach { exec =>
val status = executorIdToBlacklistStatus.remove(exec).get
val failedExecsOnNode = nodeToBlacklistedExecs(status.node)
+ listenerBus.post(SparkListenerExecutorUnblacklisted(now, exec))
failedExecsOnNode.remove(exec)
if (failedExecsOnNode.isEmpty) {
nodeToBlacklistedExecs.remove(status.node)
@@ -114,7 +120,10 @@ private[scheduler] class BlacklistTracker (
// Un-blacklist any nodes that have been blacklisted longer than the blacklist timeout.
logInfo(s"Removing nodes $nodesToUnblacklist from blacklist because the blacklist " +
s"has timed out")
- nodeIdToBlacklistExpiryTime --= nodesToUnblacklist
+ nodesToUnblacklist.foreach { node =>
+ nodeIdToBlacklistExpiryTime.remove(node)
+ listenerBus.post(SparkListenerNodeUnblacklisted(now, node))
+ }
_nodeBlacklist.set(nodeIdToBlacklistExpiryTime.keySet.toSet)
}
updateNextExpiryTime()
@@ -161,6 +170,8 @@ private[scheduler] class BlacklistTracker (
s" task failures in successful task sets")
val node = failuresInTaskSet.node
executorIdToBlacklistStatus.put(exec, BlacklistedExecutor(node, expiryTimeForNewBlacklists))
+ listenerBus.post(SparkListenerExecutorBlacklisted(now, exec, newTotal))
+ executorIdToFailureList.remove(exec)
updateNextExpiryTime()
// In addition to blacklisting the executor, we also update the data for failures on the
@@ -174,6 +185,7 @@ private[scheduler] class BlacklistTracker (
logInfo(s"Blacklisting node $node because it has ${blacklistedExecsOnNode.size} " +
s"executors blacklisted: ${blacklistedExecsOnNode}")
nodeIdToBlacklistExpiryTime.put(node, expiryTimeForNewBlacklists)
+ listenerBus.post(SparkListenerNodeBlacklisted(now, node, blacklistedExecsOnNode.size))
_nodeBlacklist.set(nodeIdToBlacklistExpiryTime.keySet.toSet)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
index f39565e..af9bdef 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
@@ -193,6 +193,22 @@ private[spark] class EventLoggingListener(
logEvent(event, flushLogger = true)
}
+ override def onExecutorBlacklisted(event: SparkListenerExecutorBlacklisted): Unit = {
+ logEvent(event, flushLogger = true)
+ }
+
+ override def onExecutorUnblacklisted(event: SparkListenerExecutorUnblacklisted): Unit = {
+ logEvent(event, flushLogger = true)
+ }
+
+ override def onNodeBlacklisted(event: SparkListenerNodeBlacklisted): Unit = {
+ logEvent(event, flushLogger = true)
+ }
+
+ override def onNodeUnblacklisted(event: SparkListenerNodeUnblacklisted): Unit = {
+ logEvent(event, flushLogger = true)
+ }
+
// No-op because logging every update would be overkill
override def onBlockUpdated(event: SparkListenerBlockUpdated): Unit = {}
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
index 7618dfe..1b12af7 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
@@ -106,6 +106,28 @@ case class SparkListenerExecutorRemoved(time: Long, executorId: String, reason:
extends SparkListenerEvent
@DeveloperApi
+case class SparkListenerExecutorBlacklisted(
+ time: Long,
+ executorId: String,
+ taskFailures: Int)
+ extends SparkListenerEvent
+
+@DeveloperApi
+case class SparkListenerExecutorUnblacklisted(time: Long, executorId: String)
+ extends SparkListenerEvent
+
+@DeveloperApi
+case class SparkListenerNodeBlacklisted(
+ time: Long,
+ hostId: String,
+ executorFailures: Int)
+ extends SparkListenerEvent
+
+@DeveloperApi
+case class SparkListenerNodeUnblacklisted(time: Long, hostId: String)
+ extends SparkListenerEvent
+
+@DeveloperApi
case class SparkListenerBlockUpdated(blockUpdatedInfo: BlockUpdatedInfo) extends SparkListenerEvent
/**
@@ -239,6 +261,26 @@ private[spark] trait SparkListenerInterface {
def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit
/**
+ * Called when the driver blacklists an executor for a Spark application.
+ */
+ def onExecutorBlacklisted(executorBlacklisted: SparkListenerExecutorBlacklisted): Unit
+
+ /**
+ * Called when the driver re-enables a previously blacklisted executor.
+ */
+ def onExecutorUnblacklisted(executorUnblacklisted: SparkListenerExecutorUnblacklisted): Unit
+
+ /**
+ * Called when the driver blacklists a node for a Spark application.
+ */
+ def onNodeBlacklisted(nodeBlacklisted: SparkListenerNodeBlacklisted): Unit
+
+ /**
+ * Called when the driver re-enables a previously blacklisted node.
+ */
+ def onNodeUnblacklisted(nodeUnblacklisted: SparkListenerNodeUnblacklisted): Unit
+
+ /**
* Called when the driver receives a block update info.
*/
def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit
@@ -293,6 +335,18 @@ abstract class SparkListener extends SparkListenerInterface {
override def onExecutorRemoved(executorRemoved: SparkListenerExecutorRemoved): Unit = { }
+ override def onExecutorBlacklisted(
+ executorBlacklisted: SparkListenerExecutorBlacklisted): Unit = { }
+
+ override def onExecutorUnblacklisted(
+ executorUnblacklisted: SparkListenerExecutorUnblacklisted): Unit = { }
+
+ override def onNodeBlacklisted(
+ nodeBlacklisted: SparkListenerNodeBlacklisted): Unit = { }
+
+ override def onNodeUnblacklisted(
+ nodeUnblacklisted: SparkListenerNodeUnblacklisted): Unit = { }
+
override def onBlockUpdated(blockUpdated: SparkListenerBlockUpdated): Unit = { }
override def onOtherEvent(event: SparkListenerEvent): Unit = { }
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala b/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala
index 471586a..3ff3633 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala
@@ -61,6 +61,14 @@ private[spark] trait SparkListenerBus
listener.onExecutorAdded(executorAdded)
case executorRemoved: SparkListenerExecutorRemoved =>
listener.onExecutorRemoved(executorRemoved)
+ case executorBlacklisted: SparkListenerExecutorBlacklisted =>
+ listener.onExecutorBlacklisted(executorBlacklisted)
+ case executorUnblacklisted: SparkListenerExecutorUnblacklisted =>
+ listener.onExecutorUnblacklisted(executorUnblacklisted)
+ case nodeBlacklisted: SparkListenerNodeBlacklisted =>
+ listener.onNodeBlacklisted(nodeBlacklisted)
+ case nodeUnblacklisted: SparkListenerNodeUnblacklisted =>
+ listener.onNodeUnblacklisted(nodeUnblacklisted)
case blockUpdated: SparkListenerBlockUpdated =>
listener.onBlockUpdated(blockUpdated)
case logStart: SparkListenerLogStart => // ignore event log metadata
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
index 9a8e313..72ed55a 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskSchedulerImpl.scala
@@ -63,14 +63,14 @@ private[spark] class TaskSchedulerImpl private[scheduler](
this(
sc,
sc.conf.get(config.MAX_TASK_FAILURES),
- TaskSchedulerImpl.maybeCreateBlacklistTracker(sc.conf))
+ TaskSchedulerImpl.maybeCreateBlacklistTracker(sc))
}
def this(sc: SparkContext, maxTaskFailures: Int, isLocal: Boolean) = {
this(
sc,
maxTaskFailures,
- TaskSchedulerImpl.maybeCreateBlacklistTracker(sc.conf),
+ TaskSchedulerImpl.maybeCreateBlacklistTracker(sc),
isLocal = isLocal)
}
@@ -717,9 +717,9 @@ private[spark] object TaskSchedulerImpl {
retval.toList
}
- private def maybeCreateBlacklistTracker(conf: SparkConf): Option[BlacklistTracker] = {
- if (BlacklistTracker.isBlacklistEnabled(conf)) {
- Some(new BlacklistTracker(conf))
+ private def maybeCreateBlacklistTracker(sc: SparkContext): Option[BlacklistTracker] = {
+ if (BlacklistTracker.isBlacklistEnabled(sc.conf)) {
+ Some(new BlacklistTracker(sc))
} else {
None
}
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index 44a929b..7d035b1 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -73,6 +73,7 @@ class ExecutorSummary private[spark](
val totalInputBytes: Long,
val totalShuffleRead: Long,
val totalShuffleWrite: Long,
+ val isBlacklisted: Boolean,
val maxMemory: Long,
val executorLogs: Map[String, String])
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/ui/ToolTips.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/ToolTips.scala b/core/src/main/scala/org/apache/spark/ui/ToolTips.scala
index 3cc5353..766cc65 100644
--- a/core/src/main/scala/org/apache/spark/ui/ToolTips.scala
+++ b/core/src/main/scala/org/apache/spark/ui/ToolTips.scala
@@ -91,6 +91,9 @@ private[spark] object ToolTips {
val TASK_TIME =
"Shaded red when garbage collection (GC) time is over 10% of task time"
+ val BLACKLISTED =
+ "Shows if this executor has been blacklisted by the scheduler due to task failures."
+
val APPLICATION_EXECUTOR_LIMIT =
"""Maximum number of executors that this application will use. This limit is finite only when
dynamic allocation is enabled. The number of granted executors may exceed the limit
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
index 7953d77..2d1691e 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
@@ -39,6 +39,7 @@ private[ui] case class ExecutorSummaryInfo(
totalInputBytes: Long,
totalShuffleRead: Long,
totalShuffleWrite: Long,
+ isBlacklisted: Int,
maxMemory: Long,
executorLogs: Map[String, String])
@@ -101,6 +102,7 @@ private[spark] object ExecutorsPage {
taskSummary.inputBytes,
taskSummary.shuffleRead,
taskSummary.shuffleWrite,
+ taskSummary.isBlacklisted,
maxMem,
taskSummary.executorLogs
)
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
index 678571f..8ae712f 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsTab.scala
@@ -53,7 +53,8 @@ private[ui] case class ExecutorTaskSummary(
var shuffleRead: Long = 0L,
var shuffleWrite: Long = 0L,
var executorLogs: Map[String, String] = Map.empty,
- var isAlive: Boolean = true
+ var isAlive: Boolean = true,
+ var isBlacklisted: Boolean = false
)
/**
@@ -73,7 +74,8 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener, conf: Spar
def deadStorageStatusList: Seq[StorageStatus] = storageStatusListener.deadStorageStatusList
- override def onExecutorAdded(executorAdded: SparkListenerExecutorAdded): Unit = synchronized {
+ override def onExecutorAdded(
+ executorAdded: SparkListenerExecutorAdded): Unit = synchronized {
val eid = executorAdded.executorId
val taskSummary = executorToTaskSummary.getOrElseUpdate(eid, ExecutorTaskSummary(eid))
taskSummary.executorLogs = executorAdded.executorInfo.logUrlMap
@@ -100,7 +102,8 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener, conf: Spar
executorToTaskSummary.get(executorRemoved.executorId).foreach(e => e.isAlive = false)
}
- override def onApplicationStart(applicationStart: SparkListenerApplicationStart): Unit = {
+ override def onApplicationStart(
+ applicationStart: SparkListenerApplicationStart): Unit = {
applicationStart.driverLogs.foreach { logs =>
val storageStatus = activeStorageStatusList.find { s =>
s.blockManagerId.executorId == SparkContext.LEGACY_DRIVER_IDENTIFIER ||
@@ -114,13 +117,15 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener, conf: Spar
}
}
- override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = synchronized {
+ override def onTaskStart(
+ taskStart: SparkListenerTaskStart): Unit = synchronized {
val eid = taskStart.taskInfo.executorId
val taskSummary = executorToTaskSummary.getOrElseUpdate(eid, ExecutorTaskSummary(eid))
taskSummary.tasksActive += 1
}
- override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = synchronized {
+ override def onTaskEnd(
+ taskEnd: SparkListenerTaskEnd): Unit = synchronized {
val info = taskEnd.taskInfo
if (info != null) {
val eid = info.executorId
@@ -157,4 +162,46 @@ class ExecutorsListener(storageStatusListener: StorageStatusListener, conf: Spar
}
}
+ private def updateExecutorBlacklist(
+ eid: String,
+ isBlacklisted: Boolean): Unit = {
+ val execTaskSummary = executorToTaskSummary.getOrElseUpdate(eid, ExecutorTaskSummary(eid))
+ execTaskSummary.isBlacklisted = isBlacklisted
+ }
+
+ override def onExecutorBlacklisted(
+ executorBlacklisted: SparkListenerExecutorBlacklisted)
+ : Unit = synchronized {
+ updateExecutorBlacklist(executorBlacklisted.executorId, true)
+ }
+
+ override def onExecutorUnblacklisted(
+ executorUnblacklisted: SparkListenerExecutorUnblacklisted)
+ : Unit = synchronized {
+ updateExecutorBlacklist(executorUnblacklisted.executorId, false)
+ }
+
+ override def onNodeBlacklisted(
+ nodeBlacklisted: SparkListenerNodeBlacklisted)
+ : Unit = synchronized {
+ // Implicitly blacklist every executor associated with this node, and show this in the UI.
+ activeStorageStatusList.foreach { status =>
+ if (status.blockManagerId.host == nodeBlacklisted.hostId) {
+ updateExecutorBlacklist(status.blockManagerId.executorId, true)
+ }
+ }
+ }
+
+ override def onNodeUnblacklisted(
+ nodeUnblacklisted: SparkListenerNodeUnblacklisted)
+ : Unit = synchronized {
+ // Implicitly unblacklist every executor associated with this node, regardless of how
+ // they may have been blacklisted initially (either explicitly through executor blacklisting
+ // or implicitly through node blacklisting). Show this in the UI.
+ activeStorageStatusList.foreach { status =>
+ if (status.blockManagerId.host == nodeUnblacklisted.hostId) {
+ updateExecutorBlacklist(status.blockManagerId.executorId, false)
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
index 9fb3f35..cd1b02a 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
@@ -85,6 +85,11 @@ private[ui] class ExecutorTable(stageId: Int, stageAttemptId: Int, parent: Stage
<th>Shuffle Spill (Memory)</th>
<th>Shuffle Spill (Disk)</th>
}}
+ <th>
+ <span data-toggle="tooltip" title={ToolTips.BLACKLISTED}>
+ Blacklisted
+ </span>
+ </th>
</thead>
<tbody>
{createExecutorTable()}
@@ -160,6 +165,7 @@ private[ui] class ExecutorTable(stageId: Int, stageAttemptId: Int, parent: Stage
{Utils.bytesToString(v.diskBytesSpilled)}
</td>
}}
+ <td>{v.isBlacklisted}</td>
</tr>
}
case None =>
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
index 9ce8542..371dad9 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
@@ -43,6 +43,7 @@ private[spark] object UIData {
var shuffleWriteRecords : Long = 0
var memoryBytesSpilled : Long = 0
var diskBytesSpilled : Long = 0
+ var isBlacklisted : Int = 0
}
class JobUIData(
@@ -92,6 +93,7 @@ private[spark] object UIData {
var shuffleWriteRecords: Long = _
var memoryBytesSpilled: Long = _
var diskBytesSpilled: Long = _
+ var isBlacklisted: Int = _
var schedulingPool: String = ""
var description: Option[String] = None
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
index cba44c8..10902ab 100644
--- a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
@@ -1,4 +1,32 @@
[ {
+ "id" : "app-20161116163331-0000",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2016-11-16T22:33:29.916GMT",
+ "endTime" : "2016-11-16T22:33:40.587GMT",
+ "lastUpdated" : "",
+ "duration" : 10671,
+ "sparkUser" : "jose",
+ "completed" : true,
+ "endTimeEpoch" : 1479335620587,
+ "startTimeEpoch" : 1479335609916,
+ "lastUpdatedEpoch" : 0
+ } ]
+}, {
+ "id" : "app-20161115172038-0000",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2016-11-15T23:20:37.079GMT",
+ "endTime" : "2016-11-15T23:22:18.874GMT",
+ "lastUpdated" : "",
+ "duration" : 101795,
+ "sparkUser" : "jose",
+ "completed" : true,
+ "endTimeEpoch" : 1479252138874,
+ "startTimeEpoch" : 1479252037079,
+ "lastUpdatedEpoch" : 0
+ } ]
+}, {
"id" : "local-1430917381534",
"name" : "Spark shell",
"attempts" : [ {
@@ -8,8 +36,8 @@
"duration" : 10505,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1430917380893,
"endTimeEpoch" : 1430917391398,
+ "startTimeEpoch" : 1430917380893,
"lastUpdatedEpoch" : 0
} ]
}, {
@@ -23,8 +51,8 @@
"duration" : 57,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1430917380893,
"endTimeEpoch" : 1430917380950,
+ "startTimeEpoch" : 1430917380893,
"lastUpdatedEpoch" : 0
}, {
"attemptId" : "1",
@@ -34,8 +62,8 @@
"duration" : 10,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1430917380880,
"endTimeEpoch" : 1430917380890,
+ "startTimeEpoch" : 1430917380880,
"lastUpdatedEpoch" : 0
} ]
}, {
@@ -49,8 +77,8 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1426633910242,
"endTimeEpoch" : 1426633945177,
+ "startTimeEpoch" : 1426633910242,
"lastUpdatedEpoch" : 0
}, {
"attemptId" : "1",
@@ -60,8 +88,8 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1426533910242,
"endTimeEpoch" : 1426533945177,
+ "startTimeEpoch" : 1426533910242,
"lastUpdatedEpoch" : 0
} ]
}, {
@@ -74,8 +102,8 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1425081758277,
"endTimeEpoch" : 1425081766912,
+ "startTimeEpoch" : 1425081758277,
"lastUpdatedEpoch" : 0
} ]
}, {
@@ -88,8 +116,8 @@
"duration" : 9011,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1422981779720,
"endTimeEpoch" : 1422981788731,
+ "startTimeEpoch" : 1422981779720,
"lastUpdatedEpoch" : 0
} ]
}, {
@@ -102,8 +130,8 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1422981758277,
"endTimeEpoch" : 1422981766912,
+ "startTimeEpoch" : 1422981758277,
"lastUpdatedEpoch" : 0
} ]
} ]
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
index cba44c8..10902ab 100644
--- a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
@@ -1,4 +1,32 @@
[ {
+ "id" : "app-20161116163331-0000",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2016-11-16T22:33:29.916GMT",
+ "endTime" : "2016-11-16T22:33:40.587GMT",
+ "lastUpdated" : "",
+ "duration" : 10671,
+ "sparkUser" : "jose",
+ "completed" : true,
+ "endTimeEpoch" : 1479335620587,
+ "startTimeEpoch" : 1479335609916,
+ "lastUpdatedEpoch" : 0
+ } ]
+}, {
+ "id" : "app-20161115172038-0000",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2016-11-15T23:20:37.079GMT",
+ "endTime" : "2016-11-15T23:22:18.874GMT",
+ "lastUpdated" : "",
+ "duration" : 101795,
+ "sparkUser" : "jose",
+ "completed" : true,
+ "endTimeEpoch" : 1479252138874,
+ "startTimeEpoch" : 1479252037079,
+ "lastUpdatedEpoch" : 0
+ } ]
+}, {
"id" : "local-1430917381534",
"name" : "Spark shell",
"attempts" : [ {
@@ -8,8 +36,8 @@
"duration" : 10505,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1430917380893,
"endTimeEpoch" : 1430917391398,
+ "startTimeEpoch" : 1430917380893,
"lastUpdatedEpoch" : 0
} ]
}, {
@@ -23,8 +51,8 @@
"duration" : 57,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1430917380893,
"endTimeEpoch" : 1430917380950,
+ "startTimeEpoch" : 1430917380893,
"lastUpdatedEpoch" : 0
}, {
"attemptId" : "1",
@@ -34,8 +62,8 @@
"duration" : 10,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1430917380880,
"endTimeEpoch" : 1430917380890,
+ "startTimeEpoch" : 1430917380880,
"lastUpdatedEpoch" : 0
} ]
}, {
@@ -49,8 +77,8 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1426633910242,
"endTimeEpoch" : 1426633945177,
+ "startTimeEpoch" : 1426633910242,
"lastUpdatedEpoch" : 0
}, {
"attemptId" : "1",
@@ -60,8 +88,8 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1426533910242,
"endTimeEpoch" : 1426533945177,
+ "startTimeEpoch" : 1426533910242,
"lastUpdatedEpoch" : 0
} ]
}, {
@@ -74,8 +102,8 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1425081758277,
"endTimeEpoch" : 1425081766912,
+ "startTimeEpoch" : 1425081758277,
"lastUpdatedEpoch" : 0
} ]
}, {
@@ -88,8 +116,8 @@
"duration" : 9011,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1422981779720,
"endTimeEpoch" : 1422981788731,
+ "startTimeEpoch" : 1422981779720,
"lastUpdatedEpoch" : 0
} ]
}, {
@@ -102,8 +130,8 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1422981758277,
"endTimeEpoch" : 1422981766912,
+ "startTimeEpoch" : 1422981758277,
"lastUpdatedEpoch" : 0
} ]
} ]
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
index e7db674..6b9f29e 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
@@ -16,6 +16,7 @@
"totalInputBytes" : 28000288,
"totalShuffleRead" : 0,
"totalShuffleWrite" : 13180,
+ "isBlacklisted" : false,
"maxMemory" : 278302556,
"executorLogs" : { }
} ]
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
new file mode 100644
index 0000000..5914a1c
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
@@ -0,0 +1,118 @@
+[ {
+ "id" : "2",
+ "hostPort" : "172.22.0.167:51487",
+ "isActive" : true,
+ "rddBlocks" : 0,
+ "memoryUsed" : 0,
+ "diskUsed" : 0,
+ "totalCores" : 4,
+ "maxTasks" : 4,
+ "activeTasks" : 0,
+ "failedTasks" : 4,
+ "completedTasks" : 0,
+ "totalTasks" : 4,
+ "totalDuration" : 2537,
+ "totalGCTime" : 88,
+ "totalInputBytes" : 0,
+ "totalShuffleRead" : 0,
+ "totalShuffleWrite" : 0,
+ "isBlacklisted" : true,
+ "maxMemory" : 384093388,
+ "executorLogs" : {
+ "stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout",
+ "stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr"
+ }
+}, {
+ "id" : "driver",
+ "hostPort" : "172.22.0.167:51475",
+ "isActive" : true,
+ "rddBlocks" : 0,
+ "memoryUsed" : 0,
+ "diskUsed" : 0,
+ "totalCores" : 0,
+ "maxTasks" : 0,
+ "activeTasks" : 0,
+ "failedTasks" : 0,
+ "completedTasks" : 0,
+ "totalTasks" : 0,
+ "totalDuration" : 0,
+ "totalGCTime" : 0,
+ "totalInputBytes" : 0,
+ "totalShuffleRead" : 0,
+ "totalShuffleWrite" : 0,
+ "isBlacklisted" : true,
+ "maxMemory" : 384093388,
+ "executorLogs" : { }
+}, {
+ "id" : "1",
+ "hostPort" : "172.22.0.167:51490",
+ "isActive" : true,
+ "rddBlocks" : 0,
+ "memoryUsed" : 0,
+ "diskUsed" : 0,
+ "totalCores" : 4,
+ "maxTasks" : 4,
+ "activeTasks" : 0,
+ "failedTasks" : 0,
+ "completedTasks" : 4,
+ "totalTasks" : 4,
+ "totalDuration" : 3152,
+ "totalGCTime" : 68,
+ "totalInputBytes" : 0,
+ "totalShuffleRead" : 0,
+ "totalShuffleWrite" : 0,
+ "isBlacklisted" : true,
+ "maxMemory" : 384093388,
+ "executorLogs" : {
+ "stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout",
+ "stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr"
+ }
+}, {
+ "id" : "0",
+ "hostPort" : "172.22.0.167:51491",
+ "isActive" : true,
+ "rddBlocks" : 0,
+ "memoryUsed" : 0,
+ "diskUsed" : 0,
+ "totalCores" : 4,
+ "maxTasks" : 4,
+ "activeTasks" : 0,
+ "failedTasks" : 4,
+ "completedTasks" : 0,
+ "totalTasks" : 4,
+ "totalDuration" : 2551,
+ "totalGCTime" : 116,
+ "totalInputBytes" : 0,
+ "totalShuffleRead" : 0,
+ "totalShuffleWrite" : 0,
+ "isBlacklisted" : true,
+ "maxMemory" : 384093388,
+ "executorLogs" : {
+ "stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout",
+ "stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr"
+ }
+}, {
+ "id" : "3",
+ "hostPort" : "172.22.0.167:51485",
+ "isActive" : true,
+ "rddBlocks" : 0,
+ "memoryUsed" : 0,
+ "diskUsed" : 0,
+ "totalCores" : 4,
+ "maxTasks" : 4,
+ "activeTasks" : 0,
+ "failedTasks" : 0,
+ "completedTasks" : 12,
+ "totalTasks" : 12,
+ "totalDuration" : 2453,
+ "totalGCTime" : 72,
+ "totalInputBytes" : 0,
+ "totalShuffleRead" : 0,
+ "totalShuffleWrite" : 0,
+ "isBlacklisted" : true,
+ "maxMemory" : 384093388,
+ "executorLogs" : {
+ "stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout",
+ "stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr"
+ }
+} ]
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
new file mode 100644
index 0000000..92e249c
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_unblacklisting_expectation.json
@@ -0,0 +1,118 @@
+[ {
+ "id" : "2",
+ "hostPort" : "172.22.0.111:64539",
+ "isActive" : true,
+ "rddBlocks" : 0,
+ "memoryUsed" : 0,
+ "diskUsed" : 0,
+ "totalCores" : 4,
+ "maxTasks" : 4,
+ "activeTasks" : 0,
+ "failedTasks" : 6,
+ "completedTasks" : 0,
+ "totalTasks" : 6,
+ "totalDuration" : 2792,
+ "totalGCTime" : 128,
+ "totalInputBytes" : 0,
+ "totalShuffleRead" : 0,
+ "totalShuffleWrite" : 0,
+ "isBlacklisted" : false,
+ "maxMemory" : 384093388,
+ "executorLogs" : {
+ "stdout" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stdout",
+ "stderr" : "http://172.22.0.111:64519/logPage/?appId=app-20161115172038-0000&executorId=2&logType=stderr"
+ }
+}, {
+ "id" : "driver",
+ "hostPort" : "172.22.0.111:64527",
+ "isActive" : true,
+ "rddBlocks" : 0,
+ "memoryUsed" : 0,
+ "diskUsed" : 0,
+ "totalCores" : 0,
+ "maxTasks" : 0,
+ "activeTasks" : 0,
+ "failedTasks" : 0,
+ "completedTasks" : 0,
+ "totalTasks" : 0,
+ "totalDuration" : 0,
+ "totalGCTime" : 0,
+ "totalInputBytes" : 0,
+ "totalShuffleRead" : 0,
+ "totalShuffleWrite" : 0,
+ "isBlacklisted" : false,
+ "maxMemory" : 384093388,
+ "executorLogs" : { }
+}, {
+ "id" : "1",
+ "hostPort" : "172.22.0.111:64541",
+ "isActive" : true,
+ "rddBlocks" : 0,
+ "memoryUsed" : 0,
+ "diskUsed" : 0,
+ "totalCores" : 4,
+ "maxTasks" : 4,
+ "activeTasks" : 0,
+ "failedTasks" : 0,
+ "completedTasks" : 12,
+ "totalTasks" : 12,
+ "totalDuration" : 2613,
+ "totalGCTime" : 84,
+ "totalInputBytes" : 0,
+ "totalShuffleRead" : 0,
+ "totalShuffleWrite" : 0,
+ "isBlacklisted" : false,
+ "maxMemory" : 384093388,
+ "executorLogs" : {
+ "stdout" : "http://172.22.0.111:64518/logPage/?appId=app-20161115172038-0000&executorId=1&logType=stdout",
+ "stderr" : "http://172.22.0.111:64518/logPage/?appId=app-20161115172038-0000&executorId=1&logType=stderr"
+ }
+}, {
+ "id" : "0",
+ "hostPort" : "172.22.0.111:64540",
+ "isActive" : true,
+ "rddBlocks" : 0,
+ "memoryUsed" : 0,
+ "diskUsed" : 0,
+ "totalCores" : 4,
+ "maxTasks" : 4,
+ "activeTasks" : 0,
+ "failedTasks" : 4,
+ "completedTasks" : 0,
+ "totalTasks" : 4,
+ "totalDuration" : 2741,
+ "totalGCTime" : 120,
+ "totalInputBytes" : 0,
+ "totalShuffleRead" : 0,
+ "totalShuffleWrite" : 0,
+ "isBlacklisted" : false,
+ "maxMemory" : 384093388,
+ "executorLogs" : {
+ "stdout" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stdout",
+ "stderr" : "http://172.22.0.111:64517/logPage/?appId=app-20161115172038-0000&executorId=0&logType=stderr"
+ }
+}, {
+ "id" : "3",
+ "hostPort" : "172.22.0.111:64543",
+ "isActive" : true,
+ "rddBlocks" : 0,
+ "memoryUsed" : 0,
+ "diskUsed" : 0,
+ "totalCores" : 4,
+ "maxTasks" : 4,
+ "activeTasks" : 0,
+ "failedTasks" : 0,
+ "completedTasks" : 4,
+ "totalTasks" : 4,
+ "totalDuration" : 3457,
+ "totalGCTime" : 72,
+ "totalInputBytes" : 0,
+ "totalShuffleRead" : 0,
+ "totalShuffleWrite" : 0,
+ "isBlacklisted" : false,
+ "maxMemory" : 384093388,
+ "executorLogs" : {
+ "stdout" : "http://172.22.0.111:64521/logPage/?appId=app-20161115172038-0000&executorId=3&logType=stdout",
+ "stderr" : "http://172.22.0.111:64521/logPage/?appId=app-20161115172038-0000&executorId=3&logType=stderr"
+ }
+} ]
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
index 9165f54..8820c71 100644
--- a/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
@@ -1,67 +1,43 @@
[ {
- "id" : "local-1430917381534",
+ "id" : "app-20161116163331-0000",
"name" : "Spark shell",
"attempts" : [ {
- "startTime" : "2015-05-06T13:03:00.893GMT",
- "endTime" : "2015-05-06T13:03:11.398GMT",
+ "startTime" : "2016-11-16T22:33:29.916GMT",
+ "endTime" : "2016-11-16T22:33:40.587GMT",
"lastUpdated" : "",
- "duration" : 10505,
- "sparkUser" : "irashid",
+ "duration" : 10671,
+ "sparkUser" : "jose",
"completed" : true,
- "startTimeEpoch" : 1430917380893,
- "endTimeEpoch" : 1430917391398,
+ "endTimeEpoch" : 1479335620587,
+ "startTimeEpoch" : 1479335609916,
"lastUpdatedEpoch" : 0
} ]
}, {
- "id" : "local-1430917381535",
+ "id" : "app-20161115172038-0000",
"name" : "Spark shell",
"attempts" : [ {
- "attemptId" : "2",
- "startTime" : "2015-05-06T13:03:00.893GMT",
- "endTime" : "2015-05-06T13:03:00.950GMT",
- "lastUpdated" : "",
- "duration" : 57,
- "sparkUser" : "irashid",
- "completed" : true,
- "startTimeEpoch" : 1430917380893,
- "endTimeEpoch" : 1430917380950,
- "lastUpdatedEpoch" : 0
- }, {
- "attemptId" : "1",
- "startTime" : "2015-05-06T13:03:00.880GMT",
- "endTime" : "2015-05-06T13:03:00.890GMT",
+ "startTime" : "2016-11-15T23:20:37.079GMT",
+ "endTime" : "2016-11-15T23:22:18.874GMT",
"lastUpdated" : "",
- "duration" : 10,
- "sparkUser" : "irashid",
+ "duration" : 101795,
+ "sparkUser" : "jose",
"completed" : true,
- "startTimeEpoch" : 1430917380880,
- "endTimeEpoch" : 1430917380890,
+ "endTimeEpoch" : 1479252138874,
+ "startTimeEpoch" : 1479252037079,
"lastUpdatedEpoch" : 0
} ]
}, {
- "id" : "local-1426533911241",
+ "id" : "local-1430917381534",
"name" : "Spark shell",
"attempts" : [ {
- "attemptId" : "2",
- "startTime" : "2015-03-17T23:11:50.242GMT",
- "endTime" : "2015-03-17T23:12:25.177GMT",
- "lastUpdated" : "",
- "duration" : 34935,
- "sparkUser" : "irashid",
- "completed" : true,
- "startTimeEpoch" : 1426633910242,
- "endTimeEpoch" : 1426633945177,
- "lastUpdatedEpoch" : 0
- }, {
- "attemptId" : "1",
- "startTime" : "2015-03-16T19:25:10.242GMT",
- "endTime" : "2015-03-16T19:25:45.177GMT",
+ "startTime" : "2015-05-06T13:03:00.893GMT",
+ "endTime" : "2015-05-06T13:03:11.398GMT",
"lastUpdated" : "",
- "duration" : 34935,
+ "duration" : 10505,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1426533910242,
- "endTimeEpoch" : 1426533945177,
+ "endTimeEpoch" : 1430917391398,
+ "startTimeEpoch" : 1430917380893,
"lastUpdatedEpoch" : 0
} ]
} ]
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
index a525d61..c3fe4db 100644
--- a/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
@@ -8,8 +8,8 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1422981758277,
"endTimeEpoch" : 1422981766912,
+ "startTimeEpoch" : 1422981758277,
"lastUpdatedEpoch" : 0
} ]
} ]
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
index cc567f6..8281fa7 100644
--- a/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
@@ -8,8 +8,8 @@
"duration" : 9011,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1422981779720,
"endTimeEpoch" : 1422981788731,
+ "startTimeEpoch" : 1422981779720,
"lastUpdatedEpoch" : 0
} ]
}, {
@@ -22,8 +22,8 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1422981758277,
"endTimeEpoch" : 1422981766912,
+ "startTimeEpoch" : 1422981758277,
"lastUpdatedEpoch" : 0
} ]
} ]
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
index c934a87..1930281 100644
--- a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
@@ -1,4 +1,32 @@
[ {
+ "id" : "app-20161116163331-0000",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2016-11-16T22:33:29.916GMT",
+ "endTime" : "2016-11-16T22:33:40.587GMT",
+ "lastUpdated" : "",
+ "duration" : 10671,
+ "sparkUser" : "jose",
+ "completed" : true,
+ "endTimeEpoch" : 1479335620587,
+ "startTimeEpoch" : 1479335609916,
+ "lastUpdatedEpoch" : 0
+ } ]
+}, {
+ "id" : "app-20161115172038-0000",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2016-11-15T23:20:37.079GMT",
+ "endTime" : "2016-11-15T23:22:18.874GMT",
+ "lastUpdated" : "",
+ "duration" : 101795,
+ "sparkUser" : "jose",
+ "completed" : true,
+ "endTimeEpoch" : 1479252138874,
+ "startTimeEpoch" : 1479252037079,
+ "lastUpdatedEpoch" : 0
+ } ]
+}, {
"id" : "local-1430917381534",
"name" : "Spark shell",
"attempts" : [ {
@@ -8,8 +36,8 @@
"duration" : 10505,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1430917380893,
"endTimeEpoch" : 1430917391398,
+ "startTimeEpoch" : 1430917380893,
"lastUpdatedEpoch" : 0
} ]
}, {
@@ -23,8 +51,8 @@
"duration" : 57,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1430917380893,
"endTimeEpoch" : 1430917380950,
+ "startTimeEpoch" : 1430917380893,
"lastUpdatedEpoch" : 0
}, {
"attemptId" : "1",
@@ -34,8 +62,8 @@
"duration" : 10,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1430917380880,
"endTimeEpoch" : 1430917380890,
+ "startTimeEpoch" : 1430917380880,
"lastUpdatedEpoch" : 0
} ]
}, {
@@ -49,8 +77,8 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1426633910242,
"endTimeEpoch" : 1426633945177,
+ "startTimeEpoch" : 1426633910242,
"lastUpdatedEpoch" : 0
}, {
"attemptId" : "1",
@@ -60,8 +88,8 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1426533910242,
"endTimeEpoch" : 1426533945177,
+ "startTimeEpoch" : 1426533910242,
"lastUpdatedEpoch" : 0
} ]
}, {
@@ -74,8 +102,8 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1425081758277,
"endTimeEpoch" : 1425081766912,
+ "startTimeEpoch" : 1425081758277,
"lastUpdatedEpoch" : 0
} ]
} ]
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
index f486d46..e8ed96d 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
@@ -8,8 +8,8 @@
"duration" : 9011,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1422981779720,
"endTimeEpoch" : 1422981788731,
+ "startTimeEpoch" : 1422981779720,
"lastUpdatedEpoch" : 0
} ]
}
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
index e63039f..88c6015 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
@@ -9,8 +9,8 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1426633910242,
"endTimeEpoch" : 1426633945177,
+ "startTimeEpoch" : 1426633910242,
"lastUpdatedEpoch" : 0
}, {
"attemptId" : "1",
@@ -20,8 +20,8 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
- "startTimeEpoch" : 1426533910242,
"endTimeEpoch" : 1426533945177,
+ "startTimeEpoch" : 1426533910242,
"lastUpdatedEpoch" : 0
} ]
}
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
index 0084339..cdebb5f 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
@@ -68,9 +68,9 @@
}
}
},
- "11" : {
- "taskId" : 11,
- "index" : 3,
+ "9" : {
+ "taskId" : 9,
+ "index" : 1,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.830GMT",
"executorId" : "<driver>",
@@ -79,13 +79,13 @@
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 2,
+ "executorDeserializeTime" : 1,
"executorDeserializeCpuTime" : 0,
- "executorRunTime" : 434,
+ "executorRunTime" : 436,
"executorCpuTime" : 0,
"resultSize" : 1902,
"jvmGcTime" : 19,
- "resultSerializationTime" : 1,
+ "resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -105,17 +105,17 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 1647,
- "writeTime" : 83000,
+ "bytesWritten" : 1648,
+ "writeTime" : 98000,
"recordsWritten" : 0
}
}
},
- "14" : {
- "taskId" : 14,
- "index" : 6,
+ "10" : {
+ "taskId" : 10,
+ "index" : 2,
"attempt" : 0,
- "launchTime" : "2015-02-03T16:43:05.832GMT",
+ "launchTime" : "2015-02-03T16:43:05.830GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -149,16 +149,16 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
- "writeTime" : 88000,
+ "writeTime" : 76000,
"recordsWritten" : 0
}
}
},
- "13" : {
- "taskId" : 13,
- "index" : 5,
+ "11" : {
+ "taskId" : 11,
+ "index" : 3,
"attempt" : 0,
- "launchTime" : "2015-02-03T16:43:05.831GMT",
+ "launchTime" : "2015-02-03T16:43:05.830GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -171,7 +171,7 @@
"executorCpuTime" : 0,
"resultSize" : 1902,
"jvmGcTime" : 19,
- "resultSerializationTime" : 2,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -191,17 +191,17 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 1648,
- "writeTime" : 73000,
+ "bytesWritten" : 1647,
+ "writeTime" : 83000,
"recordsWritten" : 0
}
}
},
- "10" : {
- "taskId" : 10,
- "index" : 2,
+ "12" : {
+ "taskId" : 12,
+ "index" : 4,
"attempt" : 0,
- "launchTime" : "2015-02-03T16:43:05.830GMT",
+ "launchTime" : "2015-02-03T16:43:05.831GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -234,30 +234,30 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 1648,
- "writeTime" : 76000,
+ "bytesWritten" : 1645,
+ "writeTime" : 101000,
"recordsWritten" : 0
}
}
},
- "9" : {
- "taskId" : 9,
- "index" : 1,
+ "13" : {
+ "taskId" : 13,
+ "index" : 5,
"attempt" : 0,
- "launchTime" : "2015-02-03T16:43:05.830GMT",
+ "launchTime" : "2015-02-03T16:43:05.831GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 1,
+ "executorDeserializeTime" : 2,
"executorDeserializeCpuTime" : 0,
- "executorRunTime" : 436,
+ "executorRunTime" : 434,
"executorCpuTime" : 0,
"resultSize" : 1902,
"jvmGcTime" : 19,
- "resultSerializationTime" : 0,
+ "resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -278,16 +278,16 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
- "writeTime" : 98000,
+ "writeTime" : 73000,
"recordsWritten" : 0
}
}
},
- "12" : {
- "taskId" : 12,
- "index" : 4,
+ "14" : {
+ "taskId" : 14,
+ "index" : 6,
"attempt" : 0,
- "launchTime" : "2015-02-03T16:43:05.831GMT",
+ "launchTime" : "2015-02-03T16:43:05.832GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -320,8 +320,8 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 1645,
- "writeTime" : 101000,
+ "bytesWritten" : 1648,
+ "writeTime" : 88000,
"recordsWritten" : 0
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
index 63fe3b2..85dafcf 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
@@ -68,9 +68,9 @@
}
}
},
- "11" : {
- "taskId" : 11,
- "index" : 3,
+ "9" : {
+ "taskId" : 9,
+ "index" : 1,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.830GMT",
"executorId" : "<driver>",
@@ -79,13 +79,13 @@
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 2,
+ "executorDeserializeTime" : 1,
"executorDeserializeCpuTime" : 0,
- "executorRunTime" : 434,
+ "executorRunTime" : 436,
"executorCpuTime" : 0,
"resultSize" : 1902,
"jvmGcTime" : 19,
- "resultSerializationTime" : 1,
+ "resultSerializationTime" : 0,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -105,17 +105,17 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 1647,
- "writeTime" : 83000,
+ "bytesWritten" : 1648,
+ "writeTime" : 98000,
"recordsWritten" : 0
}
}
},
- "14" : {
- "taskId" : 14,
- "index" : 6,
+ "10" : {
+ "taskId" : 10,
+ "index" : 2,
"attempt" : 0,
- "launchTime" : "2015-02-03T16:43:05.832GMT",
+ "launchTime" : "2015-02-03T16:43:05.830GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -149,16 +149,16 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
- "writeTime" : 88000,
+ "writeTime" : 76000,
"recordsWritten" : 0
}
}
},
- "13" : {
- "taskId" : 13,
- "index" : 5,
+ "11" : {
+ "taskId" : 11,
+ "index" : 3,
"attempt" : 0,
- "launchTime" : "2015-02-03T16:43:05.831GMT",
+ "launchTime" : "2015-02-03T16:43:05.830GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -171,7 +171,7 @@
"executorCpuTime" : 0,
"resultSize" : 1902,
"jvmGcTime" : 19,
- "resultSerializationTime" : 2,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -191,17 +191,17 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 1648,
- "writeTime" : 73000,
+ "bytesWritten" : 1647,
+ "writeTime" : 83000,
"recordsWritten" : 0
}
}
},
- "10" : {
- "taskId" : 10,
- "index" : 2,
+ "12" : {
+ "taskId" : 12,
+ "index" : 4,
"attempt" : 0,
- "launchTime" : "2015-02-03T16:43:05.830GMT",
+ "launchTime" : "2015-02-03T16:43:05.831GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -234,30 +234,30 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 1648,
- "writeTime" : 76000,
+ "bytesWritten" : 1645,
+ "writeTime" : 101000,
"recordsWritten" : 0
}
}
},
- "9" : {
- "taskId" : 9,
- "index" : 1,
+ "13" : {
+ "taskId" : 13,
+ "index" : 5,
"attempt" : 0,
- "launchTime" : "2015-02-03T16:43:05.830GMT",
+ "launchTime" : "2015-02-03T16:43:05.831GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
"taskMetrics" : {
- "executorDeserializeTime" : 1,
+ "executorDeserializeTime" : 2,
"executorDeserializeCpuTime" : 0,
- "executorRunTime" : 436,
+ "executorRunTime" : 434,
"executorCpuTime" : 0,
"resultSize" : 1902,
"jvmGcTime" : 19,
- "resultSerializationTime" : 0,
+ "resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -278,16 +278,16 @@
},
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
- "writeTime" : 98000,
+ "writeTime" : 73000,
"recordsWritten" : 0
}
}
},
- "12" : {
- "taskId" : 12,
- "index" : 4,
+ "14" : {
+ "taskId" : 14,
+ "index" : 6,
"attempt" : 0,
- "launchTime" : "2015-02-03T16:43:05.831GMT",
+ "launchTime" : "2015-02-03T16:43:05.832GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -320,8 +320,8 @@
"recordsRead" : 0
},
"shuffleWriteMetrics" : {
- "bytesWritten" : 1645,
- "writeTime" : 101000,
+ "bytesWritten" : 1648,
+ "writeTime" : 88000,
"recordsWritten" : 0
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
index 5dcbc89..0ed609d 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
@@ -3,7 +3,7 @@
"executorDeserializeTime" : [ 1.0, 3.0, 36.0 ],
"executorDeserializeCpuTime" : [ 0.0, 0.0, 0.0 ],
"executorRunTime" : [ 16.0, 28.0, 351.0 ],
- "executorCpuTime" : [ 0.0, 0.0, 0.0],
+ "executorCpuTime" : [ 0.0, 0.0, 0.0 ],
"resultSize" : [ 2010.0, 2065.0, 2065.0 ],
"jvmGcTime" : [ 0.0, 0.0, 7.0 ],
"resultSerializationTime" : [ 0.0, 0.0, 2.0 ],
http://git-wip-us.apache.org/repos/asf/spark/blob/640f9423/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
----------------------------------------------------------------------
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
index aaeef1f..c2ece6f 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
@@ -29,11 +29,11 @@
"value" : "5050"
} ],
"tasks" : {
- "2" : {
- "taskId" : 2,
- "index" : 2,
+ "0" : {
+ "taskId" : 0,
+ "index" : 0,
"attempt" : 0,
- "launchTime" : "2015-03-16T19:25:36.522GMT",
+ "launchTime" : "2015-03-16T19:25:36.515GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -41,11 +41,11 @@
"accumulatorUpdates" : [ {
"id" : 1,
"name" : "my counter",
- "update" : "378",
- "value" : "378"
+ "update" : "78",
+ "value" : "5050"
} ],
"taskMetrics" : {
- "executorDeserializeTime" : 13,
+ "executorDeserializeTime" : 14,
"executorDeserializeCpuTime" : 0,
"executorRunTime" : 15,
"executorCpuTime" : 0,
@@ -77,11 +77,11 @@
}
}
},
- "5" : {
- "taskId" : 5,
- "index" : 5,
+ "1" : {
+ "taskId" : 1,
+ "index" : 1,
"attempt" : 0,
- "launchTime" : "2015-03-16T19:25:36.523GMT",
+ "launchTime" : "2015-03-16T19:25:36.521GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -89,11 +89,11 @@
"accumulatorUpdates" : [ {
"id" : 1,
"name" : "my counter",
- "update" : "897",
- "value" : "3750"
+ "update" : "247",
+ "value" : "2175"
} ],
"taskMetrics" : {
- "executorDeserializeTime" : 12,
+ "executorDeserializeTime" : 14,
"executorDeserializeCpuTime" : 0,
"executorRunTime" : 15,
"executorCpuTime" : 0,
@@ -125,9 +125,9 @@
}
}
},
- "4" : {
- "taskId" : 4,
- "index" : 4,
+ "2" : {
+ "taskId" : 2,
+ "index" : 2,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.522GMT",
"executorId" : "<driver>",
@@ -137,17 +137,17 @@
"accumulatorUpdates" : [ {
"id" : 1,
"name" : "my counter",
- "update" : "678",
- "value" : "2853"
+ "update" : "378",
+ "value" : "378"
} ],
"taskMetrics" : {
- "executorDeserializeTime" : 12,
+ "executorDeserializeTime" : 13,
"executorDeserializeCpuTime" : 0,
"executorRunTime" : 15,
"executorCpuTime" : 0,
"resultSize" : 697,
"jvmGcTime" : 0,
- "resultSerializationTime" : 1,
+ "resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -173,11 +173,11 @@
}
}
},
- "7" : {
- "taskId" : 7,
- "index" : 7,
+ "3" : {
+ "taskId" : 3,
+ "index" : 3,
"attempt" : 0,
- "launchTime" : "2015-03-16T19:25:36.524GMT",
+ "launchTime" : "2015-03-16T19:25:36.522GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -185,11 +185,11 @@
"accumulatorUpdates" : [ {
"id" : 1,
"name" : "my counter",
- "update" : "1222",
- "value" : "4972"
+ "update" : "572",
+ "value" : "950"
} ],
"taskMetrics" : {
- "executorDeserializeTime" : 12,
+ "executorDeserializeTime" : 13,
"executorDeserializeCpuTime" : 0,
"executorRunTime" : 15,
"executorCpuTime" : 0,
@@ -221,11 +221,11 @@
}
}
},
- "1" : {
- "taskId" : 1,
- "index" : 1,
+ "4" : {
+ "taskId" : 4,
+ "index" : 4,
"attempt" : 0,
- "launchTime" : "2015-03-16T19:25:36.521GMT",
+ "launchTime" : "2015-03-16T19:25:36.522GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -233,17 +233,17 @@
"accumulatorUpdates" : [ {
"id" : 1,
"name" : "my counter",
- "update" : "247",
- "value" : "2175"
+ "update" : "678",
+ "value" : "2853"
} ],
"taskMetrics" : {
- "executorDeserializeTime" : 14,
+ "executorDeserializeTime" : 12,
"executorDeserializeCpuTime" : 0,
"executorRunTime" : 15,
"executorCpuTime" : 0,
"resultSize" : 697,
"jvmGcTime" : 0,
- "resultSerializationTime" : 2,
+ "resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
"diskBytesSpilled" : 0,
"inputMetrics" : {
@@ -269,11 +269,11 @@
}
}
},
- "3" : {
- "taskId" : 3,
- "index" : 3,
+ "5" : {
+ "taskId" : 5,
+ "index" : 5,
"attempt" : 0,
- "launchTime" : "2015-03-16T19:25:36.522GMT",
+ "launchTime" : "2015-03-16T19:25:36.523GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -281,11 +281,11 @@
"accumulatorUpdates" : [ {
"id" : 1,
"name" : "my counter",
- "update" : "572",
- "value" : "950"
+ "update" : "897",
+ "value" : "3750"
} ],
"taskMetrics" : {
- "executorDeserializeTime" : 13,
+ "executorDeserializeTime" : 12,
"executorDeserializeCpuTime" : 0,
"executorRunTime" : 15,
"executorCpuTime" : 0,
@@ -365,11 +365,11 @@
}
}
},
- "0" : {
- "taskId" : 0,
- "index" : 0,
+ "7" : {
+ "taskId" : 7,
+ "index" : 7,
"attempt" : 0,
- "launchTime" : "2015-03-16T19:25:36.515GMT",
+ "launchTime" : "2015-03-16T19:25:36.524GMT",
"executorId" : "<driver>",
"host" : "localhost",
"taskLocality" : "PROCESS_LOCAL",
@@ -377,11 +377,11 @@
"accumulatorUpdates" : [ {
"id" : 1,
"name" : "my counter",
- "update" : "78",
- "value" : "5050"
+ "update" : "1222",
+ "value" : "4972"
} ],
"taskMetrics" : {
- "executorDeserializeTime" : 14,
+ "executorDeserializeTime" : 12,
"executorDeserializeCpuTime" : 0,
"executorRunTime" : 15,
"executorCpuTime" : 0,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org