You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/11/05 23:47:55 UTC
[1/4] git commit: [SPARK-4168][WebUI] web statges number should show
correctly when stages are more than 1000
Repository: spark
Updated Branches:
refs/heads/branch-1.2 866c7bbe5 -> b27d7dcaa
[SPARK-4168][WebUI] web statges number should show correctly when stages are more than 1000
The number of completed stages and failed stages showed on webUI will always be less than 1000. This is really misleading when there are already thousands of stages completed or failed. The number should be correct even when only partial stages listed on the webUI (stage info will be removed if the number is too large).
Author: Zhang, Liye <li...@intel.com>
Closes #3035 from liyezhang556520/webStageNum and squashes the following commits:
d9e29fb [Zhang, Liye] add detailed comments for variables
4ea8fd1 [Zhang, Liye] change variable name accroding to comments
f4c404d [Zhang, Liye] [SPARK-4168][WebUI] web statges number should show correctly when stages are more than 1000
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7517c37a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7517c37a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7517c37a
Branch: refs/heads/branch-1.2
Commit: 7517c37aee373c8bd3ccbf1eae079b0fc6b89c91
Parents: 866c7bb
Author: Zhang, Liye <li...@intel.com>
Authored: Mon Nov 3 18:17:32 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Wed Nov 5 14:45:59 2014 -0800
----------------------------------------------------------------------
.../org/apache/spark/ui/jobs/JobProgressListener.scala | 9 +++++++++
.../scala/org/apache/spark/ui/jobs/JobProgressPage.scala | 10 ++++++----
2 files changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/7517c37a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
index b520736..e322340 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala
@@ -59,6 +59,13 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
val failedStages = ListBuffer[StageInfo]()
val stageIdToData = new HashMap[(StageId, StageAttemptId), StageUIData]
val stageIdToInfo = new HashMap[StageId, StageInfo]
+
+ // Number of completed and failed stages, may not actually equal to completedStages.size and
+ // failedStages.size respectively due to completedStage and failedStages only maintain the latest
+ // part of the stages, the earlier ones will be removed when there are too many stages for
+ // memory sake.
+ var numCompletedStages = 0
+ var numFailedStages = 0
// Map from pool name to a hash map (map from stage id to StageInfo).
val poolToActiveStages = HashMap[String, HashMap[Int, StageInfo]]()
@@ -110,9 +117,11 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
activeStages.remove(stage.stageId)
if (stage.failureReason.isEmpty) {
completedStages += stage
+ numCompletedStages += 1
trimIfNecessary(completedStages)
} else {
failedStages += stage
+ numFailedStages += 1
trimIfNecessary(failedStages)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/7517c37a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala
index 6e718ee..83a7898 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobProgressPage.scala
@@ -34,7 +34,9 @@ private[ui] class JobProgressPage(parent: JobProgressTab) extends WebUIPage("")
listener.synchronized {
val activeStages = listener.activeStages.values.toSeq
val completedStages = listener.completedStages.reverse.toSeq
+ val numCompletedStages = listener.numCompletedStages
val failedStages = listener.failedStages.reverse.toSeq
+ val numFailedStages = listener.numFailedStages
val now = System.currentTimeMillis
val activeStagesTable =
@@ -69,11 +71,11 @@ private[ui] class JobProgressPage(parent: JobProgressTab) extends WebUIPage("")
</li>
<li>
<a href="#completed"><strong>Completed Stages:</strong></a>
- {completedStages.size}
+ {numCompletedStages}
</li>
<li>
<a href="#failed"><strong>Failed Stages:</strong></a>
- {failedStages.size}
+ {numFailedStages}
</li>
</ul>
</div>
@@ -86,9 +88,9 @@ private[ui] class JobProgressPage(parent: JobProgressTab) extends WebUIPage("")
}} ++
<h4 id="active">Active Stages ({activeStages.size})</h4> ++
activeStagesTable.toNodeSeq ++
- <h4 id="completed">Completed Stages ({completedStages.size})</h4> ++
+ <h4 id="completed">Completed Stages ({numCompletedStages})</h4> ++
completedStagesTable.toNodeSeq ++
- <h4 id ="failed">Failed Stages ({failedStages.size})</h4> ++
+ <h4 id ="failed">Failed Stages ({numFailedStages})</h4> ++
failedStagesTable.toNodeSeq
UIUtils.headerSparkPage("Spark Stages", content, parent)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[2/4] git commit: [SPARK-4163][Core] Add a backward compatibility
test for FetchFailed
Posted by pw...@apache.org.
[SPARK-4163][Core] Add a backward compatibility test for FetchFailed
/cc aarondav
Author: zsxwing <zs...@gmail.com>
Closes #3086 from zsxwing/SPARK-4163-back-comp and squashes the following commits:
21cb2a8 [zsxwing] Add a backward compatibility test for FetchFailed
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e0a043b7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e0a043b7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e0a043b7
Branch: refs/heads/branch-1.2
Commit: e0a043b79c250515a680485f0dc7b1a149835445
Parents: 7517c37
Author: zsxwing <zs...@gmail.com>
Authored: Mon Nov 3 22:40:43 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Wed Nov 5 14:46:46 2014 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/util/JsonProtocolSuite.scala | 11 +++++++++++
1 file changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e0a043b7/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index a91c9dd..0103012 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -177,6 +177,17 @@ class JsonProtocolSuite extends FunSuite {
deserializedBmRemoved)
}
+ test("FetchFailed backwards compatibility") {
+ // FetchFailed in Spark 1.1.0 does not have an "Message" property.
+ val fetchFailed = FetchFailed(BlockManagerId("With or", "without you", 15), 17, 18, 19,
+ "ignored")
+ val oldEvent = JsonProtocol.taskEndReasonToJson(fetchFailed)
+ .removeField({ _._1 == "Message" })
+ val expectedFetchFailed = FetchFailed(BlockManagerId("With or", "without you", 15), 17, 18, 19,
+ "Unknown reason")
+ assert(expectedFetchFailed === JsonProtocol.taskEndReasonFromJson(oldEvent))
+ }
+
test("SparkListenerApplicationStart backwards compatibility") {
// SparkListenerApplicationStart in Spark 1.0.0 do not have an "appId" property.
val applicationStart = SparkListenerApplicationStart("test", None, 1L, "user")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[4/4] git commit: [EC2] Factor out Mesos spark-ec2 branch
Posted by pw...@apache.org.
[EC2] Factor out Mesos spark-ec2 branch
We reference a specific branch in two places. This patch makes it one place.
Author: Nicholas Chammas <ni...@gmail.com>
Closes #3008 from nchammas/mesos-spark-ec2-branch and squashes the following commits:
10a6089 [Nicholas Chammas] factor out mess spark-ec2 branch
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b27d7dca
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b27d7dca
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b27d7dca
Branch: refs/heads/branch-1.2
Commit: b27d7dcaaad0bf04d341660ffbeb742cd4eecfd3
Parents: 68be37b
Author: Nicholas Chammas <ni...@gmail.com>
Authored: Mon Nov 3 09:02:35 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Wed Nov 5 14:47:29 2014 -0800
----------------------------------------------------------------------
ec2/spark_ec2.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b27d7dca/ec2/spark_ec2.py
----------------------------------------------------------------------
diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py
index 0d6b82b..50f88f7 100755
--- a/ec2/spark_ec2.py
+++ b/ec2/spark_ec2.py
@@ -41,8 +41,9 @@ from boto import ec2
DEFAULT_SPARK_VERSION = "1.1.0"
+MESOS_SPARK_EC2_BRANCH = "v4"
# A URL prefix from which to fetch AMI information
-AMI_PREFIX = "https://raw.github.com/mesos/spark-ec2/v2/ami-list"
+AMI_PREFIX = "https://raw.github.com/mesos/spark-ec2/{b}/ami-list".format(b=MESOS_SPARK_EC2_BRANCH)
class UsageError(Exception):
@@ -583,7 +584,13 @@ def setup_cluster(conn, master_nodes, slave_nodes, opts, deploy_ssh_key):
# NOTE: We should clone the repository before running deploy_files to
# prevent ec2-variables.sh from being overwritten
- ssh(master, opts, "rm -rf spark-ec2 && git clone https://github.com/mesos/spark-ec2.git -b v4")
+ ssh(
+ host=master,
+ opts=opts,
+ command="rm -rf spark-ec2"
+ + " && "
+ + "git clone https://github.com/mesos/spark-ec2.git -b {b}".format(b=MESOS_SPARK_EC2_BRANCH)
+ )
print "Deploying files to master..."
deploy_files(conn, "deploy.generic", opts, master_nodes, slave_nodes, modules)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[3/4] git commit: [SPARK-4166][Core] Add a backward compatibility
test for ExecutorLostFailure
Posted by pw...@apache.org.
[SPARK-4166][Core] Add a backward compatibility test for ExecutorLostFailure
Author: zsxwing <zs...@gmail.com>
Closes #3085 from zsxwing/SPARK-4166-back-comp and squashes the following commits:
89329f4 [zsxwing] Add a backward compatibility test for ExecutorLostFailure
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/68be37b8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/68be37b8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/68be37b8
Branch: refs/heads/branch-1.2
Commit: 68be37b823516dbeda066776bb060bf894db4e95
Parents: e0a043b
Author: zsxwing <zs...@gmail.com>
Authored: Mon Nov 3 22:47:45 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Wed Nov 5 14:46:54 2014 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/util/JsonProtocolSuite.scala | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/68be37b8/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index 0103012..aec1e40 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -196,6 +196,15 @@ class JsonProtocolSuite extends FunSuite {
assert(applicationStart === JsonProtocol.applicationStartFromJson(oldEvent))
}
+ test("ExecutorLostFailure backward compatibility") {
+ // ExecutorLostFailure in Spark 1.1.0 does not have an "Executor ID" property.
+ val executorLostFailure = ExecutorLostFailure("100")
+ val oldEvent = JsonProtocol.taskEndReasonToJson(executorLostFailure)
+ .removeField({ _._1 == "Executor ID" })
+ val expectedExecutorLostFailure = ExecutorLostFailure("Unknown")
+ assert(expectedExecutorLostFailure === JsonProtocol.taskEndReasonFromJson(oldEvent))
+ }
+
/** -------------------------- *
| Helper test running methods |
* --------------------------- */
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org