You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by YanTangZhai <gi...@git.apache.org> on 2014/08/08 13:42:54 UTC
[GitHub] spark pull request: [Spark 2643] Stages web ui has ERROR when pool...
GitHub user YanTangZhai opened a pull request:
https://github.com/apache/spark/pull/1854
[Spark 2643] Stages web ui has ERROR when pool name is None
14/07/23 16:01:44 WARN servlet.ServletHandler: /stages/
java.util.NoSuchElementException: None.get
at scala.None$.get(Option.scala:313)
at scala.None$.get(Option.scala:311)
at org.apache.spark.ui.jobs.StageTableBase.stageRow(StageTable.scala:132)
at org.apache.spark.ui.jobs.StageTableBase.org$apache$spark$ui$jobs$StageTableBase$$renderStageRow(StageTable.scala:150)
at org.apache.spark.ui.jobs.StageTableBase$$anonfun$toNodeSeq$1.apply(StageTable.scala:52)
at org.apache.spark.ui.jobs.StageTableBase$$anonfun$toNodeSeq$1.apply(StageTable.scala:52)
at org.apache.spark.ui.jobs.StageTableBase$$anonfun$stageTable$1.apply(StageTable.scala:61)
at org.apache.spark.ui.jobs.StageTableBase$$anonfun$stageTable$1.apply(StageTable.scala:61)
at scala.collection.immutable.Stream$$anonfun$map$1.apply(Stream.scala:376)
at scala.collection.immutable.Stream$$anonfun$map$1.apply(Stream.scala:376)
at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1085)
at scala.collection.immutable.Stream$Cons.tail(Stream.scala:1077)
at scala.collection.immutable.StreamIterator$$anonfun$next$1.apply(Stream.scala:980)
at scala.collection.immutable.StreamIterator$$anonfun$next$1.apply(Stream.scala:980)
at scala.collection.immutable.StreamIterator$LazyCell.v$lzycompute(Stream.scala:969)
at scala.collection.immutable.StreamIterator$LazyCell.v(Stream.scala:969)
at scala.collection.immutable.StreamIterator.hasNext(Stream.scala:974)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.xml.NodeBuffer.$amp$plus(NodeBuffer.scala:38)
at scala.xml.NodeBuffer.$amp$plus(NodeBuffer.scala:40)
at org.apache.spark.ui.jobs.StageTableBase.stageTable(StageTable.scala:60)
at org.apache.spark.ui.jobs.StageTableBase.toNodeSeq(StageTable.scala:52)
at org.apache.spark.ui.jobs.JobProgressPage.render(JobProgressPage.scala:91)
at org.apache.spark.ui.WebUI$$anonfun$attachPage$1.apply(WebUI.scala:65)
at org.apache.spark.ui.WebUI$$anonfun$attachPage$1.apply(WebUI.scala:65)
at org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:70)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:707)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:501)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:428)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.eclipse.jetty.server.Server.handle(Server.java:370)
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494)
at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:971)
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1033)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:667)
at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:744)
14/07/23 16:01:44 WARN server.AbstractHttpConnection: /stages/
java.lang.NoSuchMethodError: javax.servlet.http.HttpServletRequest.isAsyncStarted()Z
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:583)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:428)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:255)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
at org.eclipse.jetty.server.Server.handle(Server.java:370)
at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494)
at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:971)
at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1033)
at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:644)
at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
at org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)
at org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:667)
at org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
at java.lang.Thread.run(Thread.java:744)
This is because:
(1) One stage for example the stage 10 has been completed and then it's resubmitted.
(2) Then JobProgressListener.trimIfNecessary is triggered by another SparkListenerStageCompleted event. The StageUIData of the stage 10 will be removed from memory although the stage 10 is active at present.
(3) JobProgressPage will not get the StageUIData of the stage 10 any more.
You can merge this pull request into a Git repository by running:
$ git pull https://github.com/YanTangZhai/spark SPARK-2643
Alternatively you can review and apply these changes as the patch at:
https://github.com/apache/spark/pull/1854.patch
To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:
This closes #1854
----
commit cdef539abc5d2d42d4661373939bdd52ca8ee8e6
Author: YanTangZhai <ha...@tencent.com>
Date: 2014-08-06T13:07:08Z
Merge pull request #1 from apache/master
update
commit efd550c2b742bf7036a96cda60c15702de3f0d70
Author: yantangzhai <ty...@163.com>
Date: 2014-08-08T11:30:48Z
[SPARK-2643] Stages web ui has ERROR when pool name is None
----
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by srowen <gi...@git.apache.org>.
Github user srowen commented on a diff in the pull request:
https://github.com/apache/spark/pull/1854#discussion_r15993719
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala ---
@@ -84,8 +84,11 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
private def trimIfNecessary(stages: ListBuffer[StageInfo]) = synchronized {
if (stages.size > retainedStages) {
val toRemove = math.max(retainedStages / 10, 1)
- stages.take(toRemove).foreach { s => stageIdToData.remove(s.stageId) }
- stages.trimStart(toRemove)
+ val notActiveStages = stages.filter(s => !activeStages.contains(s.stageId))
+ notActiveStages.take(toRemove).foreach { s =>
+ stageIdToData.remove(s.stageId)
+ }
+ notActiveStages.trimStart(toRemove)
--- End diff --
This will only return the rest of the non-active stages, rather than all the stages from `stages` that were not chosen to be removed. I can't think of a good one-liner for this!
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-51604210
QA tests have started for PR 1854. This patch merges cleanly. <br>View progress: https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/18202/consoleFull
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by srowen <gi...@git.apache.org>.
Github user srowen commented on a diff in the pull request:
https://github.com/apache/spark/pull/1854#discussion_r15989065
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala ---
@@ -84,7 +84,11 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
private def trimIfNecessary(stages: ListBuffer[StageInfo]) = synchronized {
if (stages.size > retainedStages) {
val toRemove = math.max(retainedStages / 10, 1)
- stages.take(toRemove).foreach { s => stageIdToData.remove(s.stageId) }
+ stages.take(toRemove).foreach { s =>
+ if (!activeStages.contains(s.stageId)) {
+ stageIdToData.remove(s.stageId)
+ }
+ }
stages.trimStart(toRemove)
--- End diff --
Does it matter than the first `toRemove` elements now may not have been removed? I think this has to be consistent with the previous line.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by srowen <gi...@git.apache.org>.
Github user srowen commented on a diff in the pull request:
https://github.com/apache/spark/pull/1854#discussion_r15989111
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala ---
@@ -84,7 +84,11 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
private def trimIfNecessary(stages: ListBuffer[StageInfo]) = synchronized {
if (stages.size > retainedStages) {
val toRemove = math.max(retainedStages / 10, 1)
- stages.take(toRemove).foreach { s => stageIdToData.remove(s.stageId) }
+ stages.take(toRemove).foreach { s =>
+ if (!activeStages.contains(s.stageId)) {
+ stageIdToData.remove(s.stageId)
+ }
+ }
stages.trimStart(toRemove)
--- End diff --
Putting the two comments together, I think that to avoid getting different results when querying `activeStages`, this has to proceed by separating the stages to be removed from the rest, invoking the `foreach` on the former, and then returning the rest.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-51610426
QA results for PR 1854:<br>- This patch FAILED unit tests.<br>- This patch merges cleanly<br>- This patch adds no public classes<br><br>For more information see test ouptut:<br>https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/18202/consoleFull
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by srowen <gi...@git.apache.org>.
Github user srowen commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-51596179
@YanTangZhai Yes, that's fine. The code in this case would just try to remove 10 from `stageIdToData` several times, which is a bit redundant but harmless.
My first comment was that, with this change, if I request 5 elements to be trimmed, it will actually trim nothing. The first 5 elements of `stages` are all 10, and none can be removed because 10 is active. I would more readily expect that the behavior is to remove up to 5 elements that can be removed. 11 and 18 can be removed in this case. If you filter out active, *then* take, you get this behavior.
The second potential issue is that, with this change, the method will return [11,18] as `stages` after 5 values have been trimmed, even though 10 is still active. Is that OK? If the comment above is addressed, then this return value also has to change to reflect it.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by YanTangZhai <gi...@git.apache.org>.
Github user YanTangZhai commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-55254070
@jkbradley I will close this PR. Thank you very much.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by YanTangZhai <gi...@git.apache.org>.
Github user YanTangZhai commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-51597142
@srowen I see, thanks. I will modify.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-51594917
QA results for PR 1854:<br>- This patch PASSES unit tests.<br>- This patch merges cleanly<br>- This patch adds no public classes<br><br>For more information see test ouptut:<br>https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/18201/consoleFull
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-51591214
QA tests have started for PR 1854. This patch merges cleanly. <br>View progress: https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/18201/consoleFull
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-51881015
QA tests have started for PR 1854. This patch merges cleanly. <br>View progress: https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/18364/consoleFull
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by YanTangZhai <gi...@git.apache.org>.
Github user YanTangZhai commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-51604647
Please review again, thanks.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by srowen <gi...@git.apache.org>.
Github user srowen commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-51602933
@YanTangZhai Ah good point. Well, I think your original change is actually OK then after all that. It might be more optimal to try to remove up to N *inactive* stages from `stages` in this method instead of just the first N, all of which may be active. I don't know if it's complicated to do that or what, or whether it ends up being unimportant.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by YanTangZhai <gi...@git.apache.org>.
Github user YanTangZhai commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-51601604
@srowen The stage 10 will be removed from stageIdToData later. Since it will be added into completedStages or failedStages again and will be removed from activeStages when it is completed. Some time later, it will be removed from stageIdToData by trimIfNecessary since it's not in activeStages.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by srowen <gi...@git.apache.org>.
Github user srowen commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-51599157
@YanTangZhai I'm not sure I wrote that clearly. I should say that the behavior you have here could be correct if it's fine that a call to remove some elements merely removes some stuff from `stages` whether they're active or not. But then, I wonder about this problem: here, if I remove 5 elements, and remove all of the 10s from `stages`, and of course nothing from `stageIdToData` since they're active, then will I ever remove 10 from `stageIdToData` when it's not active? because it's already disappeared from `stages` earlier when 10 was active. I think that might be a problem but not sure. Anyway that's another motivation for not removing anything from `stages` unless it is known to be inactive.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by SparkQA <gi...@git.apache.org>.
Github user SparkQA commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-51884838
QA results for PR 1854:<br>- This patch PASSES unit tests.<br>- This patch merges cleanly<br>- This patch adds no public classes<br><br>For more information see test ouptut:<br>https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/18364/consoleFull
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by YanTangZhai <gi...@git.apache.org>.
Github user YanTangZhai closed the pull request at:
https://github.com/apache/spark/pull/1854
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by YanTangZhai <gi...@git.apache.org>.
Github user YanTangZhai commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-51595260
@srowen The completedStages may contains stages as follows ...10, 10, 10, 10, 10, 11, 18... and the activeStages may contains 1, 10, 5 with unique 10 and the stageIdToData may contains ...10, 11, 18... with unique 10. When the completedStages is trimmed, ...10, 10 may be removed whereas the stageIdToData should not remove 11, 18. If the stageIdToData has removed 11, 18, web ui could not show poolname or description of 11, 18 in completed stages, this problem still exists.
Please review again, thanks.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by jkbradley <gi...@git.apache.org>.
Github user jkbradley commented on the pull request:
https://github.com/apache/spark/pull/1854#issuecomment-55061571
@YanTangZhai Hi, I was asked to check out this PR. It looks like it may no longer be necessary since stages are now paired with attempt IDs. Could you please try running the test which failed with the old Spark version, and see if it still fails? Hopefully the issue is fixed, so that the PR can be closed. Thanks very much!
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org
[GitHub] spark pull request: [SPARK-2643] Stages web ui has ERROR when pool...
Posted by srowen <gi...@git.apache.org>.
Github user srowen commented on a diff in the pull request:
https://github.com/apache/spark/pull/1854#discussion_r15989068
--- Diff: core/src/main/scala/org/apache/spark/ui/jobs/JobProgressListener.scala ---
@@ -84,7 +84,11 @@ class JobProgressListener(conf: SparkConf) extends SparkListener with Logging {
private def trimIfNecessary(stages: ListBuffer[StageInfo]) = synchronized {
if (stages.size > retainedStages) {
val toRemove = math.max(retainedStages / 10, 1)
- stages.take(toRemove).foreach { s => stageIdToData.remove(s.stageId) }
--- End diff --
This may result in removing fewer than `toRemove` stages. That may be OK, but how about removing from the list of non-active stages? `stages.filter(s => !activeStages.contains(s.stageId)).take(toRemove)...`
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org