You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2020/05/14 03:11:28 UTC
[spark] branch branch-2.4 updated: [SPARK-31632][CORE][WEBUI]
Enrich the exception message when application information is unavailable
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 4ba9421 [SPARK-31632][CORE][WEBUI] Enrich the exception message when application information is unavailable
4ba9421 is described below
commit 4ba9421a2a1fae57ea21a8167d31c8fd4858aee4
Author: Xingcan Cui <xc...@apache.org>
AuthorDate: Thu May 14 12:07:22 2020 +0900
[SPARK-31632][CORE][WEBUI] Enrich the exception message when application information is unavailable
### What changes were proposed in this pull request?
This PR caught the `NoSuchElementException` and enriched the error message for `AppStatusStore.applicationInfo()` when Spark is starting up and the application information is unavailable.
### Why are the changes needed?
During the initialization of `SparkContext`, it first starts the Web UI and then set up the `LiveListenerBus` thread for dispatching the `SparkListenerApplicationStart` event (which will trigger writing the requested `ApplicationInfo` to `InMemoryStore`). If the Web UI is accessed before this info's being written to `InMemoryStore`, the following `NoSuchElementException` will be thrown.
```
WARN org.eclipse.jetty.server.HttpChannel: /jobs/
java.util.NoSuchElementException
at java.util.Collections$EmptyIterator.next(Collections.java:4191)
at org.apache.spark.util.kvstore.InMemoryStore$InMemoryIterator.next(InMemoryStore.java:467)
at org.apache.spark.status.AppStatusStore.applicationInfo(AppStatusStore.scala:39)
at org.apache.spark.ui.jobs.AllJobsPage.render(AllJobsPage.scala:266)
at org.apache.spark.ui.WebUI.$anonfun$attachPage$1(WebUI.scala:89)
at org.apache.spark.ui.JettyUtils$$anon$1.doGet(JettyUtils.scala:80)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:687)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:790)
at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:873)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1623)
at org.apache.spark.ui.HttpSecurityFilter.doFilter(HttpSecurityFilter.scala:95)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1610)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:540)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:255)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1345)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:203)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:480)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:201)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1247)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:144)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:753)
at org.eclipse.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:220)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.eclipse.jetty.server.Server.handle(Server.java:505)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:370)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:267)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:305)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:103)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:117)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:698)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:804)
at java.lang.Thread.run(Thread.java:748)
```
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Manually tested.
This can be reproduced:
1. `./bin/spark-shell`
2. at the same time, open `http://localhost:4040/jobs/` in your browser with quickly refreshing.
Closes #28444 from xccui/SPARK-31632.
Authored-by: Xingcan Cui <xc...@apache.org>
Signed-off-by: HyukjinKwon <gu...@apache.org>
(cherry picked from commit 42951e6786319481220ba4abfad015a8d11749f3)
Signed-off-by: HyukjinKwon <gu...@apache.org>
---
.../main/scala/org/apache/spark/status/AppStatusStore.scala | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
index 6568f5f..e2086d6 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStatusStore.scala
@@ -21,7 +21,7 @@ import java.util.{List => JList}
import scala.collection.JavaConverters._
-import org.apache.spark.{JobExecutionStatus, SparkConf}
+import org.apache.spark.{JobExecutionStatus, SparkConf, SparkException}
import org.apache.spark.status.api.v1
import org.apache.spark.ui.scope._
import org.apache.spark.util.{Distribution, Utils}
@@ -35,7 +35,14 @@ private[spark] class AppStatusStore(
val listener: Option[AppStatusListener] = None) {
def applicationInfo(): v1.ApplicationInfo = {
- store.view(classOf[ApplicationInfoWrapper]).max(1).iterator().next().info
+ try {
+ // The ApplicationInfo may not be available when Spark is starting up.
+ store.view(classOf[ApplicationInfoWrapper]).max(1).iterator().next().info
+ } catch {
+ case _: NoSuchElementException =>
+ throw new SparkException("Failed to get the application information. " +
+ "If you are starting up Spark, please wait a while until it's ready.")
+ }
}
def environmentInfo(): v1.ApplicationEnvironmentInfo = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org