You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2016/09/29 19:47:31 UTC
spark git commit: [SPARK-17672] Spark 2.0 history server web Ui takes
too long for a single application
Repository: spark
Updated Branches:
refs/heads/master 7f779e743 -> cb87b3ced
[SPARK-17672] Spark 2.0 history server web Ui takes too long for a single application
Added a new API getApplicationInfo(appId: String) in class ApplicationHistoryProvider and class SparkUI to get app info. In this change, FsHistoryProvider can directly fetch one app info in O(1) time complexity compared to O(n) before the change which used an Iterator.find() interface.
Both ApplicationCache and OneApplicationResource classes adopt this new api.
manual tests
Author: Gang Wu <wg...@uber.com>
Closes #15247 from wgtmac/SPARK-17671.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cb87b3ce
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cb87b3ce
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cb87b3ce
Branch: refs/heads/master
Commit: cb87b3ced9453b5717fa8e8637b97a2f3f25fdd7
Parents: 7f779e7
Author: Gang Wu <wg...@uber.com>
Authored: Thu Sep 29 15:51:05 2016 -0400
Committer: Andrew Or <an...@gmail.com>
Committed: Thu Sep 29 15:51:38 2016 -0400
----------------------------------------------------------------------
.../spark/deploy/history/ApplicationHistoryProvider.scala | 5 +++++
.../org/apache/spark/deploy/history/FsHistoryProvider.scala | 4 ++++
.../scala/org/apache/spark/deploy/history/HistoryServer.scala | 4 ++++
.../scala/org/apache/spark/status/api/v1/ApiRootResource.scala | 1 +
.../org/apache/spark/status/api/v1/OneApplicationResource.scala | 2 +-
core/src/main/scala/org/apache/spark/ui/SparkUI.scala | 4 ++++
6 files changed, 19 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/cb87b3ce/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
index 44661ed..ba42b48 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
@@ -109,4 +109,9 @@ private[history] abstract class ApplicationHistoryProvider {
@throws(classOf[SparkException])
def writeEventLogs(appId: String, attemptId: Option[String], zipStream: ZipOutputStream): Unit
+ /**
+ * @return the [[ApplicationHistoryInfo]] for the appId if it exists.
+ */
+ def getApplicationInfo(appId: String): Option[ApplicationHistoryInfo]
+
}
http://git-wip-us.apache.org/repos/asf/spark/blob/cb87b3ce/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index 6874aa5..d494ff0 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -224,6 +224,10 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
override def getListing(): Iterable[FsApplicationHistoryInfo] = applications.values
+ override def getApplicationInfo(appId: String): Option[FsApplicationHistoryInfo] = {
+ applications.get(appId)
+ }
+
override def getAppUI(appId: String, attemptId: Option[String]): Option[LoadedAppUI] = {
try {
applications.get(appId).flatMap { appInfo =>
http://git-wip-us.apache.org/repos/asf/spark/blob/cb87b3ce/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index c178917..735aa43 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -182,6 +182,10 @@ class HistoryServer(
getApplicationList().iterator.map(ApplicationsListResource.appHistoryInfoToPublicAppInfo)
}
+ def getApplicationInfo(appId: String): Option[ApplicationInfo] = {
+ provider.getApplicationInfo(appId).map(ApplicationsListResource.appHistoryInfoToPublicAppInfo)
+ }
+
override def writeEventLogs(
appId: String,
attemptId: Option[String],
http://git-wip-us.apache.org/repos/asf/spark/blob/cb87b3ce/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
index de92711..17bc043 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala
@@ -222,6 +222,7 @@ private[spark] object ApiRootResource {
private[spark] trait UIRoot {
def getSparkUI(appKey: String): Option[SparkUI]
def getApplicationInfoList: Iterator[ApplicationInfo]
+ def getApplicationInfo(appId: String): Option[ApplicationInfo]
/**
* Write the event logs for the given app to the [[ZipOutputStream]] instance. If attemptId is
http://git-wip-us.apache.org/repos/asf/spark/blob/cb87b3ce/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala
index d7e6a8b..18c3e2f 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/OneApplicationResource.scala
@@ -24,7 +24,7 @@ private[v1] class OneApplicationResource(uiRoot: UIRoot) {
@GET
def getApp(@PathParam("appId") appId: String): ApplicationInfo = {
- val apps = uiRoot.getApplicationInfoList.find { _.id == appId }
+ val apps = uiRoot.getApplicationInfo(appId)
apps.getOrElse(throw new NotFoundException("unknown app: " + appId))
}
http://git-wip-us.apache.org/repos/asf/spark/blob/cb87b3ce/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
index 39155ff..ef71db8 100644
--- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -126,6 +126,10 @@ private[spark] class SparkUI private (
))
))
}
+
+ def getApplicationInfo(appId: String): Option[ApplicationInfo] = {
+ getApplicationInfoList.find(_.id == appId)
+ }
}
private[spark] abstract class SparkUITab(parent: SparkUI, prefix: String)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org