You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jo...@apache.org on 2014/12/23 23:59:28 UTC
spark git commit: [SPARK-4913] Fix incorrect event log path
Repository: spark
Updated Branches:
refs/heads/master 27c5399f4 -> 96281cd0c
[SPARK-4913] Fix incorrect event log path
SPARK-2261 uses a single file to log events for an app. `eventLogDir` in `ApplicationDescription` is replaced with `eventLogFile`. However, `ApplicationDescription` in `SparkDeploySchedulerBackend` is initialized with `SparkContext`'s `eventLogDir`. It is just the log directory, not the actual log file path. `Master.rebuildSparkUI` can not correctly rebuild a new SparkUI for the app.
Because the `ApplicationDescription` is remotely registered with `Master` and the app's id is then generated in `Master`, we can not get the app id in advance before registration. So the received description needs to be modified with correct `eventLogFile` value.
Author: Liang-Chi Hsieh <vi...@gmail.com>
Closes #3755 from viirya/fix_app_logdir and squashes the following commits:
5e0ea35 [Liang-Chi Hsieh] Revision for comment.
b5730a1 [Liang-Chi Hsieh] Fix incorrect event log path.
Closes #3777 (a duplicate PR for the same JIRA)
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/96281cd0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/96281cd0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/96281cd0
Branch: refs/heads/master
Commit: 96281cd0c3ffb4c0fbdb64b8f7b705f863d46b33
Parents: 27c5399
Author: Liang-Chi Hsieh <vi...@gmail.com>
Authored: Tue Dec 23 14:58:33 2014 -0800
Committer: Josh Rosen <jo...@databricks.com>
Committed: Tue Dec 23 14:58:44 2014 -0800
----------------------------------------------------------------------
.../spark/deploy/ApplicationDescription.scala | 2 +-
.../org/apache/spark/deploy/master/Master.scala | 20 ++++++++++++++++----
2 files changed, 17 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/96281cd0/core/src/main/scala/org/apache/spark/deploy/ApplicationDescription.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/ApplicationDescription.scala b/core/src/main/scala/org/apache/spark/deploy/ApplicationDescription.scala
index b10b7b8..65a1a8f 100644
--- a/core/src/main/scala/org/apache/spark/deploy/ApplicationDescription.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/ApplicationDescription.scala
@@ -23,7 +23,7 @@ private[spark] class ApplicationDescription(
val memoryPerSlave: Int,
val command: Command,
var appUiUrl: String,
- val eventLogFile: Option[String] = None)
+ val eventLogDir: Option[String] = None)
extends Serializable {
val user = System.getProperty("user.name", "<unknown>")
http://git-wip-us.apache.org/repos/asf/spark/blob/96281cd0/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index f8137bf..81fa077 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -719,14 +719,26 @@ private[spark] class Master(
def rebuildSparkUI(app: ApplicationInfo): Boolean = {
val appName = app.desc.name
val notFoundBasePath = HistoryServer.UI_PATH_PREFIX + "/not-found"
- val eventLogFile = app.desc.eventLogFile.getOrElse {
- // Event logging is not enabled for this application
- app.desc.appUiUrl = notFoundBasePath
+ val eventLogFile = app.desc.eventLogDir
+ .map { dir => EventLoggingListener.getLogPath(dir, app.id) }
+ .getOrElse {
+ // Event logging is not enabled for this application
+ app.desc.appUiUrl = notFoundBasePath
+ return false
+ }
+ val fs = Utils.getHadoopFileSystem(eventLogFile, hadoopConf)
+
+ if (fs.exists(new Path(eventLogFile + EventLoggingListener.IN_PROGRESS))) {
+ // Event logging is enabled for this application, but the application is still in progress
+ val title = s"Application history not found (${app.id})"
+ var msg = s"Application $appName is still in progress."
+ logWarning(msg)
+ msg = URLEncoder.encode(msg, "UTF-8")
+ app.desc.appUiUrl = notFoundBasePath + s"?msg=$msg&title=$title"
return false
}
try {
- val fs = Utils.getHadoopFileSystem(eventLogFile, hadoopConf)
val (logInput, sparkVersion) = EventLoggingListener.openEventLog(new Path(eventLogFile), fs)
val replayBus = new ReplayListenerBus()
val ui = SparkUI.createHistoryUI(new SparkConf, replayBus, new SecurityManager(conf),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org