You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@livy.apache.org by js...@apache.org on 2018/05/25 01:36:08 UTC
incubator-livy git commit: [LIVY-472][SERVER] Improve the logs for
fail-to-create session
Repository: incubator-livy
Updated Branches:
refs/heads/master 6f1756cff -> ca4cad229
[LIVY-472][SERVER] Improve the logs for fail-to-create session
## What changes were proposed in this pull request?
Livy currently doesn't give a very clear log about the fail-to-create session, it only says that session related app tag cannot be found in RM, but doesn't tell user how to search and get the true root cause. So here change the logs to make it more clear.
## How was this patch tested?
Local verification.
Author: jerryshao <ss...@hortonworks.com>
Closes #96 from jerryshao/LIVY-472.
Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/ca4cad22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/ca4cad22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/ca4cad22
Branch: refs/heads/master
Commit: ca4cad22968e1a2f88fa0ec262c1088812e3d251
Parents: 6f1756c
Author: jerryshao <ss...@hortonworks.com>
Authored: Fri May 25 09:36:03 2018 +0800
Committer: jerryshao <ss...@hortonworks.com>
Committed: Fri May 25 09:36:03 2018 +0800
----------------------------------------------------------------------
.../interactive/InteractiveSessionServlet.scala | 2 +-
.../org/apache/livy/utils/SparkYarnApp.scala | 19 +++++++++++--------
2 files changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/ca4cad22/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSessionServlet.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSessionServlet.scala b/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSessionServlet.scala
index 54046a1..4d614f4 100644
--- a/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSessionServlet.scala
+++ b/server/src/main/scala/org/apache/livy/server/interactive/InteractiveSessionServlet.scala
@@ -70,7 +70,7 @@ class InteractiveSessionServlet(
Option(session.logLines())
.map { lines =>
val size = 10
- var from = math.max(0, lines.length - size)
+ val from = math.max(0, lines.length - size)
val until = from + size
lines.view(from, until)
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/ca4cad22/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala b/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala
index 91c70ca..d255796 100644
--- a/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala
+++ b/server/src/main/scala/org/apache/livy/utils/SparkYarnApp.scala
@@ -25,6 +25,7 @@ import scala.concurrent._
import scala.concurrent.duration._
import scala.language.postfixOps
import scala.util.Try
+import scala.util.control.NonFatal
import org.apache.hadoop.yarn.api.records.{ApplicationId, ApplicationReport, FinalApplicationStatus, YarnApplicationState}
import org.apache.hadoop.yarn.client.api.YarnClient
@@ -70,7 +71,7 @@ object SparkYarnApp extends Logging {
override def run(): Unit = {
while (true) {
if (!leakedAppTags.isEmpty) {
- // kill the app if found it and remove it if exceeding a threashold
+ // kill the app if found it and remove it if exceeding a threshold
val iter = leakedAppTags.entrySet().iterator()
var isRemoved = false
val now = System.currentTimeMillis()
@@ -179,9 +180,11 @@ class SparkYarnApp private[utils] (
if (deadline.isOverdue) {
process.foreach(_.destroy())
leakedAppTags.put(appTag, System.currentTimeMillis())
- throw new Exception(s"No YARN application is found with tag $appTagLowerCase in " +
- livyConf.getTimeAsMs(LivyConf.YARN_APP_LOOKUP_TIMEOUT)/1000 + " seconds. " +
- "Please check your cluster status, it is may be very busy.")
+ throw new IllegalStateException(s"No YARN application is found with tag" +
+ s" $appTagLowerCase in ${livyConf.getTimeAsMs(LivyConf.YARN_APP_LOOKUP_TIMEOUT)/1000}" +
+ " seconds. This may be because 1) spark-submit fail to submit application to YARN; " +
+ "or 2) YARN cluster doesn't have enough resources to start the application in time. " +
+ "Please check Livy log and YARN log to know the details.")
} else {
Clock.sleep(pollInterval.toMillis)
getAppIdFromTag(appTagLowerCase, pollInterval, deadline)
@@ -290,12 +293,12 @@ class SparkYarnApp private[utils] (
debug(s"$appId $state ${yarnDiagnostics.mkString(" ")}")
} catch {
- case e: InterruptedException =>
+ case _: InterruptedException =>
yarnDiagnostics = ArrayBuffer("Session stopped by user.")
changeState(SparkApp.State.KILLED)
- case e: Throwable =>
- error(s"Error whiling refreshing YARN state: $e")
- yarnDiagnostics = ArrayBuffer(e.toString, e.getStackTrace().mkString(" "))
+ case NonFatal(e) =>
+ error(s"Error whiling refreshing YARN state", e)
+ yarnDiagnostics = ArrayBuffer(e.getMessage)
changeState(SparkApp.State.FAILED)
}
}