You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@livy.apache.org by js...@apache.org on 2017/06/27 06:39:27 UTC
[46/50] [abbrv] incubator-livy git commit: Minor. Add session
information log when session is created (#325)
Minor. Add session information log when session is created (#325)
* Add session information log when session is created
Change-Id: I8702ebe1d893cf328b4490e5b5d09f3afd02b7ce
* add more logs
Change-Id: I969edf4e680e59e3ecfbaa50431ae61bd03d4795
* simplify the code
Change-Id: If2c7876ddb6020ace3872c9ae639d69d58b02f48
* Address the comments
Change-Id: Iba5f10ad73ff67b97af8292769b254346b8ef7c8
Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/d51e9982
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/d51e9982
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/d51e9982
Branch: refs/heads/master
Commit: d51e998204e3a10f9207280951f98edc8755c433
Parents: 61b206e
Author: Saisai Shao <sa...@gmail.com>
Authored: Fri May 12 08:36:19 2017 +0800
Committer: Jeff Zhang <zj...@gmail.com>
Committed: Thu May 11 17:36:19 2017 -0700
----------------------------------------------------------------------
.../cloudera/livy/server/batch/BatchSession.scala | 11 ++++++++---
.../livy/server/batch/CreateBatchRequest.scala | 17 +++++++++++++++++
.../interactive/CreateInteractiveRequest.scala | 18 ++++++++++++++++++
.../server/interactive/InteractiveSession.scala | 5 ++++-
4 files changed, 47 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d51e9982/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala b/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala
index 452a9d8..1a097b7 100644
--- a/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala
+++ b/server/src/main/scala/com/cloudera/livy/server/batch/BatchSession.scala
@@ -25,7 +25,7 @@ import scala.util.Random
import com.fasterxml.jackson.annotation.JsonIgnoreProperties
-import com.cloudera.livy.LivyConf
+import com.cloudera.livy.{LivyConf, Logging}
import com.cloudera.livy.server.recovery.SessionStore
import com.cloudera.livy.sessions.{Session, SessionState}
import com.cloudera.livy.sessions.Session._
@@ -41,7 +41,7 @@ case class BatchRecoveryMetadata(
version: Int = 1)
extends RecoveryMetadata
-object BatchSession {
+object BatchSession extends Logging {
val RECOVERY_SESSION_TYPE = "batch"
def create(
@@ -89,6 +89,8 @@ object BatchSession {
SparkApp.create(appTag, None, Option(sparkSubmit), livyConf, Option(s))
}
+ info(s"Creating batch session $id: [owner: $owner, request: $request]")
+
new BatchSession(
id,
appTag,
@@ -153,7 +155,10 @@ class BatchSession(
synchronized {
debug(s"$this state changed from $oldState to $newState")
newState match {
- case SparkApp.State.RUNNING => _state = SessionState.Running()
+ case SparkApp.State.RUNNING =>
+ _state = SessionState.Running()
+ info(s"Batch session $id created [appid: ${appId.orNull}, state: ${state.toString}, " +
+ s"info: ${appInfo.asJavaMap}]")
case SparkApp.State.FINISHED => _state = SessionState.Success()
case SparkApp.State.KILLED | SparkApp.State.FAILED =>
_state = SessionState.Dead()
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d51e9982/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala b/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala
index 99459d5..29c269c 100644
--- a/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala
+++ b/server/src/main/scala/com/cloudera/livy/server/batch/CreateBatchRequest.scala
@@ -37,4 +37,21 @@ class CreateBatchRequest {
var name: Option[String] = None
var conf: Map[String, String] = Map()
+ override def toString: String = {
+ s"[proxyUser: $proxyUser, " +
+ s"file: $file, " +
+ (if (args.nonEmpty) s"args: ${args.mkString(",")}, " else "") +
+ (if (jars.nonEmpty) s"jars: ${jars.mkString(",")}, " else "") +
+ (if (pyFiles.nonEmpty) s"pyFiles: ${pyFiles.mkString(",")}, " else "") +
+ (if (files.nonEmpty) s"files: ${files.mkString(",")}, " else "") +
+ (if (archives.nonEmpty) s"archives: ${archives.mkString(",")}, " else "") +
+ (if (driverMemory.isDefined) s"driverMemory: ${driverMemory.get}, " else "") +
+ (if (driverCores.isDefined) s"driverCores: ${driverCores.get}, " else "") +
+ (if (executorMemory.isDefined) s"executorMemory: ${executorMemory.get}, " else "") +
+ (if (executorCores.isDefined) s"executorCores: ${executorCores.get}, " else "") +
+ (if (numExecutors.isDefined) s"numExecutors: ${numExecutors.get}, " else "") +
+ (if (queue.isDefined) s"queue: ${queue.get}, " else "") +
+ (if (name.isDefined) s"name: ${name.get}, " else "") +
+ (if (conf.nonEmpty) s"conf: ${conf.mkString(",")}]" else "]")
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d51e9982/server/src/main/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequest.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequest.scala b/server/src/main/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequest.scala
index a837527..7261199 100644
--- a/server/src/main/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequest.scala
+++ b/server/src/main/scala/com/cloudera/livy/server/interactive/CreateInteractiveRequest.scala
@@ -36,4 +36,22 @@ class CreateInteractiveRequest {
var name: Option[String] = None
var conf: Map[String, String] = Map()
var heartbeatTimeoutInSecond: Int = 0
+
+ override def toString: String = {
+ s"[kind: $kind, " +
+ s"proxyUser: $proxyUser, " +
+ (if (jars.nonEmpty) s"jars: ${jars.mkString(",")}, " else "") +
+ (if (pyFiles.nonEmpty) s"pyFiles: ${pyFiles.mkString(",")}, " else "") +
+ (if (files.nonEmpty) s"files: ${files.mkString(",")}, " else "") +
+ (if (archives.nonEmpty) s"archives: ${archives.mkString(",")}, " else "") +
+ (if (driverMemory.isDefined) s"driverMemory: ${driverMemory.get}, " else "") +
+ (if (driverCores.isDefined) s"driverCores: ${driverCores.get}, " else "") +
+ (if (executorMemory.isDefined) s"executorMemory: ${executorMemory.get}, " else "") +
+ (if (executorCores.isDefined) s"executorCores: ${executorCores.get}, " else "") +
+ (if (numExecutors.isDefined) s"numExecutors: ${numExecutors.get}, " else "") +
+ (if (queue.isDefined) s"queue: ${queue.get}, " else "") +
+ (if (name.isDefined) s"name: ${name.get}, " else "") +
+ (if (conf.nonEmpty) s"conf: ${conf.mkString(",")}, " else "") +
+ s"heartbeatTimeoutInSecond: $heartbeatTimeoutInSecond]"
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/d51e9982/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala
----------------------------------------------------------------------
diff --git a/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala b/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala
index 6d10c57..05bc7be 100644
--- a/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala
+++ b/server/src/main/scala/com/cloudera/livy/server/interactive/InteractiveSession.scala
@@ -98,7 +98,7 @@ object InteractiveSession extends Logging {
builderProperties.getOrElseUpdate("spark.app.name", s"livy-session-$id")
- info(s"Creating LivyClient for sessionId: $id")
+ info(s"Creating Interactive session $id: [owner: $owner, request: $request]")
val builder = new LivyClientBuilder()
.setAll(builderProperties.asJava)
.setConf("livy.client.session-id", id.toString)
@@ -421,6 +421,9 @@ class InteractiveSession(
override def onJobSucceeded(job: JobHandle[Void], result: Void): Unit = {
transition(SessionState.Running())
+ info(s"Interactive session $id created [appid: ${appId.orNull}, owner: $owner, proxyUser:" +
+ s" $proxyUser, state: ${state.toString}, kind: ${kind.toString}, " +
+ s"info: ${appInfo.asJavaMap}]")
}
private def errorOut(): Unit = {