You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by fe...@apache.org on 2017/01/18 07:18:05 UTC
zeppelin git commit: ZEPPELIN-1852. Use multiple InterpreterResult
for displaying appInfo
Repository: zeppelin
Updated Branches:
refs/heads/master 8daf32563 -> 69b866adc
ZEPPELIN-1852. Use multiple InterpreterResult for displaying appInfo
### What is this PR for?
Refactor the livy interpreter to use multiple `InterpreterResult` for displaying appInfo.
### What type of PR is it?
[Refactoring]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-1852
### How should this be tested?
Add integration test and also test it manually
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #1796 from zjffdu/ZEPPELIN-1852 and squashes the following commits:
724f22e [Jeff Zhang] update doc
e1c2eb9 [Jeff Zhang] ZEPPELIN-1852. Use multiple InterpreterResult for displaying appInfo
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/69b866ad
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/69b866ad
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/69b866ad
Branch: refs/heads/master
Commit: 69b866adca5ecac00b26dfd3573013eaae133102
Parents: 8daf325
Author: Jeff Zhang <zj...@apache.org>
Authored: Wed Dec 28 19:08:22 2016 +0800
Committer: Felix Cheung <fe...@apache.org>
Committed: Tue Jan 17 23:18:01 2017 -0800
----------------------------------------------------------------------
docs/interpreter/livy.md | 5 +++
.../zeppelin/livy/BaseLivyInterprereter.java | 34 +++++++++-----------
.../src/main/resources/interpreter-setting.json | 8 ++---
.../apache/zeppelin/livy/LivyInterpreterIT.java | 34 +++++++++++++++++++-
4 files changed, 58 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/69b866ad/docs/interpreter/livy.md
----------------------------------------------------------------------
diff --git a/docs/interpreter/livy.md b/docs/interpreter/livy.md
index 47ebc46..6f04244 100644
--- a/docs/interpreter/livy.md
+++ b/docs/interpreter/livy.md
@@ -61,6 +61,11 @@ Example: `spark.driver.memory` to `livy.spark.driver.memory`
<td>Max number of Spark SQL result to display.</td>
</tr>
<tr>
+ <td>zeppelin.livy.session.create_timeout</td>
+ <td>120</td>
+ <td>Timeout in seconds for session creation</td>
+ </tr>
+ <tr>
<td>zeppelin.livy.displayAppInfo</td>
<td>false</td>
<td>Whether to display app info</td>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/69b866ad/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
index 3d84363..f0591fd 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java
@@ -62,8 +62,10 @@ public abstract class BaseLivyInterprereter extends Interpreter {
public BaseLivyInterprereter(Properties property) {
super(property);
this.livyURL = property.getProperty("zeppelin.livy.url");
+ this.displayAppInfo = Boolean.parseBoolean(
+ property.getProperty("zeppelin.livy.displayAppInfo", "false"));
this.sessionCreationTimeout = Integer.parseInt(
- property.getProperty("zeppelin.livy.create.session.timeout", 120 + ""));
+ property.getProperty("zeppelin.livy.session.create_timeout", 120 + ""));
this.pullStatusInterval = Integer.parseInt(
property.getProperty("zeppelin.livy.pull_status.interval.millis", 1000 + ""));
}
@@ -77,7 +79,6 @@ public abstract class BaseLivyInterprereter extends Interpreter {
} catch (LivyException e) {
String msg = "Fail to create session, please check livy interpreter log and " +
"livy server log";
- LOGGER.error(msg);
throw new RuntimeException(msg, e);
}
}
@@ -102,10 +103,11 @@ public abstract class BaseLivyInterprereter extends Interpreter {
.get(0).getData());
}
- interpret(
- "val webui=sc.getClass.getMethod(\"ui\").invoke(sc).asInstanceOf[Some[_]].get",
- null, false, false);
- if (StringUtils.isEmpty(sessionInfo.appInfo.get("sparkUiUrl"))) {
+ if (sessionInfo.appInfo == null ||
+ StringUtils.isEmpty(sessionInfo.appInfo.get("sparkUiUrl"))) {
+ interpret(
+ "val webui=sc.getClass.getMethod(\"ui\").invoke(sc).asInstanceOf[Some[_]].get",
+ null, false, false);
sessionInfo.webUIAddress = extractStatementResult(
interpret(
"webui.getClass.getMethod(\"appUIAddress\").invoke(webui)", null, false, false)
@@ -215,14 +217,12 @@ public abstract class BaseLivyInterprereter extends Interpreter {
if (sessionInfo.isFinished()) {
String msg = "Session " + sessionInfo.id + " is finished, appId: " + sessionInfo.appId
+ ", log: " + sessionInfo.log;
- LOGGER.error(msg);
throw new LivyException(msg);
}
if ((System.currentTimeMillis() - start) / 1000 > sessionCreationTimeout) {
String msg = "The creation of session " + sessionInfo.id + " is timeout within "
+ sessionCreationTimeout + " seconds, appId: " + sessionInfo.appId
+ ", log: " + sessionInfo.log;
- LOGGER.error(msg);
throw new LivyException(msg);
}
Thread.sleep(pullStatusInterval);
@@ -361,16 +361,14 @@ public abstract class BaseLivyInterprereter extends Interpreter {
if (displayAppInfo) {
//TODO(zjffdu), use multiple InterpreterResult to display appInfo
- StringBuilder outputBuilder = new StringBuilder();
- outputBuilder.append("%angular ");
- outputBuilder.append("<pre><code>");
- outputBuilder.append(result);
- outputBuilder.append("</code></pre>");
- outputBuilder.append("<hr/>");
- outputBuilder.append("Spark Application Id:" + sessionInfo.appId + "<br/>");
- outputBuilder.append("Spark WebUI: <a href=" + sessionInfo.webUIAddress + ">"
- + sessionInfo.webUIAddress + "</a>");
- return new InterpreterResult(InterpreterResult.Code.SUCCESS, outputBuilder.toString());
+ InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS);
+ interpreterResult.add(InterpreterResult.Type.TEXT, result);
+ String appInfoHtml = "<hr/>Spark Application Id: " + sessionInfo.appId + "<br/>"
+ + "Spark WebUI: <a href=\"" + sessionInfo.webUIAddress + "\">"
+ + sessionInfo.webUIAddress + "</a>";
+ LOGGER.info("appInfoHtml:" + appInfoHtml);
+ interpreterResult.add(InterpreterResult.Type.HTML, appInfoHtml);
+ return interpreterResult;
} else {
return new InterpreterResult(InterpreterResult.Code.SUCCESS, result);
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/69b866ad/livy/src/main/resources/interpreter-setting.json
----------------------------------------------------------------------
diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json
index a2b9758..42f64cf 100644
--- a/livy/src/main/resources/interpreter-setting.json
+++ b/livy/src/main/resources/interpreter-setting.json
@@ -11,9 +11,9 @@
"defaultValue": "http://localhost:8998",
"description": "The URL for Livy Server."
},
- "zeppelin.livy.create.session.retries": {
- "envName": "ZEPPELIN_LIVY_CREATE_SESSION_RETRIES",
- "propertyName": "zeppelin.livy.create.session.timeout",
+ "zeppelin.livy.session.create_timeout": {
+ "envName": "ZEPPELIN_LIVY_SESSION_CREATE_TIMEOUT",
+ "propertyName": "zeppelin.livy.session.create_timeout",
"defaultValue": "120",
"description": "Livy Server create session timeout (seconds)."
},
@@ -87,7 +87,7 @@
"defaultValue": "",
"description": "Adding extra libraries to livy interpreter"
},
- "livy.spark.displayAppInfo": {
+ "zeppelin.livy.displayAppInfo": {
"propertyName": "zeppelin.livy.displayAppInfo",
"defaultValue": "false",
"description": "Whether display app info"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/69b866ad/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
----------------------------------------------------------------------
diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
index 8ca8842..ada91ed 100644
--- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
+++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
@@ -51,7 +51,7 @@ public class LivyInterpreterIT {
LOGGER.info("Starting livy at {}", cluster.livyEndpoint());
properties = new Properties();
properties.setProperty("zeppelin.livy.url", cluster.livyEndpoint());
- properties.setProperty("zeppelin.livy.create.session.timeout", "120");
+ properties.setProperty("zeppelin.livy.session.create_timeout", "120");
properties.setProperty("zeppelin.livy.spark.sql.maxResult", "100");
}
@@ -314,6 +314,38 @@ public class LivyInterpreterIT {
}
@Test
+ public void testSparkInterpreterWithDisplayAppInfo() {
+ if (!checkPreCondition()) {
+ return;
+ }
+ InterpreterGroup interpreterGroup = new InterpreterGroup("group_1");
+ interpreterGroup.put("session_1", new ArrayList<Interpreter>());
+ Properties properties2 = new Properties(properties);
+ properties2.put("zeppelin.livy.displayAppInfo", "true");
+ // enable spark ui because it is disabled by livy integration test
+ properties2.put("livy.spark.ui.enabled", "true");
+ LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties2);
+ sparkInterpreter.setInterpreterGroup(interpreterGroup);
+ interpreterGroup.get("session_1").add(sparkInterpreter);
+ AuthenticationInfo authInfo = new AuthenticationInfo("user1");
+ MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener();
+ InterpreterOutput output = new InterpreterOutput(outputListener);
+ InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark",
+ "title", "text", authInfo, null, null, null, null, null, output);
+ sparkInterpreter.open();
+
+ try {
+ InterpreterResult result = sparkInterpreter.interpret("sc.version", context);
+ assertEquals(InterpreterResult.Code.SUCCESS, result.code());
+ assertEquals(2, result.message().size());
+ assertTrue(result.message().get(0).getData().contains("1.5.2"));
+ assertTrue(result.message().get(1).getData().contains("Spark Application Id"));
+ } finally {
+ sparkInterpreter.close();
+ }
+ }
+
+ @Test
public void testSparkRInterpreter() {
if (!checkPreCondition()) {
return;