You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2017/06/10 01:47:05 UTC
zeppelin git commit: [ZEPPELIN-2589]Let zeppelin show more than one
line of error log returned by livy.
Repository: zeppelin
Updated Branches:
refs/heads/master 43926485c -> 380cfbfe3
[ZEPPELIN-2589]Let zeppelin show more than one line of error log returned by livy.
### What is this PR for?
Let zeppelin show more than one line of error log returned by livy.
### What type of PR is it?
Improvement
### What is the Jira issue?
[ZEPPELIN-2589
](https://issues.apache.org/jira/browse/ZEPPELIN-2589)
### How should this be tested?
1. build: mvn clean package -DskipTests;
2. /bin/zeppelin-daemon.sh restart
3. Start a notebook
4. Use Livy interpreter and input some erroneous spark code, check the shown result is more than one line
### Screenshots (if appropriate)
When using Livy 0.4 to run spark streaming job.
Original Zeppelin:
After add traceback to interpreter result:
### Questions:
* Does the licenses files need update?No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: 汪赫扬 <qw...@gmail.com>
Closes #2390 from qwemicheal/master and squashes the following commits:
ee3b42c [汪赫扬] ci fail randomly
6e4d2aa [汪赫扬] trigger ci
c7e9b0e [汪赫扬] minor change to trigger ci
0b657a8 [汪赫扬] add check for traceback and modify way of checking msg number
4ea3edb [汪赫扬] merger evalue and traceback to one message and use join to concatenate traceback
b1caa2c [汪赫扬] minor change
e729db2 [汪赫扬] merge unit test to pyspark test case
774b370 [汪赫扬] use stringBuilder to concat traceback lines
f8de47a [汪赫扬] code style fixed
f96abbb [汪赫扬] only use pyspark
de2e06b [汪赫扬] close the pyspark interpreter after test
0212e35 [汪赫扬] change to pyspark for livy 0.2 when testing traceback msg
799f1c2 [汪赫扬] take livy livy version into consideration
e3f9791 [汪赫扬] add unit test
bf26dbe [汪赫扬] whitespace fitting
f6fc200 [汪赫扬] fix traceback type mismatch
3310055 [汪赫扬] add try catch to find which string failed ci
e306d4f [汪赫扬] add white space after comma
3307c6f [汪赫扬] add more debug msg
6a2983f [汪赫扬] fix typo
04061d3 [汪赫扬] add debug print
5a03e05 [汪赫扬] add check for livy 0.2 when traceback field is missing
45c5d9f [汪赫扬] add traceback msg from livy return json to InterpreterResult
f6135cc [汪赫扬] add traceback msg from livy return json to InterpreterResult
2eb6011 [汪赫扬] add traceback msg from livy return json to InterpreterResult
905c54d [汪赫扬] add traceback msg from livy return json to InterpreterResult
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/380cfbfe
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/380cfbfe
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/380cfbfe
Branch: refs/heads/master
Commit: 380cfbfe3cdef7e9281ad39b0b7e2cd3b161f72b
Parents: 4392648
Author: 汪赫扬 <qw...@gmail.com>
Authored: Wed Jun 7 21:31:40 2017 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Sat Jun 10 09:46:54 2017 +0800
----------------------------------------------------------------------
.../zeppelin/livy/BaseLivyInterpreter.java | 27 +++++++++++++++++---
.../apache/zeppelin/livy/LivyInterpreterIT.java | 21 ++++++++++++++-
2 files changed, 44 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/380cfbfe/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
----------------------------------------------------------------------
diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
index fcb7e9d..523dfad 100644
--- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
+++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java
@@ -341,7 +341,17 @@ public abstract class BaseLivyInterpreter extends Interpreter {
private InterpreterResult getResultFromStatementInfo(StatementInfo stmtInfo,
boolean displayAppInfo) {
if (stmtInfo.output != null && stmtInfo.output.isError()) {
- return new InterpreterResult(InterpreterResult.Code.ERROR, stmtInfo.output.evalue);
+ InterpreterResult result = new InterpreterResult(InterpreterResult.Code.ERROR);
+ StringBuilder sb = new StringBuilder();
+ sb.append(stmtInfo.output.evalue);
+ // in case evalue doesn't have newline char
+ if (!stmtInfo.output.evalue.contains("\n"))
+ sb.append("\n");
+ if (stmtInfo.output.traceback != null) {
+ sb.append(StringUtils.join(stmtInfo.output.traceback));
+ }
+ result.add(sb.toString());
+ return result;
} else if (stmtInfo.isCancelled()) {
// corner case, output might be null if it is cancelled.
return new InterpreterResult(InterpreterResult.Code.ERROR, "Job is cancelled");
@@ -659,7 +669,18 @@ public abstract class BaseLivyInterpreter extends Interpreter {
}
public static StatementInfo fromJson(String json) {
- return gson.fromJson(json, StatementInfo.class);
+ String right_json = "";
+ try {
+ gson.fromJson(json, StatementInfo.class);
+ right_json = json;
+ } catch (Exception e) {
+ if (json.contains("\"traceback\":{}")) {
+ LOGGER.debug("traceback type mismatch, replacing the mismatching part ");
+ right_json = json.replace("\"traceback\":{}", "\"traceback\":[]");
+ LOGGER.debug("new json string is {}", right_json);
+ }
+ }
+ return gson.fromJson(right_json, StatementInfo.class);
}
public boolean isAvailable() {
@@ -676,7 +697,7 @@ public abstract class BaseLivyInterpreter extends Interpreter {
public Data data;
public String ename;
public String evalue;
- public Object traceback;
+ public String[] traceback;
public TableMagic tableMagic;
public boolean isError() {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/380cfbfe/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
----------------------------------------------------------------------
diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
index 80965ff..824eac3 100644
--- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
+++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java
@@ -523,7 +523,7 @@ public class LivyInterpreterIT {
}
@Test
- public void testPySparkInterpreter() {
+ public void testPySparkInterpreter() throws LivyException {
if (!checkPreCondition()) {
return;
}
@@ -536,6 +536,24 @@ public class LivyInterpreterIT {
"title", "text", authInfo, null, null, null, null, null, output);
pysparkInterpreter.open();
+ // test traceback msg
+ try {
+ pysparkInterpreter.getLivyVersion();
+ // for livy version >=0.3 , input some erroneous spark code, check the shown result is more than one line
+ InterpreterResult result = pysparkInterpreter.interpret("sc.parallelize(wrongSyntax(1, 2)).count()", context);
+ assertEquals(InterpreterResult.Code.ERROR, result.code());
+ assertTrue(result.message().get(0).getData().split("\n").length>1);
+ assertTrue(result.message().get(0).getData().contains("Traceback"));
+ } catch (APINotFoundException e) {
+ // only livy 0.2 can throw this exception since it doesn't have /version endpoint
+ // in livy 0.2, most error msg is encapsulated in evalue field, only print(a) in pyspark would return none-empty
+ // traceback
+ InterpreterResult result = pysparkInterpreter.interpret("print(a)", context);
+ assertEquals(InterpreterResult.Code.ERROR, result.code());
+ assertTrue(result.message().get(0).getData().split("\n").length>1);
+ assertTrue(result.message().get(0).getData().contains("Traceback"));
+ }
+
try {
InterpreterResult result = pysparkInterpreter.interpret("sc.version", context);
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
@@ -763,6 +781,7 @@ public class LivyInterpreterIT {
}
}
+
private boolean isSpark2(BaseLivyInterpreter interpreter, InterpreterContext context) {
InterpreterResult result = null;
if (interpreter instanceof LivySparkRInterpreter) {