You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mi...@apache.org on 2017/08/16 05:56:34 UTC
zeppelin git commit: [ZEPPELIN-2067] SparkInterpreter prints
unnecessary newline
Repository: zeppelin
Updated Branches:
refs/heads/branch-0.7 d0ee507bc -> 5c8d7902a
[ZEPPELIN-2067] SparkInterpreter prints unnecessary newline
### What is this PR for?
Spark interpreter prints unnecessary new line before the evaluation output is printed.
See https://github.com/apache/zeppelin/pull/1975#issuecomment-277581660.
This PR make SparkInterpreter ignores unnecessary preceding newline from ReplReporter.error()
### What type of PR is it?
Bug Fix
### Todos
* [x] - Ignore unnecessary preceding newline
* [x] - unittest
### What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-2067
### How should this be tested?
run
```
%spark
import java.util.Date
import java.net.URL
```
and see if result looks like (no new line in front of each lines)
```
import java.util.Date
import java.net.URL
```
### Questions:
* Does the licenses files need update? no
* Is there breaking changes for older versions? no
* Does this needs documentation? no
Author: Lee moon soo <mo...@apache.org>
Closes #1981 from Leemoonsoo/ZEPPELIN-2067 and squashes the following commits:
1a31a66 [Lee moon soo] add issue id in the comment
23d5d55 [Lee moon soo] Add unittest
6908bdf [Lee moon soo] Ignore preceding newline from scala RepleReporter.error
(cherry picked from commit 0ab026e07b7c852454dd2b9a281f81249cf3d52f)
Signed-off-by: Mina Lee <mi...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/5c8d7902
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/5c8d7902
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/5c8d7902
Branch: refs/heads/branch-0.7
Commit: 5c8d7902a2b9d6c009a026b7bd6ee3bc46194a9b
Parents: d0ee507
Author: Lee moon soo <mo...@apache.org>
Authored: Tue Feb 7 13:17:55 2017 +0900
Committer: Mina Lee <mi...@apache.org>
Committed: Wed Aug 16 14:56:21 2017 +0900
----------------------------------------------------------------------
.../apache/zeppelin/spark/SparkInterpreter.java | 2 +-
.../util/InterpreterOutputStream.java | 30 ++++++++++++++------
.../zeppelin/rest/ZeppelinSparkClusterTest.java | 24 ++++++++++++++++
3 files changed, 46 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5c8d7902/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index 6edf654..e7bf265 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -981,6 +981,7 @@ public class SparkInterpreter extends Interpreter {
}
private Results.Result interpret(String line) {
+ out.ignoreLeadingNewLinesFromScalaReporter();
return (Results.Result) Utils.invokeMethod(
intp,
"interpret",
@@ -1231,7 +1232,6 @@ public class SparkInterpreter extends Interpreter {
if (varName == null || varName.isEmpty()) {
return;
}
-
Object lastObj = null;
try {
if (Utils.isScala2_10()) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5c8d7902/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java
index b6f01b1..6bdc2db 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/util/InterpreterOutputStream.java
@@ -30,6 +30,7 @@ import java.io.IOException;
public class InterpreterOutputStream extends LogOutputStream {
public static Logger logger;
InterpreterOutput interpreterOutput;
+ boolean ignoreLeadingNewLinesFromScalaReporter = false;
public InterpreterOutputStream(Logger logger) {
this.logger = logger;
@@ -45,6 +46,18 @@ public class InterpreterOutputStream extends LogOutputStream {
@Override
public void write(int b) throws IOException {
+ if (ignoreLeadingNewLinesFromScalaReporter && b == '\n') {
+ StackTraceElement[] stacks = Thread.currentThread().getStackTrace();
+ for (StackTraceElement stack : stacks) {
+ if (stack.getClassName().equals("scala.tools.nsc.interpreter.ReplReporter") &&
+ stack.getMethodName().equals("error")) {
+ // ignore. Please see ZEPPELIN-2067
+ return;
+ }
+ }
+ } else {
+ ignoreLeadingNewLinesFromScalaReporter = false;
+ }
super.write(b);
if (interpreterOutput != null) {
interpreterOutput.write(b);
@@ -53,17 +66,13 @@ public class InterpreterOutputStream extends LogOutputStream {
@Override
public void write(byte [] b) throws IOException {
- super.write(b);
- if (interpreterOutput != null) {
- interpreterOutput.write(b);
- }
+ write(b, 0, b.length);
}
@Override
- public void write(byte [] b, int offset, int len) throws IOException {
- super.write(b, offset, len);
- if (interpreterOutput != null) {
- interpreterOutput.write(b, offset, len);
+ public void write(byte [] b, int off, int len) throws IOException {
+ for (int i = off; i < len; i++) {
+ write(b[i]);
}
}
@@ -80,7 +89,6 @@ public class InterpreterOutputStream extends LogOutputStream {
}
}
-
@Override
public void flush() throws IOException {
super.flush();
@@ -88,4 +96,8 @@ public class InterpreterOutputStream extends LogOutputStream {
interpreterOutput.flush();
}
}
+
+ public void ignoreLeadingNewLinesFromScalaReporter() {
+ ignoreLeadingNewLinesFromScalaReporter = true;
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5c8d7902/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
index 7affdd8..9bcdbfa 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
@@ -77,6 +77,30 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi {
}
@Test
+ public void scalaOutputTest() throws IOException {
+ // create new note
+ Note note = ZeppelinServer.notebook.createNote(anonymous);
+ Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS);
+ Map config = p.getConfig();
+ config.put("enabled", true);
+ p.setConfig(config);
+ p.setText("%spark import java.util.Date\n" +
+ "import java.net.URL\n" +
+ "println(\"hello\")\n"
+ );
+ p.setAuthenticationInfo(anonymous);
+ note.run(p.getId());
+ waitForFinish(p);
+ assertEquals(Status.FINISHED, p.getStatus());
+ assertEquals("import java.util.Date\n" +
+ "import java.net.URL\n" +
+ "hello\n", p.getResult().message().get(0).getData());
+ ZeppelinServer.notebook.removeNote(note.getId(), anonymous);
+ }
+
+
+
+ @Test
public void basicRDDTransformationAndActionTest() throws IOException {
// create new note
Note note = ZeppelinServer.notebook.createNote(anonymous);