You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by kh...@apache.org on 2017/08/02 00:43:42 UTC
zeppelin git commit: [ZEPPELIN-2818] Improve to better rendering from
jupyter note
Repository: zeppelin
Updated Branches:
refs/heads/master 49af089d5 -> c66f90917
[ZEPPELIN-2818] Improve to better rendering from jupyter note
### What is this PR for?
Hi, zeppelin community.
Zeppelin currently has a way to use the Jupiter Note.
This is a great feature.
However, I have found that there are some problems with the way of showing.
I've improved this to increase the user experience.
Please check the screenshot for details.
### What type of PR is it?
Improvement
### Todos
- [x] added pre-render feature (markdown).
- [x] increased to many support type for output data.
### What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-2818
### How should this be tested?
1. build jupyter module
`mvn clean package -DskipTests -pl 'zeppelin-jupyter' --am`
2. `cd zeppelin-jupyter/target`
3. `java -classpath zeppelin-jupyter-0.8.0-SNAPSHOT.jar org.apache.zeppelin.jupyter.JupyterUtil -i {your ipynb note file path!/getting_started.ipynb`
(good sample : [go to sample](https://github.com/SciRuby/sciruby-notebooks/blob/master/getting_started.ipynb)
4. get a `note.json` and import to zeppelin on frontend!
5. enjoy
### Screenshots (if appropriate)
#### Before
#### After
### Questions:
* Does the licenses files need update? no
* Is there breaking changes for older versions? no
* Does this needs documentation? no
Author: CloverHearts <cl...@gmail.com>
Closes #2509 from cloverhearts/ZEPPELIN-2818 and squashes the following commits:
4f915739f [CloverHearts] added apache docuement
c15bbcdb7 [CloverHearts] improving test case
c8b5a86cc [CloverHearts] fix command line interpreter tag
f4cb0037f [CloverHearts] support single line text
e9076133a [CloverHearts] add setter
f077e69ae [CloverHearts] added supported single line output
c77d9c913 [CloverHearts] add supported single string for source
75c8ef540 [CloverHearts] import pegdown in pom
b27bfc93f [CloverHearts] move interpreter to pegdown
1e15581d1 [CloverHearts] improving reder feature
a56b6ffc6 [CloverHearts] fix checkstyle
4a5158603 [CloverHearts] implement markdown pre render
6ce9935b9 [CloverHearts] improving get output part
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/c66f9091
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/c66f9091
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/c66f9091
Branch: refs/heads/master
Commit: c66f90917955482e15db834566c242208dd361ba
Parents: 49af089
Author: CloverHearts <cl...@gmail.com>
Authored: Mon Jul 31 19:29:36 2017 +0900
Committer: Khalid Huseynov <kh...@gmail.com>
Committed: Wed Aug 2 09:47:58 2017 +0900
----------------------------------------------------------------------
zeppelin-jupyter/pom.xml | 10 ++
.../apache/zeppelin/jupyter/JupyterUtil.java | 96 +++++++------------
.../apache/zeppelin/jupyter/nbformat/Cell.java | 4 +-
.../zeppelin/jupyter/nbformat/DisplayData.java | 33 +++++++
.../apache/zeppelin/jupyter/nbformat/Error.java | 17 ++++
.../jupyter/nbformat/ExecuteResult.java | 14 +++
.../zeppelin/jupyter/nbformat/Output.java | 99 ++++++++++++++++++++
.../zeppelin/jupyter/nbformat/Stream.java | 35 ++++++-
.../jupyter/types/JupyterOutputType.java | 70 ++++++++++++++
.../jupyter/types/ZeppelinOutputType.java | 37 ++++++++
.../zeppelin/jupyter/zformat/Paragraph.java | 25 +++++
.../apache/zeppelin/jupyter/zformat/Result.java | 1 +
.../jupyter/nbformat/JupyterUtilTest.java | 48 +++++++++-
zeppelin-jupyter/src/test/resources/basic.ipynb | 34 ++++++-
14 files changed, 452 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c66f9091/zeppelin-jupyter/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-jupyter/pom.xml b/zeppelin-jupyter/pom.xml
index 96dabd4..ae00db3 100644
--- a/zeppelin-jupyter/pom.xml
+++ b/zeppelin-jupyter/pom.xml
@@ -34,6 +34,10 @@
<name>Zeppelin: Jupyter Support</name>
<description>Jupyter support for Apache Zeppelin</description>
+ <properties>
+ <pegdown.version>1.6.0</pegdown.version>
+ </properties>
+
<dependencies>
<dependency>
<groupId>com.google.code.gson</groupId>
@@ -55,6 +59,12 @@
<artifactId>commons-cli</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.pegdown</groupId>
+ <artifactId>pegdown</artifactId>
+ <version>${pegdown.version}</version>
+ </dependency>
+
<!-- Test -->
<dependency>
<groupId>junit</groupId>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c66f9091/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/JupyterUtil.java
----------------------------------------------------------------------
diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/JupyterUtil.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/JupyterUtil.java
index 3879efb..9bc5ca4 100644
--- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/JupyterUtil.java
+++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/JupyterUtil.java
@@ -24,9 +24,8 @@ import java.io.Reader;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
-import java.util.Map;
import com.google.common.base.Joiner;
import com.google.gson.Gson;
@@ -54,18 +53,18 @@ import org.apache.zeppelin.jupyter.zformat.Note;
import org.apache.zeppelin.jupyter.zformat.Paragraph;
import org.apache.zeppelin.jupyter.zformat.Result;
import org.apache.zeppelin.jupyter.zformat.TypeData;
+import org.pegdown.PegDownProcessor;
/**
*
*/
public class JupyterUtil {
- private static final String TEXT_PLAIN = "text/plain";
- private static final String IMAGE_PNG = "image/png";
-
private final RuntimeTypeAdapterFactory<Cell> cellTypeFactory;
private final RuntimeTypeAdapterFactory<Output> outputTypeFactory;
+ private final PegDownProcessor markdownProcessor;
+
public JupyterUtil() {
this.cellTypeFactory = RuntimeTypeAdapterFactory.of(Cell.class, "cell_type")
.registerSubtype(MarkdownCell.class, "markdown").registerSubtype(CodeCell.class, "code")
@@ -74,6 +73,7 @@ public class JupyterUtil {
.registerSubtype(ExecuteResult.class, "execute_result")
.registerSubtype(DisplayData.class, "display_data").registerSubtype(Stream.class, "stream")
.registerSubtype(Error.class, "error");
+ this.markdownProcessor = new PegDownProcessor();
}
public Nbformat getNbformat(Reader in) {
@@ -103,67 +103,53 @@ public class JupyterUtil {
note.setName(name);
String lineSeparator = System.lineSeparator();
- String emptyString = "";
Paragraph paragraph;
List<Paragraph> paragraphs = new ArrayList<>();
String interpreterName;
List<TypeData> typeDataList;
- String type;
- String result;
for (Cell cell : nbformat.getCells()) {
+ String status = Result.SUCCESS;
paragraph = new Paragraph();
typeDataList = new ArrayList<>();
+ Object cellSource = cell.getSource();
+ List<String> sourceRaws = new ArrayList<>();
+
+ if (cellSource instanceof String) {
+ sourceRaws.add((String) cellSource);
+ } else {
+ sourceRaws.addAll((List<String>) cellSource);
+ }
+
+ List<String> source = Output.verifyEndOfLine(sourceRaws);
+ String codeText = Joiner.on("").join(source);
if (cell instanceof CodeCell) {
interpreterName = codeReplaced;
for (Output output : ((CodeCell) cell).getOutputs()) {
- TypeData typeData;
- if (output instanceof Stream) {
- type = TypeData.TEXT;
- List<String> text = verifyEndOfLine(((Stream) output).getText(), lineSeparator);
- result = Joiner.on(emptyString).join(text);
- typeData = new TypeData(type, result);
- typeDataList.add(typeData);
- } else if (output instanceof ExecuteResult || output instanceof DisplayData) {
- Map<String, Object> data = (output instanceof ExecuteResult) ?
- ((ExecuteResult) output).getData() :
- ((DisplayData) output).getData();
- for (Map.Entry<String, Object> datum : data.entrySet()) {
- if (TEXT_PLAIN.equals(datum.getKey())) {
- type = TypeData.TEXT;
- List<String> text = verifyEndOfLine((List<String>) datum.getValue(), lineSeparator);
- result = Joiner.on(emptyString).join(text);
- } else if (IMAGE_PNG.equals(datum.getKey())) {
- type = TypeData.HTML;
- result = makeHTML(((String) datum.getValue()).replace("\n", ""));
- } else {
- type = TypeData.TEXT;
- result = datum.getValue().toString();
+ if (output instanceof Error) {
+ typeDataList.add(output.toZeppelinResult());
+ } else {
+ typeDataList.add(output.toZeppelinResult());
+ if (output instanceof Stream) {
+ Stream streamOutput = (Stream) output;
+ if (streamOutput.isError()) {
+ status = Result.ERROR;
}
- typeData = new TypeData(type, result);
- typeDataList.add(typeData);
}
- } else {
- // Error
- Error error = (Error) output;
- type = TypeData.TEXT;
- List<String> text = verifyEndOfLine(Arrays.asList(error.getEname(), error.getEvalue()),
- lineSeparator);
- result = Joiner.on(emptyString).join(text);
- typeData = new TypeData(type, result);
- typeDataList.add(typeData);
}
}
} else if (cell instanceof MarkdownCell || cell instanceof HeadingCell) {
interpreterName = markdownReplaced;
+ String markdownContent = markdownProcessor.markdownToHtml(codeText);
+ typeDataList.add(new TypeData(TypeData.HTML, markdownContent));
+ paragraph.setUpMarkdownConfig(true);
} else {
interpreterName = "";
}
- List<String> source = verifyEndOfLine(cell.getSource(), lineSeparator);
- paragraph.setText(interpreterName + lineSeparator + Joiner.on(emptyString).join(source));
- paragraph.setResults(new Result(Result.SUCCESS, typeDataList));
+ paragraph.setText(interpreterName + lineSeparator + codeText);
+ paragraph.setResults(new Result(status, typeDataList));
paragraphs.add(paragraph);
}
@@ -173,31 +159,13 @@ public class JupyterUtil {
return note;
}
- private List<String> verifyEndOfLine(List<String> content, String lineSeparator) {
- if (null == content || content.size() == 1) {
- // one-liners don't have line separator
- return content;
- }
- for (int i = 0; i < content.size(); i++) {
- String line = content.get(i);
- // verify to end with line separator except the last element
- if (null != line && !line.endsWith(lineSeparator) && i != (content.size() - 1)) {
- content.set(i, line + lineSeparator);
- }
- }
- return content;
- }
+
private Gson getGson(GsonBuilder gsonBuilder) {
return gsonBuilder.registerTypeAdapterFactory(cellTypeFactory)
.registerTypeAdapterFactory(outputTypeFactory).create();
}
- private String makeHTML(String image) {
- return "<div style='width:auto;height:auto'><img src=data:image/png;base64," + image
- + " style='width=auto;height:auto'/></div>";
- }
-
public static void main(String[] args) throws ParseException, IOException {
Options options = new Options();
options.addOption("i", true, "Jupyter notebook file");
@@ -216,7 +184,7 @@ public class JupyterUtil {
try (BufferedReader in = new BufferedReader(new FileReader(jupyterPath.toFile()));
FileWriter fw = new FileWriter(zeppelinPath.toFile())) {
- Note note = new JupyterUtil().getNote(in, "python", "md");
+ Note note = new JupyterUtil().getNote(in, "%python", "%md");
Gson gson = new GsonBuilder().setPrettyPrinting().create();
gson.toJson(note, fw);
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c66f9091/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Cell.java
----------------------------------------------------------------------
diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Cell.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Cell.java
index d44ec02..f593da5 100644
--- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Cell.java
+++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Cell.java
@@ -31,7 +31,7 @@ public abstract class Cell {
private CellMetadata metadata;
@SerializedName("source")
- private List<String> source;
+ private Object source;
public String getCellType() {
return cellType;
@@ -41,7 +41,7 @@ public abstract class Cell {
return metadata;
}
- public List<String> getSource() {
+ public Object getSource() {
return source;
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c66f9091/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/DisplayData.java
----------------------------------------------------------------------
diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/DisplayData.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/DisplayData.java
index 68bfbfe..6566e1d 100644
--- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/DisplayData.java
+++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/DisplayData.java
@@ -16,7 +16,14 @@
*/
package org.apache.zeppelin.jupyter.nbformat;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
import com.google.gson.annotations.SerializedName;
+import org.apache.zeppelin.jupyter.types.JupyterOutputType;
+import org.apache.zeppelin.jupyter.types.ZeppelinOutputType;
+import org.apache.zeppelin.jupyter.zformat.TypeData;
+
+import java.util.List;
import java.util.Map;
/**
@@ -30,4 +37,30 @@ public class DisplayData extends Output {
public Map<String, Object> getData() {
return data;
}
+
+ @Override
+ public ZeppelinOutputType getTypeOfZeppelin() {
+ return getType(data).getZeppelinType();
+ }
+
+ @Override
+ public TypeData toZeppelinResult() {
+ return getZeppelinResult(data, getType(data));
+ }
+
+ private static class ZeppelinResultGenerator {
+ public static String toBase64ImageHtmlElement(String image) {
+ return "<div style='width:auto;height:auto'><img src=data:image/png;base64," + image
+ + " style='width=auto;height:auto'/></div>";
+ }
+ public static String toLatex(String latexCode) {
+ String latexContents = latexCode;
+ return "<div>" +
+ "<div class='class=\"alert alert-warning\"'>" +
+ "<strong>Warning!</strong> Currently, Latex is not supported." +
+ "</div>" +
+ "<div>" + latexContents + "</div>" +
+ "</div>";
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c66f9091/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Error.java
----------------------------------------------------------------------
diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Error.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Error.java
index 61466fa..679f4cc 100644
--- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Error.java
+++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Error.java
@@ -16,7 +16,12 @@
*/
package org.apache.zeppelin.jupyter.nbformat;
+import com.google.common.base.Joiner;
import com.google.gson.annotations.SerializedName;
+import org.apache.zeppelin.jupyter.types.ZeppelinOutputType;
+import org.apache.zeppelin.jupyter.zformat.TypeData;
+
+import java.util.Arrays;
import java.util.List;
/**
@@ -43,4 +48,16 @@ public class Error extends Output {
public List<String> getTraceback() {
return traceback;
}
+
+ @Override
+ public ZeppelinOutputType getTypeOfZeppelin() {
+ return ZeppelinOutputType.TEXT;
+ }
+
+ @Override
+ public TypeData toZeppelinResult() {
+ List<String> text = verifyEndOfLine(Arrays.asList(getEname(), getEvalue()));
+ String result = Joiner.on("").join(text);
+ return new TypeData(getTypeOfZeppelin().toString(), result);
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c66f9091/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/ExecuteResult.java
----------------------------------------------------------------------
diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/ExecuteResult.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/ExecuteResult.java
index 01f0cea..9e8fe05 100644
--- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/ExecuteResult.java
+++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/ExecuteResult.java
@@ -17,6 +17,10 @@
package org.apache.zeppelin.jupyter.nbformat;
import com.google.gson.annotations.SerializedName;
+import org.apache.zeppelin.jupyter.types.JupyterOutputType;
+import org.apache.zeppelin.jupyter.types.ZeppelinOutputType;
+import org.apache.zeppelin.jupyter.zformat.TypeData;
+
import java.util.Map;
/**
@@ -33,4 +37,14 @@ public class ExecuteResult extends Output {
public Map<String, Object> getData() {
return data;
}
+
+ @Override
+ public ZeppelinOutputType getTypeOfZeppelin() {
+ return getType(data).getZeppelinType();
+ }
+
+ @Override
+ public TypeData toZeppelinResult() {
+ return getZeppelinResult(data, getType(data));
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c66f9091/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Output.java
----------------------------------------------------------------------
diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Output.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Output.java
index 50722d7..a37272a 100644
--- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Output.java
+++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Output.java
@@ -16,7 +16,15 @@
*/
package org.apache.zeppelin.jupyter.nbformat;
+import com.google.common.base.Joiner;
import com.google.gson.annotations.SerializedName;
+import org.apache.zeppelin.jupyter.types.JupyterOutputType;
+import org.apache.zeppelin.jupyter.types.ZeppelinOutputType;
+import org.apache.zeppelin.jupyter.zformat.TypeData;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
/**
*
@@ -25,4 +33,95 @@ public abstract class Output {
@SerializedName("output_type")
private String outputType;
+
+ private static final transient String lineSeparator = System.lineSeparator();
+
+ public static List<String> verifyEndOfLine(List<String> content) {
+ if (null == content || content.size() == 1) {
+ // one-liners don't have line separator
+ return content;
+ }
+ for (int i = 0; i < content.size(); i++) {
+ String line = content.get(i);
+ // verify to end with line separator except the last element
+ if (null != line && !line.endsWith(lineSeparator) && i != (content.size() - 1)) {
+ content.set(i, line + lineSeparator);
+ }
+ }
+ return content;
+ }
+
+ protected JupyterOutputType getType(Map<String, Object> data) {
+ JupyterOutputType jupyterOutputType = JupyterOutputType.TEXT_PLAIN;
+
+ if (data == null) {
+ return null;
+ }
+
+ for (String dataType : data.keySet()) {
+ if (!dataType.equals(JupyterOutputType.TEXT_PLAIN.toString())) {
+ try {
+ jupyterOutputType = JupyterOutputType.getByValue(dataType);
+ } catch (IllegalArgumentException e) {
+ // pass
+ }
+ }
+ }
+
+ return jupyterOutputType;
+ }
+
+
+ protected TypeData getZeppelinResult(Map<String, Object> data, JupyterOutputType type) {
+ TypeData result = null;
+ Object outputsObject = data.get(type.toString());
+ List<String> outputsRaws = new ArrayList<>();
+ if (outputsObject instanceof String) {
+ outputsRaws.add((String) outputsObject);
+ } else {
+ outputsRaws.addAll((List<String>) outputsObject);
+ }
+ List<String> outputs = verifyEndOfLine(outputsRaws);
+ String outputData = Joiner.on("").join(outputs);
+ if (type == JupyterOutputType.IMAGE_PNG) {
+ String base64CodeRaw = outputData;
+ String base64Code = base64CodeRaw.replace("\n", "");
+ result = new TypeData(
+ type.getZeppelinType().toString(),
+ ZeppelinResultGenerator.toBase64ImageHtmlElement(base64Code)
+ );
+ } else if (type == JupyterOutputType.LATEX) {
+ result = new TypeData(
+ type.getZeppelinType().toString(),
+ ZeppelinResultGenerator.toLatex(outputData)
+ );
+ } else if (type == JupyterOutputType.APPLICATION_JAVASCRIPT) {
+ result = new TypeData(
+ type.getZeppelinType().toString(),
+ ZeppelinResultGenerator.toJavascript(outputData)
+ );
+ } else {
+ result = new TypeData(type.getZeppelinType().toString(), outputData);
+ }
+ return result;
+ }
+
+ public abstract ZeppelinOutputType getTypeOfZeppelin();
+ public abstract TypeData toZeppelinResult();
+
+ private static class ZeppelinResultGenerator {
+ public static String toBase64ImageHtmlElement(String image) {
+ return "<div style='width:auto;height:auto'><img src=data:image/png;base64," + image
+ + " style='width=auto;height:auto'/></div>";
+ }
+ public static String toLatex(String latexCode) {
+ String latexContents = latexCode;
+ return "<div>" +
+ "<div>" + latexContents + "</div>" +
+ "</div>";
+ }
+ public static String toJavascript(String javascriptCode) {
+ return "<script type='application/javascript'>" + javascriptCode + "</script>";
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c66f9091/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Stream.java
----------------------------------------------------------------------
diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Stream.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Stream.java
index a73f635..fa0e246 100644
--- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Stream.java
+++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/nbformat/Stream.java
@@ -16,7 +16,13 @@
*/
package org.apache.zeppelin.jupyter.nbformat;
+import com.google.common.base.Joiner;
import com.google.gson.annotations.SerializedName;
+import org.apache.zeppelin.jupyter.types.ZeppelinOutputType;
+import org.apache.zeppelin.jupyter.zformat.TypeData;
+
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
/**
@@ -28,9 +34,34 @@ public class Stream extends Output {
private String name;
@SerializedName("text")
- private List<String> text;
+ private Object text;
public List<String> getText() {
- return text;
+ List<String> textList = new ArrayList<>();
+ if (text instanceof String) {
+ textList.add((String) text);
+ } else {
+ textList = (List<String>) text;
+ }
+ return textList;
+ }
+
+ public boolean isError() {
+ if (name == null) {
+ return true;
+ }
+ return name.equals("stderr");
+ }
+
+ @Override
+ public ZeppelinOutputType getTypeOfZeppelin() {
+ return ZeppelinOutputType.TEXT;
+ }
+
+ @Override
+ public TypeData toZeppelinResult() {
+ List<String> text = verifyEndOfLine(getText());
+ String result = Joiner.on("").join(text);
+ return new TypeData(getTypeOfZeppelin().toString(), result);
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c66f9091/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/JupyterOutputType.java
----------------------------------------------------------------------
diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/JupyterOutputType.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/JupyterOutputType.java
new file mode 100644
index 0000000..383ba11
--- /dev/null
+++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/JupyterOutputType.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.jupyter.types;
+
+/**
+ * Jupyter Output Types.
+ */
+public enum JupyterOutputType {
+ TEXT_PLAIN("text/plain"),
+ IMAGE_PNG("image/png"),
+ LATEX("text/latex"),
+ SVG_XML("image/svg+xml"),
+ TEXT_HTML("text/html"),
+ APPLICATION_JAVASCRIPT("application/javascript")
+ ;
+
+ private final String type;
+ private JupyterOutputType(final String type) {
+ this.type = type;
+ }
+
+ public ZeppelinOutputType getZeppelinType() {
+ return Convertor.ToZeppelin.getType(type);
+ }
+
+ public static JupyterOutputType getByValue(String value) {
+ for (JupyterOutputType type : JupyterOutputType.values()) {
+ if (type.toString().equals(value)) {
+ return type;
+ }
+ }
+ return JupyterOutputType.TEXT_PLAIN;
+ }
+
+ @Override
+ public String toString() {
+ return type;
+ }
+
+ private enum Convertor {
+ ToZeppelin;
+
+ public ZeppelinOutputType getType(String typeValue) {
+ JupyterOutputType type = JupyterOutputType.getByValue(typeValue);
+ ZeppelinOutputType outputType;
+
+ if (JupyterOutputType.TEXT_PLAIN == type) {
+ outputType = ZeppelinOutputType.TEXT;
+ } else {
+ outputType = ZeppelinOutputType.HTML;
+ }
+
+ return outputType;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c66f9091/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/ZeppelinOutputType.java
----------------------------------------------------------------------
diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/ZeppelinOutputType.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/ZeppelinOutputType.java
new file mode 100644
index 0000000..cf1a5e0
--- /dev/null
+++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/types/ZeppelinOutputType.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.jupyter.types;
+
+/**
+ * Zeppelin Output Types.
+ */
+public enum ZeppelinOutputType {
+ TEXT("TEXT"),
+ HTML("HTML"),
+ TABLE("TABLE")
+ ;
+
+ private final String type;
+ private ZeppelinOutputType(final String type) {
+ this.type = type;
+ }
+
+ @Override
+ public String toString() {
+ return type;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c66f9091/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Paragraph.java
----------------------------------------------------------------------
diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Paragraph.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Paragraph.java
index f6a7ebf..a93adae 100644
--- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Paragraph.java
+++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Paragraph.java
@@ -19,6 +19,8 @@ package org.apache.zeppelin.jupyter.zformat;
import com.google.gson.annotations.SerializedName;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
/**
*
@@ -26,6 +28,9 @@ import java.util.Date;
public class Paragraph {
public static final String FINISHED = "FINISHED";
+ @SerializedName("config")
+ private Map<String, Object> config;
+
@SerializedName("text")
private String text;
@@ -42,6 +47,22 @@ public class Paragraph {
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd-HHmmss");
this.id = dateFormat.format(new Date()) + "_" + super.hashCode();
this.status = FINISHED;
+ initializeConfig();
+ }
+
+ private void initializeConfig() {
+ this.config = new HashMap<>();
+ this.config.put("editorHide", false);
+ this.config.put("editorMode", "ace/mode/python");
+ }
+
+ public void setUpMarkdownConfig(boolean toActiveEditOnDblClickMode) {
+ Map<String, Object> editorSetting = new HashMap<>();
+ editorSetting.put("language", "markdown");
+ editorSetting.put("editOnDblClick", toActiveEditOnDblClickMode);
+ this.config.put("editorHide", toActiveEditOnDblClickMode);
+ this.config.put("editorSetting", editorSetting);
+ this.config.put("editorMode", "ace/mode/markdown");
}
public String getText() {
@@ -67,4 +88,8 @@ public class Paragraph {
public String getStatus() {
return status;
}
+
+ public Map<String, Object> getConfig() {
+ return config;
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c66f9091/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Result.java
----------------------------------------------------------------------
diff --git a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Result.java b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Result.java
index 7efa5c9..a00cdd3 100644
--- a/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Result.java
+++ b/zeppelin-jupyter/src/main/java/org/apache/zeppelin/jupyter/zformat/Result.java
@@ -24,6 +24,7 @@ import java.util.List;
*/
public class Result {
public static final String SUCCESS = "SUCCESS";
+ public static final String ERROR = "ERROR";
@SerializedName("code")
private String code;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c66f9091/zeppelin-jupyter/src/test/java/org/apache/zeppelin/jupyter/nbformat/JupyterUtilTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-jupyter/src/test/java/org/apache/zeppelin/jupyter/nbformat/JupyterUtilTest.java b/zeppelin-jupyter/src/test/java/org/apache/zeppelin/jupyter/nbformat/JupyterUtilTest.java
index be9a5ec..a73571c 100644
--- a/zeppelin-jupyter/src/test/java/org/apache/zeppelin/jupyter/nbformat/JupyterUtilTest.java
+++ b/zeppelin-jupyter/src/test/java/org/apache/zeppelin/jupyter/nbformat/JupyterUtilTest.java
@@ -20,8 +20,14 @@ import static org.junit.Assert.assertTrue;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.util.List;
+import java.util.Map;
+
+import com.google.gson.Gson;
import org.apache.zeppelin.jupyter.JupyterUtil;
import org.apache.zeppelin.jupyter.zformat.Note;
+import org.apache.zeppelin.jupyter.zformat.Paragraph;
+import org.apache.zeppelin.jupyter.zformat.TypeData;
import org.junit.Test;
/**
@@ -30,7 +36,7 @@ import org.junit.Test;
public class JupyterUtilTest {
@Test
- public void getNbFormat() {
+ public void getNbFormat() throws Exception {
InputStream resource = getClass().getResourceAsStream("/basic.ipynb");
Nbformat nbformat = new JupyterUtil().getNbformat(new InputStreamReader(resource));
assertTrue(nbformat.getCells().get(0) instanceof CodeCell);
@@ -40,9 +46,47 @@ public class JupyterUtilTest {
}
@Test
- public void getNote() {
+ public void getNote() throws Exception {
InputStream resource = getClass().getResourceAsStream("/examples.ipynb");
Note n = new JupyterUtil().getNote(new InputStreamReader(resource), "%python", "%md");
}
+ @Test
+ public void getNoteAndVerifyData() throws Exception {
+ String noteName = "Note converted from Jupyter";
+ InputStream resource = getClass().getResourceAsStream("/basic.ipynb");
+ Note n = new JupyterUtil().getNote(new InputStreamReader(resource), "%python", "%md");
+ Gson gson = new Gson();
+ System.out.println(gson.toJson(n));
+ System.out.println(n.getParagraphs().size());
+ assertTrue(n.getParagraphs().size() == 8);
+ assertTrue(noteName.equals(n.getName()));
+
+ Paragraph firstParagraph = n.getParagraphs().get(0);
+ assertTrue(firstParagraph.getText().equals("%python\nimport numpy as np"));
+ assertTrue(firstParagraph.getStatus().equals("FINISHED"));
+ Map<String, Object> config = firstParagraph.getConfig();
+
+ assertTrue(((String) config.get("editorMode")).equals("ace/mode/python"));
+ assertTrue(((boolean) config.get("editorHide")) == false);
+
+ Paragraph markdownParagraph = n.getParagraphs().get(6);
+
+ assertTrue(markdownParagraph.getText().equals("%md\n" +
+ "<div class=\"alert\" style=\"border: 1px solid #aaa; background: radial-gradient(ellipse at center, #ffffff 50%, #eee 100%);\">\n" +
+ "<div class=\"row\">\n" +
+ " <div class=\"col-sm-1\"><img src=\"https://knowledgeanyhow.org/static/images/favicon_32x32.png\" style=\"margin-top: -6px\"/></div>\n" +
+ " <div class=\"col-sm-11\">This notebook was created using <a href=\"https://knowledgeanyhow.org\">IBM Knowledge Anyhow Workbench</a>. To learn more, visit us at <a href=\"https://knowledgeanyhow.org\">https://knowledgeanyhow.org</a>.</div>\n" +
+ " </div>\n" +
+ "</div>"));
+ assertTrue(markdownParagraph.getStatus().equals("FINISHED"));
+
+ Map<String, Object> markdownConfig = markdownParagraph.getConfig();
+ assertTrue(((String) markdownConfig.get("editorMode")).equals("ace/mode/markdown"));
+ assertTrue(((boolean) markdownConfig.get("editorHide")) == true);
+ assertTrue(markdownParagraph.getResults().getCode().equals("SUCCESS"));
+ List<TypeData> results = markdownParagraph.getResults().getMsg();
+ assertTrue(results.get(0).getData().equals("\u003cdiv class\u003d\"alert\" style\u003d\"border: 1px solid #aaa; background: radial-gradient(ellipse at center, #ffffff 50%, #eee 100%);\"\u003e\n\u003cdiv class\u003d\"row\"\u003e\n \u003cdiv class\u003d\"col-sm-1\"\u003e\u003cimg src\u003d\"https://knowledgeanyhow.org/static/images/favicon_32x32.png\" style\u003d\"margin-top: -6px\"/\u003e\u003c/div\u003e\n \u003cdiv class\u003d\"col-sm-11\"\u003eThis notebook was created using \u003ca href\u003d\"https://knowledgeanyhow.org\"\u003eIBM Knowledge Anyhow Workbench\u003c/a\u003e. To learn more, visit us at \u003ca href\u003d\"https://knowledgeanyhow.org\"\u003ehttps://knowledgeanyhow.org\u003c/a\u003e.\u003c/div\u003e\n \u003c/div\u003e\n\u003c/div\u003e"));
+ assertTrue(results.get(0).getType().equals("HTML"));
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c66f9091/zeppelin-jupyter/src/test/resources/basic.ipynb
----------------------------------------------------------------------
diff --git a/zeppelin-jupyter/src/test/resources/basic.ipynb b/zeppelin-jupyter/src/test/resources/basic.ipynb
index 472f868..6faf17e 100644
--- a/zeppelin-jupyter/src/test/resources/basic.ipynb
+++ b/zeppelin-jupyter/src/test/resources/basic.ipynb
@@ -17,7 +17,7 @@
* under the License.
*/
- {
+{
"cells": [
{
"cell_type": "code",
@@ -115,6 +115,38 @@
},
{
"cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "collapsed": false,
+ "trusted": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " precision recall f1-score support\n\n alt.atheism 0.86 0.95 0.90 137\n comp.graphics 0.89 0.84 0.86 178\n comp.os.ms-windows.misc 0.85 0.90 0.88 168\ncomp.sys.ibm.pc.hardware 0.85 0.78 0.81 180\n comp.sys.mac.hardware 0.95 0.83 0.89 174\n comp.windows.x 0.88 0.91 0.90 193\n misc.forsale 0.78 0.92 0.84 157\n rec.autos 0.93 0.89 0.91 188\n rec.motorcycles 0.96 0.95 0.95 185\n rec.sport.baseball 0.94 0.95 0.95 183\n rec.sport.hockey 0.92 0.98 0.95 180\n sci.crypt 0.95 0.97 0.96 181\n sci.electronics 0.93 0.80 0.86 180\n sci.med 0.95 0.94
0.95 159\n sci.space 0.91 0.98 0.95 185\n soc.religion.christian 0.88 0.92 0.90 176\n talk.politics.guns 0.93 0.98 0.95 176\n talk.politics.mideast 0.92 0.98 0.95 180\n talk.politics.misc 0.97 0.91 0.94 127\n talk.religion.misc 0.99 0.69 0.81 108\n\n avg / total 0.91 0.91 0.91 3395\n\n"
+ ]
+ }
+ ],
+ "source": [
+ "# <help:scikit_pipeline>\n",
+ "# train the model and predict the test set\n",
+ "y_pred = clf.fit(X_train, y_train).predict(X_test)\n",
+ "\n",
+ "# standard information retrieval metrics\n",
+ "print metrics.classification_report(y_test, y_pred, target_names=labels)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "<div class=\"alert\" style=\"border: 1px solid #aaa; background: radial-gradient(ellipse at center, #ffffff 50%, #eee 100%);\">\n<div class=\"row\">\n <div class=\"col-sm-1\"><img src=\"https://knowledgeanyhow.org/static/images/favicon_32x32.png\" style=\"margin-top: -6px\"/></div>\n <div class=\"col-sm-11\">This notebook was created using <a href=\"https://knowledgeanyhow.org\">IBM Knowledge Anyhow Workbench</a>. To learn more, visit us at <a href=\"https://knowledgeanyhow.org\">https://knowledgeanyhow.org</a>.</div>\n </div>\n</div>"
+ ]
+ },
+ {
+ "cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true