You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by jo...@apache.org on 2017/09/18 08:01:29 UTC
zeppelin git commit: [HOTFIX] Forward compatibility of Note class
Repository: zeppelin
Updated Branches:
refs/heads/branch-0.7 54eb18f41 -> abdf3352e
[HOTFIX] Forward compatibility of Note class
### What is this PR for?
Solving inconsistency issue of Note's dateformat between branch-0.7 and master. It's related to #2272
### What type of PR is it?
[Hot Fix]
### Todos
* [x] - Support several dateformat for Note class
### What is the Jira issue?
* N/A
### How should this be tested?
* Sync with some remote repository
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jongyoul Lee <jo...@gmail.com>
Closes #2591 from jongyoul/hotfix/note-json-forward-compatibility and squashes the following commits:
046a245e [Jongyoul Lee] Fixed style
492e7d30 [Jongyoul Lee] Make test case more concrete
13b9aef5 [Jongyoul Lee] Added one more deserialize option
53add839 [Jongyoul Lee] Hotfix for forward compatibility of Note.class
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/abdf3352
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/abdf3352
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/abdf3352
Branch: refs/heads/branch-0.7
Commit: abdf3352e5817e2da0de29f51b72d761cabb4919
Parents: 54eb18f
Author: Jongyoul Lee <jo...@gmail.com>
Authored: Mon Sep 18 13:53:44 2017 +0900
Committer: Jongyoul Lee <jo...@apache.org>
Committed: Mon Sep 18 17:01:19 2017 +0900
----------------------------------------------------------------------
.../java/org/apache/zeppelin/notebook/Note.java | 42 +++++++++++-----
.../org/apache/zeppelin/notebook/Notebook.java | 2 +-
.../notebook/NotebookImportDeserializer.java | 12 ++---
.../notebook/repo/AzureNotebookRepo.java | 6 +--
.../zeppelin/notebook/repo/S3NotebookRepo.java | 2 +-
.../zeppelin/notebook/repo/VFSNotebookRepo.java | 2 +-
.../repo/zeppelinhub/ZeppelinHubRepo.java | 4 +-
.../apache/zeppelin/notebook/ParagraphTest.java | 50 +++++++++++---------
8 files changed, 69 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
index 1964a08..ae0de5d 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
@@ -17,21 +17,37 @@
package org.apache.zeppelin.notebook;
-import static java.lang.String.format;
-
+import com.google.common.base.Preconditions;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
import java.io.IOException;
import java.io.Serializable;
-import java.util.*;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-
import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.Input;
-import org.apache.zeppelin.interpreter.*;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterFactory;
+import org.apache.zeppelin.interpreter.InterpreterGroup;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+import org.apache.zeppelin.interpreter.InterpreterResultMessage;
+import org.apache.zeppelin.interpreter.InterpreterSetting;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
@@ -42,11 +58,8 @@ import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.user.Credentials;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
-import com.google.gson.Gson;
+import static java.lang.String.format;
/**
* Binded interpreters for a note
@@ -59,6 +72,10 @@ public class Note implements Serializable, ParagraphJobListener {
private static final ScheduledThreadPoolExecutor delayedPersistThreadPool =
new ScheduledThreadPoolExecutor(0);
+ public static final Gson GSON = new GsonBuilder().setPrettyPrinting()
+ .setDateFormat("yyyy-MM-dd HH:mm:ss.SSS")
+ .registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create();
+
static {
delayedPersistThreadPool.setRemoveOnCancelPolicy(true);
}
@@ -323,9 +340,8 @@ public class Note implements Serializable, ParagraphJobListener {
newParagraph.setTitle(srcParagraph.getTitle());
try {
- Gson gson = new Gson();
- String resultJson = gson.toJson(srcParagraph.getReturn());
- InterpreterResult result = gson.fromJson(resultJson, InterpreterResult.class);
+ String resultJson = GSON.toJson(srcParagraph.getReturn());
+ InterpreterResult result = GSON.fromJson(resultJson, InterpreterResult.class);
newParagraph.setReturn(result, null);
} catch (Exception e) {
// 'result' part of Note consists of exception, instead of actual interpreter results
@@ -589,7 +605,7 @@ public class Note implements Serializable, ParagraphJobListener {
public void run(String paragraphId) {
Paragraph p = getParagraph(paragraphId);
p.setListener(jobListenerFactory.getParagraphJobListener(this));
-
+
if (p.isBlankParagraph()) {
logger.info("skip to run blank paragraph. {}", p.getId());
p.setStatus(Job.Status.FINISHED);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
index 953a73f..e8739f4 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java
@@ -211,7 +211,7 @@ public class Notebook implements NoteEventListener {
reader.setLenient(true);
Note newNote;
try {
- Note oldNote = gson.fromJson(reader, Note.class);
+ Note oldNote = Note.GSON.fromJson(reader, Note.class);
convertFromSingleResultToMultipleResultsFormat(oldNote);
newNote = createNote(subject);
if (noteName != null)
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookImportDeserializer.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookImportDeserializer.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookImportDeserializer.java
index 86e0f0b..7211039 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookImportDeserializer.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookImportDeserializer.java
@@ -21,7 +21,6 @@ import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonParseException;
-
import java.lang.reflect.Type;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@@ -30,18 +29,19 @@ import java.util.Date;
import java.util.Locale;
/**
- * importNote date format deserializer
+ * importNote date format deserializer
*/
public class NotebookImportDeserializer implements JsonDeserializer<Date> {
- private static final String[] DATE_FORMATS = new String[] {
+ private static final String[] DATE_FORMATS = new String[]{
"yyyy-MM-dd'T'HH:mm:ssZ",
"MMM d, yyyy h:mm:ss a",
- "MMM dd, yyyy HH:mm:ss"
+ "MMM dd, yyyy HH:mm:ss",
+ "yyyy-MM-dd HH:mm:ss.SSS"
};
@Override
public Date deserialize(JsonElement jsonElement, Type typeOF,
- JsonDeserializationContext context) throws JsonParseException {
+ JsonDeserializationContext context) throws JsonParseException {
for (String format : DATE_FORMATS) {
try {
return new SimpleDateFormat(format, Locale.US).parse(jsonElement.getAsString());
@@ -49,6 +49,6 @@ public class NotebookImportDeserializer implements JsonDeserializer<Date> {
}
}
throw new JsonParseException("Unparsable date: \"" + jsonElement.getAsString()
- + "\". Supported formats: " + Arrays.toString(DATE_FORMATS));
+ + "\". Supported formats: " + Arrays.toString(DATE_FORMATS));
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
index 79f5dd6..1ccc78f 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
@@ -25,7 +25,6 @@ import java.io.Writer;
import java.net.URISyntaxException;
import java.security.InvalidKeyException;
import java.util.Collections;
-import java.util.Date;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -35,7 +34,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NoteInfo;
-import org.apache.zeppelin.notebook.NotebookImportDeserializer;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.user.AuthenticationInfo;
@@ -135,10 +133,8 @@ public class AzureNotebookRepo implements NotebookRepo {
GsonBuilder gsonBuilder = new GsonBuilder();
gsonBuilder.setPrettyPrinting();
- Gson gson = gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer())
- .create();
- Note note = gson.fromJson(json, Note.class);
+ Note note = Note.GSON.fromJson(json, Note.class);
for (Paragraph p : note.getParagraphs()) {
if (p.getStatus() == Job.Status.PENDING || p.getStatus() == Job.Status.RUNNING) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
index 26781b8..314c14e 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
@@ -199,7 +199,7 @@ public class S3NotebookRepo implements NotebookRepo {
Note note;
try (InputStream ins = s3object.getObjectContent()) {
String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
- note = gson.fromJson(json, Note.class);
+ note = Note.GSON.fromJson(json, Note.class);
}
for (Paragraph p : note.getParagraphs()) {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
index 04a7075..aebda97 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
@@ -175,7 +175,7 @@ public class VFSNotebookRepo implements NotebookRepo {
String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
ins.close();
- Note note = gson.fromJson(json, Note.class);
+ Note note = Note.GSON.fromJson(json, Note.class);
// note.setReplLoader(replLoader);
// note.jobListenerFactory = jobListenerFactory;
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java
index cd94180..a4e41a8 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/ZeppelinHubRepo.java
@@ -186,7 +186,7 @@ public class ZeppelinHubRepo implements NotebookRepo {
}
String token = getUserToken(subject.getUser());
String response = restApiClient.get(token, noteId);
- Note note = GSON.fromJson(response, Note.class);
+ Note note = Note.GSON.fromJson(response, Note.class);
if (note == null) {
return EMPTY_NOTE;
}
@@ -245,7 +245,7 @@ public class ZeppelinHubRepo implements NotebookRepo {
String token = getUserToken(subject.getUser());
String response = restApiClient.get(token, endpoint);
- Note note = GSON.fromJson(response, Note.class);
+ Note note = Note.GSON.fromJson(response, Note.class);
if (note == null) {
return EMPTY_NOTE;
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/abdf3352/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
index 0e77846..c7d2a6a 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java
@@ -18,22 +18,16 @@
package org.apache.zeppelin.notebook;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyObject;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
import com.google.common.collect.Lists;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import org.junit.Test;
+import org.mockito.Mockito;
+
import org.apache.zeppelin.display.AngularObject;
import org.apache.zeppelin.display.AngularObjectBuilder;
import org.apache.zeppelin.display.AngularObjectRegistry;
@@ -41,7 +35,6 @@ import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.interpreter.Interpreter;
import org.apache.zeppelin.interpreter.Interpreter.FormType;
import org.apache.zeppelin.interpreter.InterpreterContext;
-import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterResult;
@@ -52,15 +45,19 @@ import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.interpreter.InterpreterSetting.Status;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.resource.ResourcePool;
-import org.apache.zeppelin.scheduler.JobListener;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.apache.zeppelin.user.Credentials;
-import org.junit.Test;
-import java.util.HashMap;
-import java.util.Map;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Mockito;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
public class ParagraphTest {
@Test
@@ -84,7 +81,7 @@ public class ParagraphTest {
assertEquals("md", Paragraph.getRequiredReplName(text));
assertEquals("", Paragraph.getScriptBody(text));
}
-
+
@Test
public void replSingleCharName() {
String text = "%r a";
@@ -232,4 +229,13 @@ public class ParagraphTest {
}
+
+ @Test
+ public void deserializeParagraphFromDifferentDateFormat() throws ParseException {
+ String dateString = "2017-09-01 00:01:02.345";
+ Date date = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").parse(dateString);
+ Paragraph paragraph = Note.GSON
+ .fromJson("{\"dateUpdated\":\"" + dateString + "\"}", Paragraph.class);
+ assertEquals(date, paragraph.dateUpdated);
+ }
}