You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2015/11/08 01:30:47 UTC
incubator-zeppelin git commit: ZEPPELIN-367 Sync with secondary
storage when listing notebooks
Repository: incubator-zeppelin
Updated Branches:
refs/heads/master fd2a28853 -> ae08e713e
ZEPPELIN-367 Sync with secondary storage when listing notebooks
This PR resolves https://issues.apache.org/jira/browse/ZEPPELIN-367 by synchronizing with secondary storage when reloading all notebooks into memory. Currently, existing `ZEPPELIN_NOTEBOOK_RELOAD_FROM_STORAGE` from https://github.com/apache/incubator-zeppelin/pull/331 is used as a flag since these functionalities are correlated, however it can be separated into distinct flag as well.
Author: Khalid Huseynov <kh...@nflabs.com>
Closes #375 from khalidhuseynov/sync-on-list and squashes the following commits:
e2cf5d9 [Khalid Huseynov] minor fix
878a23e [Khalid Huseynov] initial working commit
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/ae08e713
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/ae08e713
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/ae08e713
Branch: refs/heads/master
Commit: ae08e713e43ce9ebfb11f0f9cf7a7ea42885d677
Parents: fd2a288
Author: Khalid Huseynov <kh...@nflabs.com>
Authored: Wed Oct 28 20:40:16 2015 +0900
Committer: Lee moon soo <mo...@apache.org>
Committed: Sun Nov 8 09:30:48 2015 +0900
----------------------------------------------------------------------
.../notebook/repo/NotebookRepoSync.java | 6 ++++
.../notebook/repo/NotebookRepoSyncTest.java | 35 ++++++++++++++++++++
2 files changed, 41 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/ae08e713/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
index 288cff9..49f40fc 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
@@ -42,12 +42,15 @@ public class NotebookRepoSync implements NotebookRepo{
private static final int maxRepoNum = 2;
private static final String pushKey = "pushNoteIDs";
private static final String pullKey = "pullNoteIDs";
+ private static ZeppelinConfiguration config;
/**
* @param (conf)
* @throws - Exception
*/
public NotebookRepoSync(ZeppelinConfiguration conf) throws Exception {
+
+ config = conf;
String allStorageClassNames = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE).trim();
if (allStorageClassNames.isEmpty()) {
@@ -72,6 +75,9 @@ public class NotebookRepoSync implements NotebookRepo{
/* by default lists from first repository */
public List<NoteInfo> list() throws IOException {
+ if (config.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_RELOAD_FROM_STORAGE) && getRepoCount() > 1) {
+ sync(0, 1);
+ }
return getRepo(0).list();
}
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/ae08e713/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
index 981ed6e..bfc6561 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
+import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.interpreter.InterpreterFactory;
@@ -180,6 +181,40 @@ public class NotebookRepoSyncTest implements JobListenerFactory{
notebookRepoSync.list(1).get(0).getId()).getLastParagraph().getId());
}
+ @Test
+ public void testSyncOnList() throws IOException {
+
+ /* check that both storage repos are empty */
+ assertTrue(notebookRepoSync.getRepoCount() > 1);
+ assertEquals(0, notebookRepoSync.list(0).size());
+ assertEquals(0, notebookRepoSync.list(1).size());
+
+ File srcDir = new File("src/test/resources/2A94M5J1Z");
+ File destDir = new File(secNotebookDir + "/2A94M5J1Z");
+
+ /* copy manually new notebook into secondary storage repo and check repos */
+ try {
+ FileUtils.copyDirectory(srcDir, destDir);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ assertEquals(0, notebookRepoSync.list(0).size());
+ assertEquals(1, notebookRepoSync.list(1).size());
+
+ /* Although new notebook is added to secondary storage it's not displayed
+ * on list() with ZEPPELIN_NOTEBOOK_RELOAD_FROM_STORAGE set to false
+ */
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_RELOAD_FROM_STORAGE.getVarName(), "false");
+ assertEquals(0, notebookRepoSync.list().size());
+
+ /* notebook is synced after ZEPPELIN_NOTEBOOK_RELOAD_FROM_STORAGE variable is set to true */
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_RELOAD_FROM_STORAGE.getVarName(), "true");
+ assertEquals(1, notebookRepoSync.list().size());
+
+ assertEquals(1, notebookRepoSync.list(0).size());
+ assertEquals(1, notebookRepoSync.list(1).size());
+ }
+
private void delete(File file){
if(file.isFile()) file.delete();
else if(file.isDirectory()){