You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2018/03/22 13:19:55 UTC
[6/7] zeppelin git commit: ZEPPELIN-3196. Plugin framework for
Zeppelin Engine
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java b/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java
new file mode 100644
index 0000000..7729d52
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/git/src/main/java/org/apache/zeppelin/notebook/repo/GitNotebookRepo.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.api.errors.NoHeadException;
+import org.eclipse.jgit.diff.DiffEntry;
+import org.eclipse.jgit.dircache.DirCache;
+import org.eclipse.jgit.internal.storage.file.FileRepository;
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.eclipse.jgit.treewalk.filter.PathFilter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * NotebookRepo that hosts all the notebook FS in a single Git repo
+ *
+ * This impl intended to be simple and straightforward:
+ * - does not handle branches
+ * - only basic local git file repo, no remote Github push\pull. GitHub integration is
+ * implemented in @see {@link org.apache.zeppelin.notebook.repo.GitNotebookRepo}
+ *
+ * TODO(bzz): add default .gitignore
+ */
+public class GitNotebookRepo extends VFSNotebookRepo implements NotebookRepoWithVersionControl {
+ private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepo.class);
+
+ private String localPath;
+ private Git git;
+
+ public GitNotebookRepo() {
+ super();
+ }
+
+ @VisibleForTesting
+ public GitNotebookRepo(ZeppelinConfiguration conf) throws IOException {
+ this();
+ init(conf);
+ }
+
+ @Override
+ public void init(ZeppelinConfiguration conf) throws IOException {
+ //TODO(zjffdu), it is weird that I can not call super.init directly here, as it would cause
+ //AbstractMethodError
+ this.conf = conf;
+ setNotebookDirectory(conf.getNotebookDir());
+
+ localPath = getRootDir().getName().getPath();
+ LOG.info("Opening a git repo at '{}'", localPath);
+ Repository localRepo = new FileRepository(Joiner.on(File.separator).join(localPath, ".git"));
+ if (!localRepo.getDirectory().exists()) {
+ LOG.info("Git repo {} does not exist, creating a new one", localRepo.getDirectory());
+ localRepo.create();
+ }
+ git = new Git(localRepo);
+ }
+
+ @Override
+ public synchronized void save(Note note, AuthenticationInfo subject) throws IOException {
+ super.save(note, subject);
+ }
+
+ /* implemented as git add+commit
+ * @param pattern is the noteId
+ * @param commitMessage is a commit message (checkpoint message)
+ * (non-Javadoc)
+ * @see org.apache.zeppelin.notebook.repo.VFSNotebookRepo#checkpoint(String, String)
+ */
+ @Override
+ public Revision checkpoint(String pattern, String commitMessage, AuthenticationInfo subject) {
+ Revision revision = Revision.EMPTY;
+ try {
+ List<DiffEntry> gitDiff = git.diff().call();
+ if (!gitDiff.isEmpty()) {
+ LOG.debug("Changes found for pattern '{}': {}", pattern, gitDiff);
+ DirCache added = git.add().addFilepattern(pattern).call();
+ LOG.debug("{} changes are about to be commited", added.getEntryCount());
+ RevCommit commit = git.commit().setMessage(commitMessage).call();
+ revision = new Revision(commit.getName(), commit.getShortMessage(), commit.getCommitTime());
+ } else {
+ LOG.debug("No changes found {}", pattern);
+ }
+ } catch (GitAPIException e) {
+ LOG.error("Failed to add+commit {} to Git", pattern, e);
+ }
+ return revision;
+ }
+
+ /**
+ * the idea is to:
+ * 1. stash current changes
+ * 2. remember head commit and checkout to the desired revision
+ * 3. get note and checkout back to the head
+ * 4. apply stash on top and remove it
+ */
+ @Override
+ public synchronized Note get(String noteId, String revId, AuthenticationInfo subject)
+ throws IOException {
+ Note note = null;
+ RevCommit stash = null;
+ try {
+ List<DiffEntry> gitDiff = git.diff().setPathFilter(PathFilter.create(noteId)).call();
+ boolean modified = !gitDiff.isEmpty();
+ if (modified) {
+ // stash changes
+ stash = git.stashCreate().call();
+ Collection<RevCommit> stashes = git.stashList().call();
+ LOG.debug("Created stash : {}, stash size : {}", stash, stashes.size());
+ }
+ ObjectId head = git.getRepository().resolve(Constants.HEAD);
+ // checkout to target revision
+ git.checkout().setStartPoint(revId).addPath(noteId).call();
+ // get the note
+ note = super.get(noteId, subject);
+ // checkout back to head
+ git.checkout().setStartPoint(head.getName()).addPath(noteId).call();
+ if (modified && stash != null) {
+ // unstash changes
+ ObjectId applied = git.stashApply().setStashRef(stash.getName()).call();
+ ObjectId dropped = git.stashDrop().setStashRef(0).call();
+ Collection<RevCommit> stashes = git.stashList().call();
+ LOG.debug("Stash applied as : {}, and dropped : {}, stash size: {}", applied, dropped,
+ stashes.size());
+ }
+ } catch (GitAPIException e) {
+ LOG.error("Failed to return note from revision \"{}\"", revId, e);
+ }
+ return note;
+ }
+
+ @Override
+ public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject) {
+ List<Revision> history = Lists.newArrayList();
+ LOG.debug("Listing history for {}:", noteId);
+ try {
+ Iterable<RevCommit> logs = git.log().addPath(noteId).call();
+ for (RevCommit log: logs) {
+ history.add(new Revision(log.getName(), log.getShortMessage(), log.getCommitTime()));
+ LOG.debug(" - ({},{},{})", log.getName(), log.getCommitTime(), log.getFullMessage());
+ }
+ } catch (NoHeadException e) {
+ //when no initial commit exists
+ LOG.warn("No Head found for {}, {}", noteId, e.getMessage());
+ } catch (GitAPIException e) {
+ LOG.error("Failed to get logs for {}", noteId, e);
+ }
+ return history;
+ }
+
+ @Override
+ public Note setNoteRevision(String noteId, String revId, AuthenticationInfo subject)
+ throws IOException {
+ Note revisionNote = get(noteId, revId, subject);
+ if (revisionNote != null) {
+ save(revisionNote, subject);
+ }
+ return revisionNote;
+ }
+
+ @Override
+ public void close() {
+ git.getRepository().close();
+ }
+
+ //DI replacements for Tests
+ protected Git getGit() {
+ return git;
+ }
+
+ void setGit(Git git) {
+ this.git = git;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/git/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/git/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
new file mode 100644
index 0000000..713407b
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/git/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.GitNotebookRepo
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java
new file mode 100644
index 0000000..58d7cc1
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/GitNotebookRepoTest.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.mockito.Mockito.mock;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.interpreter.InterpreterFactory;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.NoteInfo;
+import org.apache.zeppelin.notebook.Paragraph;
+import org.apache.zeppelin.notebook.repo.NotebookRepoWithVersionControl.Revision;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.diff.DiffEntry;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Joiner;
+
+public class GitNotebookRepoTest {
+ private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepoTest.class);
+
+ private static final String TEST_NOTE_ID = "2A94M5J1Z";
+ private static final String TEST_NOTE_ID2 = "2A94M5J2Z";
+
+ private File zeppelinDir;
+ private String notebooksDir;
+ private ZeppelinConfiguration conf;
+ private GitNotebookRepo notebookRepo;
+
+ @Before
+ public void setUp() throws Exception {
+ String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinTest_" + System.currentTimeMillis();
+ zeppelinDir = new File(zpath);
+ zeppelinDir.mkdirs();
+ new File(zeppelinDir, "conf").mkdirs();
+
+ notebooksDir = Joiner.on(File.separator).join(zpath, "notebook");
+ File notebookDir = new File(notebooksDir);
+ notebookDir.mkdirs();
+
+ String testNoteDir = Joiner.on(File.separator).join(notebooksDir, TEST_NOTE_ID);
+ String testNoteDir2 = Joiner.on(File.separator).join(notebooksDir, TEST_NOTE_ID2);
+ FileUtils.copyDirectory(
+ new File(
+ GitNotebookRepoTest.class.getResource(
+ Joiner.on(File.separator).join("", TEST_NOTE_ID)
+ ).getFile()
+ ),
+ new File(testNoteDir));
+ FileUtils.copyDirectory(
+ new File(
+ GitNotebookRepoTest.class.getResource(
+ Joiner.on(File.separator).join("", TEST_NOTE_ID2)
+ ).getFile()
+ ),
+ new File(testNoteDir2)
+ );
+
+ System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), zeppelinDir.getAbsolutePath());
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.GitNotebookRepo");
+
+ conf = ZeppelinConfiguration.create();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (!FileUtils.deleteQuietly(zeppelinDir)) {
+ LOG.error("Failed to delete {} ", zeppelinDir.getName());
+ }
+ }
+
+ @Test
+ public void initNonemptyNotebookDir() throws IOException, GitAPIException {
+ //given - .git does not exit
+ File dotGit = new File(Joiner.on(File.separator).join(notebooksDir, ".git"));
+ assertThat(dotGit.exists()).isEqualTo(false);
+
+ //when
+ notebookRepo = new GitNotebookRepo(conf);
+
+ //then
+ Git git = notebookRepo.getGit();
+ assertThat(git).isNotNull();
+
+ assertThat(dotGit.exists()).isEqualTo(true);
+ assertThat(notebookRepo.list(null)).isNotEmpty();
+
+ List<DiffEntry> diff = git.diff().call();
+ // no commit, diff isn't empty
+ assertThat(diff).isNotEmpty();
+ }
+
+ @Test
+ public void showNotebookHistoryEmptyTest() throws GitAPIException, IOException {
+ //given
+ notebookRepo = new GitNotebookRepo(conf);
+ assertThat(notebookRepo.list(null)).isNotEmpty();
+
+ //when
+ List<Revision> testNotebookHistory = notebookRepo.revisionHistory(TEST_NOTE_ID, null);
+
+ //then
+ //no initial commit, empty history
+ assertThat(testNotebookHistory).isEmpty();
+ }
+
+ @Test
+ public void showNotebookHistoryMultipleNotesTest() throws IOException {
+ //initial checks
+ notebookRepo = new GitNotebookRepo(conf);
+ assertThat(notebookRepo.list(null)).isNotEmpty();
+ assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue();
+ assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID2)).isTrue();
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty();
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null)).isEmpty();
+
+ //add commit to both notes
+ notebookRepo.checkpoint(TEST_NOTE_ID, "first commit, note1", null);
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1);
+ notebookRepo.checkpoint(TEST_NOTE_ID2, "first commit, note2", null);
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1);
+
+ //modify, save and checkpoint first note
+ Note note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
+ Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
+ Map<String, Object> config = p.getConfig();
+ config.put("enabled", true);
+ p.setConfig(config);
+ p.setText("%md note1 test text");
+ notebookRepo.save(note, null);
+ assertThat(notebookRepo.checkpoint(TEST_NOTE_ID, "second commit, note1", null)).isNotNull();
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2);
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1);
+ assertThat(notebookRepo.checkpoint(TEST_NOTE_ID2, "first commit, note2", null))
+ .isEqualTo(Revision.EMPTY);
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(1);
+
+ //modify, save and checkpoint second note
+ note = notebookRepo.get(TEST_NOTE_ID2, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
+ p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
+ config = p.getConfig();
+ config.put("enabled", false);
+ p.setConfig(config);
+ p.setText("%md note2 test text");
+ notebookRepo.save(note, null);
+ assertThat(notebookRepo.checkpoint(TEST_NOTE_ID2, "second commit, note2", null)).isNotNull();
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2);
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID2, null).size()).isEqualTo(2);
+ }
+
+ @Test
+ public void addCheckpointTest() throws IOException {
+ // initial checks
+ notebookRepo = new GitNotebookRepo(conf);
+ assertThat(notebookRepo.list(null)).isNotEmpty();
+ assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue();
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty();
+
+ notebookRepo.checkpoint(TEST_NOTE_ID, "first commit", null);
+ List<Revision> notebookHistoryBefore = notebookRepo.revisionHistory(TEST_NOTE_ID, null);
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isNotEmpty();
+ int initialCount = notebookHistoryBefore.size();
+
+ // add changes to note
+ Note note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
+ Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
+ Map<String, Object> config = p.getConfig();
+ config.put("enabled", true);
+ p.setConfig(config);
+ p.setText("%md checkpoint test text");
+
+ // save and checkpoint note
+ notebookRepo.save(note, null);
+ notebookRepo.checkpoint(TEST_NOTE_ID, "second commit", null);
+
+ // see if commit is added
+ List<Revision> notebookHistoryAfter = notebookRepo.revisionHistory(TEST_NOTE_ID, null);
+ assertThat(notebookHistoryAfter.size()).isEqualTo(initialCount + 1);
+ }
+
+ private boolean containsNote(List<NoteInfo> notes, String noteId) {
+ for (NoteInfo note: notes) {
+ if (note.getId().equals(noteId)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Test
+ public void getRevisionTest() throws IOException {
+ // initial checks
+ notebookRepo = new GitNotebookRepo(conf);
+ assertThat(notebookRepo.list(null)).isNotEmpty();
+ assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue();
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty();
+
+ // add first checkpoint
+ Revision revision_1 = notebookRepo.checkpoint(TEST_NOTE_ID, "first commit", null);
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1);
+ int paragraphCount_1 = notebookRepo.get(TEST_NOTE_ID, null).getParagraphs().size();
+
+ // add paragraph and save
+ Note note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
+ Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
+ Map<String, Object> config = p1.getConfig();
+ config.put("enabled", true);
+ p1.setConfig(config);
+ p1.setText("checkpoint test text");
+ notebookRepo.save(note, null);
+
+ // second checkpoint
+ notebookRepo.checkpoint(TEST_NOTE_ID, "second commit", null);
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2);
+ int paragraphCount_2 = notebookRepo.get(TEST_NOTE_ID, null).getParagraphs().size();
+ assertThat(paragraphCount_2).isEqualTo(paragraphCount_1 + 1);
+
+ // get note from revision 1
+ Note noteRevision_1 = notebookRepo.get(TEST_NOTE_ID, revision_1.id, null);
+ assertThat(noteRevision_1.getParagraphs().size()).isEqualTo(paragraphCount_1);
+
+ // get current note
+ note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
+ assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_2);
+
+ // add one more paragraph and save
+ Paragraph p2 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
+ config.put("enabled", false);
+ p2.setConfig(config);
+ p2.setText("get revision when modified note test text");
+ notebookRepo.save(note, null);
+ note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
+ int paragraphCount_3 = note.getParagraphs().size();
+ assertThat(paragraphCount_3).isEqualTo(paragraphCount_2 + 1);
+
+ // get revision 1 again
+ noteRevision_1 = notebookRepo.get(TEST_NOTE_ID, revision_1.id, null);
+ assertThat(noteRevision_1.getParagraphs().size()).isEqualTo(paragraphCount_1);
+
+ // check that note is unchanged
+ note = notebookRepo.get(TEST_NOTE_ID, null);
+ assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_3);
+ }
+
+ @Test
+ public void getRevisionFailTest() throws IOException {
+ // initial checks
+ notebookRepo = new GitNotebookRepo(conf);
+ assertThat(notebookRepo.list(null)).isNotEmpty();
+ assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue();
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty();
+
+ // add first checkpoint
+ Revision revision_1 = notebookRepo.checkpoint(TEST_NOTE_ID, "first commit", null);
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1);
+ int paragraphCount_1 = notebookRepo.get(TEST_NOTE_ID, null).getParagraphs().size();
+
+ // get current note
+ Note note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
+ assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_1);
+
+ // add one more paragraph and save
+ Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
+ Map<String, Object> config = p1.getConfig();
+ config.put("enabled", true);
+ p1.setConfig(config);
+ p1.setText("get revision when modified note test text");
+ notebookRepo.save(note, null);
+ int paragraphCount_2 = note.getParagraphs().size();
+
+ // get note from revision 1
+ Note noteRevision_1 = notebookRepo.get(TEST_NOTE_ID, revision_1.id, null);
+ assertThat(noteRevision_1.getParagraphs().size()).isEqualTo(paragraphCount_1);
+
+ // get current note
+ note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
+ assertThat(note.getParagraphs().size()).isEqualTo(paragraphCount_2);
+
+ // test for absent revision
+ Revision absentRevision = new Revision("absentId", StringUtils.EMPTY, 0);
+ note = notebookRepo.get(TEST_NOTE_ID, absentRevision.id, null);
+ assertThat(note).isNull();
+ }
+
+ @Test
+ public void setRevisionTest() throws IOException {
+ //create repo and check that note doesn't contain revisions
+ notebookRepo = new GitNotebookRepo(conf);
+ assertThat(notebookRepo.list(null)).isNotEmpty();
+ assertThat(containsNote(notebookRepo.list(null), TEST_NOTE_ID)).isTrue();
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null)).isEmpty();
+
+ // get current note
+ Note note = notebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
+ int paragraphCount_1 = note.getParagraphs().size();
+ LOG.info("initial paragraph count: {}", paragraphCount_1);
+
+ // checkpoint revision1
+ Revision revision1 = notebookRepo.checkpoint(TEST_NOTE_ID, "set revision: first commit", null);
+ //TODO(khalid): change to EMPTY after rebase
+ assertThat(revision1).isNotNull();
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(1);
+
+ // add one more paragraph and save
+ Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
+ Map<String, Object> config = p1.getConfig();
+ config.put("enabled", true);
+ p1.setConfig(config);
+ p1.setText("set revision sample text");
+ notebookRepo.save(note, null);
+ int paragraphCount_2 = note.getParagraphs().size();
+ assertThat(paragraphCount_2).isEqualTo(paragraphCount_1 + 1);
+ LOG.info("paragraph count after modification: {}", paragraphCount_2);
+
+ // checkpoint revision2
+ Revision revision2 = notebookRepo.checkpoint(TEST_NOTE_ID, "set revision: second commit", null);
+ //TODO(khalid): change to EMPTY after rebase
+ assertThat(revision2).isNotNull();
+ assertThat(notebookRepo.revisionHistory(TEST_NOTE_ID, null).size()).isEqualTo(2);
+
+ // set note to revision1
+ Note returnedNote = notebookRepo.setNoteRevision(note.getId(), revision1.id, null);
+ assertThat(returnedNote).isNotNull();
+ assertThat(returnedNote.getParagraphs().size()).isEqualTo(paragraphCount_1);
+
+ // check note from repo
+ Note updatedNote = notebookRepo.get(note.getId(), null);
+ assertThat(updatedNote).isNotNull();
+ assertThat(updatedNote.getParagraphs().size()).isEqualTo(paragraphCount_1);
+
+ // set back to revision2
+ returnedNote = notebookRepo.setNoteRevision(note.getId(), revision2.id, null);
+ assertThat(returnedNote).isNotNull();
+ assertThat(returnedNote.getParagraphs().size()).isEqualTo(paragraphCount_2);
+
+ // check note from repo
+ updatedNote = notebookRepo.get(note.getId(), null);
+ assertThat(updatedNote).isNotNull();
+ assertThat(updatedNote.getParagraphs().size()).isEqualTo(paragraphCount_2);
+
+ // try failure case - set to invalid revision
+ returnedNote = notebookRepo.setNoteRevision(note.getId(), "nonexistent_id", null);
+ assertThat(returnedNote).isNull();
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java
new file mode 100644
index 0000000..738d708
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncInitializationTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+//TODO(zjffdu) move it to zeppelin-zengine
+public class NotebookRepoSyncInitializationTest {
+ private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncInitializationTest.class);
+ private String validFirstStorageClass = "org.apache.zeppelin.notebook.repo.VFSNotebookRepo";
+ private String validSecondStorageClass = "org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock";
+ private String invalidStorageClass = "org.apache.zeppelin.notebook.repo.DummyNotebookRepo";
+ private String validOneStorageConf = validFirstStorageClass;
+ private String validTwoStorageConf = validFirstStorageClass + "," + validSecondStorageClass;
+ private String invalidTwoStorageConf = validFirstStorageClass + "," + invalidStorageClass;
+ private String unsupportedStorageConf = validFirstStorageClass + "," + validSecondStorageClass + "," + validSecondStorageClass;
+ private String emptyStorageConf = "";
+
+ @Before
+ public void setUp(){
+ System.setProperty(ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath());
+ //setup routine
+ }
+
+ @After
+ public void tearDown() {
+ //tear-down routine
+ }
+
+ @Test
+ public void validInitOneStorageTest() throws IOException {
+ // no need to initialize folder due to one storage
+ // set confs
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), validOneStorageConf);
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create();
+ // create repo
+ NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf);
+ // check proper initialization of one storage
+ assertEquals(notebookRepoSync.getRepoCount(), 1);
+ assertTrue(notebookRepoSync.getRepo(0) instanceof VFSNotebookRepo);
+ }
+
+ @Test
+ public void validInitTwoStorageTest() throws IOException {
+ // initialize folders for each storage
+ String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis();
+ File mainZepDir = new File(zpath);
+ mainZepDir.mkdirs();
+ new File(mainZepDir, "conf").mkdirs();
+ String mainNotePath = zpath+"/notebook";
+ String secNotePath = mainNotePath + "_secondary";
+ File mainNotebookDir = new File(mainNotePath);
+ File secNotebookDir = new File(secNotePath);
+ mainNotebookDir.mkdirs();
+ secNotebookDir.mkdirs();
+
+ // set confs
+ System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath());
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath());
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), validTwoStorageConf);
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create();
+ // create repo
+ NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf);
+ // check that both initialized
+ assertEquals(notebookRepoSync.getRepoCount(), 2);
+ assertTrue(notebookRepoSync.getRepo(0) instanceof VFSNotebookRepo);
+ assertTrue(notebookRepoSync.getRepo(1) instanceof VFSNotebookRepoMock);
+ }
+
+ @Test
+ public void invalidInitTwoStorageTest() throws IOException {
+ // set confs
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), invalidTwoStorageConf);
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create();
+ // create repo
+ NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf);
+ // check that second didn't initialize
+ LOG.info(" " + notebookRepoSync.getRepoCount());
+ assertEquals(notebookRepoSync.getRepoCount(), 1);
+ assertTrue(notebookRepoSync.getRepo(0) instanceof VFSNotebookRepo);
+ }
+
+ @Test
+ public void initUnsupportedNumberStoragesTest() throws IOException {
+ // initialize folders for each storage, currently for 2 only
+ String zpath = System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis();
+ File mainZepDir = new File(zpath);
+ mainZepDir.mkdirs();
+ new File(mainZepDir, "conf").mkdirs();
+ String mainNotePath = zpath+"/notebook";
+ String secNotePath = mainNotePath + "_secondary";
+ File mainNotebookDir = new File(mainNotePath);
+ File secNotebookDir = new File(secNotePath);
+ mainNotebookDir.mkdirs();
+ secNotebookDir.mkdirs();
+
+ // set confs
+ System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath());
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath());
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), unsupportedStorageConf);
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create();
+ // create repo
+ NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf);
+ // check that first two storages initialized instead of three
+ assertEquals(notebookRepoSync.getRepoCount(), 2);
+ assertTrue(notebookRepoSync.getRepo(0) instanceof VFSNotebookRepo);
+ assertTrue(notebookRepoSync.getRepo(1) instanceof VFSNotebookRepoMock);
+ }
+
+ @Test
+ public void initEmptyStorageTest() throws IOException {
+ // set confs
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), emptyStorageConf);
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create();
+ // create repo
+ NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf);
+ // check initialization of one default storage
+ assertEquals(notebookRepoSync.getRepoCount(), 1);
+ assertTrue(notebookRepoSync.getRepo(0) instanceof NotebookRepoWithVersionControl);
+ }
+
+ @Test
+ public void initOneDummyStorageTest() throws IOException {
+ // set confs
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), invalidStorageClass);
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create();
+ // create repo
+ NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf);
+ // check initialization of one default storage instead of invalid one
+ assertEquals(notebookRepoSync.getRepoCount(), 1);
+ assertTrue(notebookRepoSync.getRepo(0) instanceof NotebookRepo);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
new file mode 100644
index 0000000..274c08a
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.dep.DependencyResolver;
+import org.apache.zeppelin.display.AngularObjectRegistryListener;
+import org.apache.zeppelin.helium.ApplicationEventListener;
+import org.apache.zeppelin.interpreter.InterpreterFactory;
+import org.apache.zeppelin.interpreter.InterpreterResultMessage;
+import org.apache.zeppelin.interpreter.InterpreterSettingManager;
+import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener;
+import org.apache.zeppelin.notebook.JobListenerFactory;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.Notebook;
+import org.apache.zeppelin.notebook.NotebookAuthorization;
+import org.apache.zeppelin.notebook.Paragraph;
+import org.apache.zeppelin.notebook.ParagraphJobListener;
+import org.apache.zeppelin.scheduler.Job;
+import org.apache.zeppelin.scheduler.Job.Status;
+import org.apache.zeppelin.scheduler.SchedulerFactory;
+import org.apache.zeppelin.search.SearchService;
+import org.apache.zeppelin.storage.ConfigStorage;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.apache.zeppelin.user.Credentials;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.quartz.SchedulerException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+
+
+//TODO(zjffdu) move it to zeppelin-zengine
+public class NotebookRepoSyncTest implements JobListenerFactory {
+
+ private File mainZepDir;
+ private ZeppelinConfiguration conf;
+ private SchedulerFactory schedulerFactory;
+ private File mainNotebookDir;
+ private File secNotebookDir;
+ private Notebook notebookSync;
+ private NotebookRepoSync notebookRepoSync;
+ private InterpreterFactory factory;
+ private InterpreterSettingManager interpreterSettingManager;
+ private DependencyResolver depResolver;
+ private SearchService search;
+ private NotebookAuthorization notebookAuthorization;
+ private Credentials credentials;
+ private AuthenticationInfo anonymous;
+ private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class);
+
+ @Before
+ public void setUp() throws Exception {
+ String zpath = System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis();
+ mainZepDir = new File(zpath);
+ mainZepDir.mkdirs();
+ new File(mainZepDir, "conf").mkdirs();
+ String mainNotePath = zpath+"/notebook";
+ String secNotePath = mainNotePath + "_secondary";
+ mainNotebookDir = new File(mainNotePath);
+ secNotebookDir = new File(secNotePath);
+ mainNotebookDir.mkdirs();
+ secNotebookDir.mkdirs();
+
+ System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath());
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath());
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo,org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock");
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC.getVarName(), "false");
+ System.setProperty(ConfVars.ZEPPELIN_CONFIG_FS_DIR.getVarName(), mainZepDir.getAbsolutePath() + "/conf");
+ System.setProperty(ConfVars.ZEPPELIN_PLUGINS_DIR.getVarName(), new File("../../../plugins").getAbsolutePath());
+
+ LOG.info("main Note dir : " + mainNotePath);
+ LOG.info("secondary note dir : " + secNotePath);
+ conf = ZeppelinConfiguration.create();
+
+ ConfigStorage.reset();
+
+ this.schedulerFactory = SchedulerFactory.singleton();
+
+ depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo");
+ interpreterSettingManager = new InterpreterSettingManager(conf,
+ mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class));
+ factory = new InterpreterFactory(interpreterSettingManager);
+
+ search = mock(SearchService.class);
+ notebookRepoSync = new NotebookRepoSync(conf);
+ notebookAuthorization = NotebookAuthorization.init(conf);
+ credentials = new Credentials(conf.credentialsPersist(), conf.getCredentialsPath(), null);
+ notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search,
+ notebookAuthorization, credentials);
+ anonymous = new AuthenticationInfo("anonymous");
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ delete(mainZepDir);
+ }
+
+ @Test
+ public void testRepoCount() throws IOException {
+ assertTrue(notebookRepoSync.getMaxRepoNum() >= notebookRepoSync.getRepoCount());
+ }
+
+ @Test
+ public void testSyncOnCreate() throws IOException {
+ /* check that both storage systems are empty */
+ assertTrue(notebookRepoSync.getRepoCount() > 1);
+ assertEquals(0, notebookRepoSync.list(0, anonymous).size());
+ assertEquals(0, notebookRepoSync.list(1, anonymous).size());
+
+ /* create note */
+ Note note = notebookSync.createNote(anonymous);
+
+ // check that automatically saved on both storages
+ assertEquals(1, notebookRepoSync.list(0, anonymous).size());
+ assertEquals(1, notebookRepoSync.list(1, anonymous).size());
+ assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId());
+
+ notebookSync.removeNote(notebookRepoSync.list(0, null).get(0).getId(), anonymous);
+ }
+
+ @Test
+ public void testSyncOnDelete() throws IOException {
+ /* create note */
+ assertTrue(notebookRepoSync.getRepoCount() > 1);
+ assertEquals(0, notebookRepoSync.list(0, anonymous).size());
+ assertEquals(0, notebookRepoSync.list(1, anonymous).size());
+
+ Note note = notebookSync.createNote(anonymous);
+
+ /* check that created in both storage systems */
+ assertEquals(1, notebookRepoSync.list(0, anonymous).size());
+ assertEquals(1, notebookRepoSync.list(1, anonymous).size());
+ assertEquals(notebookRepoSync.list(0, anonymous).get(0).getId(),notebookRepoSync.list(1, anonymous).get(0).getId());
+
+ /* remove Note */
+ notebookSync.removeNote(notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous);
+
+ /* check that deleted in both storages */
+ assertEquals(0, notebookRepoSync.list(0, anonymous).size());
+ assertEquals(0, notebookRepoSync.list(1, anonymous).size());
+
+ }
+
+ @Test
+ public void testSyncUpdateMain() throws IOException {
+
+ /* create note */
+ Note note = notebookSync.createNote(anonymous);
+ Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
+ Map config = p1.getConfig();
+ config.put("enabled", true);
+ p1.setConfig(config);
+ p1.setText("hello world");
+
+ /* new paragraph exists in note instance */
+ assertEquals(1, note.getParagraphs().size());
+
+ /* new paragraph not yet saved into storages */
+ assertEquals(0, notebookRepoSync.get(0,
+ notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size());
+ assertEquals(0, notebookRepoSync.get(1,
+ notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size());
+
+ /* save to storage under index 0 (first storage) */
+ notebookRepoSync.save(0, note, anonymous);
+
+ /* check paragraph saved to first storage */
+ assertEquals(1, notebookRepoSync.get(0,
+ notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getParagraphs().size());
+ /* check paragraph isn't saved to second storage */
+ assertEquals(0, notebookRepoSync.get(1,
+ notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size());
+ /* apply sync */
+ notebookRepoSync.sync(null);
+ /* check whether added to second storage */
+ assertEquals(1, notebookRepoSync.get(1,
+ notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getParagraphs().size());
+ /* check whether same paragraph id */
+ assertEquals(p1.getId(), notebookRepoSync.get(0,
+ notebookRepoSync.list(0, anonymous).get(0).getId(), anonymous).getLastParagraph().getId());
+ assertEquals(p1.getId(), notebookRepoSync.get(1,
+ notebookRepoSync.list(1, anonymous).get(0).getId(), anonymous).getLastParagraph().getId());
+ notebookRepoSync.remove(note.getId(), anonymous);
+ }
+
+ @Test
+ public void testSyncOnReloadedList() throws IOException {
+ /* check that both storage repos are empty */
+ assertTrue(notebookRepoSync.getRepoCount() > 1);
+ assertEquals(0, notebookRepoSync.list(0, anonymous).size());
+ assertEquals(0, notebookRepoSync.list(1, anonymous).size());
+
+ File srcDir = new File("src/test/resources/2A94M5J1Z");
+ File destDir = new File(secNotebookDir + "/2A94M5J1Z");
+
+ /* copy manually new notebook into secondary storage repo and check repos */
+ try {
+ FileUtils.copyDirectory(srcDir, destDir);
+ } catch (IOException e) {
+ LOG.error(e.toString(), e);
+ }
+ assertEquals(0, notebookRepoSync.list(0, anonymous).size());
+ assertEquals(1, notebookRepoSync.list(1, anonymous).size());
+
+ // After reloading notebooks repos should be synchronized
+ notebookSync.reloadAllNotes(anonymous);
+ assertEquals(1, notebookRepoSync.list(0, anonymous).size());
+ assertEquals(1, notebookRepoSync.list(1, anonymous).size());
+ }
+
+ @Test
+ public void testOneWaySyncOnReloadedList() throws IOException, SchedulerException {
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath());
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC.getVarName(), "true");
+ conf = ZeppelinConfiguration.create();
+ notebookRepoSync = new NotebookRepoSync(conf);
+ notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search,
+ notebookAuthorization, credentials);
+
+ // check that both storage repos are empty
+ assertTrue(notebookRepoSync.getRepoCount() > 1);
+ assertEquals(0, notebookRepoSync.list(0, null).size());
+ assertEquals(0, notebookRepoSync.list(1, null).size());
+
+ File srcDir = new File("src/test/resources/2A94M5J1Z");
+ File destDir = new File(secNotebookDir + "/2A94M5J1Z");
+
+ // copy manually new notebook into secondary storage repo and check repos
+ try {
+ FileUtils.copyDirectory(srcDir, destDir);
+ } catch (IOException e) {
+ LOG.error(e.toString(), e);
+ }
+ assertEquals(0, notebookRepoSync.list(0, null).size());
+ assertEquals(1, notebookRepoSync.list(1, null).size());
+
+ // after reloading the notebook should be wiped from secondary storage
+ notebookSync.reloadAllNotes(null);
+ assertEquals(0, notebookRepoSync.list(0, null).size());
+ assertEquals(0, notebookRepoSync.list(1, null).size());
+
+ destDir = new File(mainNotebookDir + "/2A94M5J1Z");
+
+ // copy manually new notebook into primary storage repo and check repos
+ try {
+ FileUtils.copyDirectory(srcDir, destDir);
+ } catch (IOException e) {
+ LOG.error(e.toString(), e);
+ }
+ assertEquals(1, notebookRepoSync.list(0, null).size());
+ assertEquals(0, notebookRepoSync.list(1, null).size());
+
+ // after reloading notebooks repos should be synchronized
+ notebookSync.reloadAllNotes(null);
+ assertEquals(1, notebookRepoSync.list(0, null).size());
+ assertEquals(1, notebookRepoSync.list(1, null).size());
+ }
+
+ @Test
+ public void testCheckpointOneStorage() throws IOException, SchedulerException {
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.GitNotebookRepo");
+ ZeppelinConfiguration vConf = ZeppelinConfiguration.create();
+
+ NotebookRepoSync vRepoSync = new NotebookRepoSync(vConf);
+ Notebook vNotebookSync = new Notebook(vConf, vRepoSync, schedulerFactory, factory, interpreterSettingManager, this, search,
+ notebookAuthorization, credentials);
+
+ // one git versioned storage initialized
+ assertThat(vRepoSync.getRepoCount()).isEqualTo(1);
+ assertThat(vRepoSync.getRepo(0)).isInstanceOf(GitNotebookRepo.class);
+
+ GitNotebookRepo gitRepo = (GitNotebookRepo) vRepoSync.getRepo(0);
+
+ // no notes
+ assertThat(vRepoSync.list(anonymous).size()).isEqualTo(0);
+ // create note
+ Note note = vNotebookSync.createNote(anonymous);
+ assertThat(vRepoSync.list(anonymous).size()).isEqualTo(1);
+
+ String noteId = vRepoSync.list(anonymous).get(0).getId();
+ // first checkpoint
+ vRepoSync.checkpoint(noteId, "checkpoint message", anonymous);
+ int vCount = gitRepo.revisionHistory(noteId, anonymous).size();
+ assertThat(vCount).isEqualTo(1);
+
+ Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
+ Map<String, Object> config = p.getConfig();
+ config.put("enabled", true);
+ p.setConfig(config);
+ p.setText("%md checkpoint test");
+
+ // save and checkpoint again
+ vRepoSync.save(note, anonymous);
+ vRepoSync.checkpoint(noteId, "checkpoint message 2", anonymous);
+ assertThat(gitRepo.revisionHistory(noteId, anonymous).size()).isEqualTo(vCount + 1);
+ notebookRepoSync.remove(note.getId(), anonymous);
+ }
+
+ @Test
+ public void testSyncWithAcl() throws IOException {
+ /* scenario 1 - note exists with acl on main storage */
+ AuthenticationInfo user1 = new AuthenticationInfo("user1");
+ Note note = notebookSync.createNote(user1);
+ assertEquals(0, note.getParagraphs().size());
+
+ // saved on both storages
+ assertEquals(1, notebookRepoSync.list(0, null).size());
+ assertEquals(1, notebookRepoSync.list(1, null).size());
+
+ /* check that user1 is the only owner */
+ NotebookAuthorization authInfo = NotebookAuthorization.getInstance();
+ Set<String> entity = new HashSet<String>();
+ entity.add(user1.getUser());
+ assertEquals(true, authInfo.isOwner(note.getId(), entity));
+ assertEquals(1, authInfo.getOwners(note.getId()).size());
+ assertEquals(0, authInfo.getReaders(note.getId()).size());
+ assertEquals(0, authInfo.getRunners(note.getId()).size());
+ assertEquals(0, authInfo.getWriters(note.getId()).size());
+
+ /* update note and save on secondary storage */
+ Paragraph p1 = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
+ p1.setText("hello world");
+ assertEquals(1, note.getParagraphs().size());
+ notebookRepoSync.save(1, note, null);
+
+ /* check paragraph isn't saved into first storage */
+ assertEquals(0, notebookRepoSync.get(0,
+ notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size());
+ /* check paragraph is saved into second storage */
+ assertEquals(1, notebookRepoSync.get(1,
+ notebookRepoSync.list(1, null).get(0).getId(), null).getParagraphs().size());
+
+ /* now sync by user1 */
+ notebookRepoSync.sync(user1);
+
+ /* check that note updated and acl are same on main storage*/
+ assertEquals(1, notebookRepoSync.get(0,
+ notebookRepoSync.list(0, null).get(0).getId(), null).getParagraphs().size());
+ assertEquals(true, authInfo.isOwner(note.getId(), entity));
+ assertEquals(1, authInfo.getOwners(note.getId()).size());
+ assertEquals(0, authInfo.getReaders(note.getId()).size());
+ assertEquals(0, authInfo.getRunners(note.getId()).size());
+ assertEquals(0, authInfo.getWriters(note.getId()).size());
+
+ /* scenario 2 - note doesn't exist on main storage */
+ /* remove from main storage */
+ notebookRepoSync.remove(0, note.getId(), user1);
+ assertEquals(0, notebookRepoSync.list(0, null).size());
+ assertEquals(1, notebookRepoSync.list(1, null).size());
+ authInfo.removeNote(note.getId());
+ assertEquals(0, authInfo.getOwners(note.getId()).size());
+ assertEquals(0, authInfo.getReaders(note.getId()).size());
+ assertEquals(0, authInfo.getRunners(note.getId()).size());
+ assertEquals(0, authInfo.getWriters(note.getId()).size());
+
+ /* now sync - should bring note from secondary storage with added acl */
+ notebookRepoSync.sync(user1);
+ assertEquals(1, notebookRepoSync.list(0, null).size());
+ assertEquals(1, notebookRepoSync.list(1, null).size());
+ assertEquals(1, authInfo.getOwners(note.getId()).size());
+ assertEquals(1, authInfo.getReaders(note.getId()).size());
+ assertEquals(1, authInfo.getRunners(note.getId()).size());
+ assertEquals(1, authInfo.getWriters(note.getId()).size());
+ assertEquals(true, authInfo.isOwner(note.getId(), entity));
+ assertEquals(true, authInfo.isReader(note.getId(), entity));
+ assertEquals(true, authInfo.isRunner(note.getId(), entity));
+ assertEquals(true, authInfo.isWriter(note.getId(), entity));
+ }
+
+ static void delete(File file){
+ if(file.isFile()) file.delete();
+ else if(file.isDirectory()){
+ File [] files = file.listFiles();
+ if(files!=null && files.length>0){
+ for(File f : files){
+ delete(f);
+ }
+ }
+ file.delete();
+ }
+ }
+
+ @Override
+ public ParagraphJobListener getParagraphJobListener(Note note) {
+ return new ParagraphJobListener(){
+
+ @Override
+ public void onOutputAppend(Paragraph paragraph, int idx, String output) {
+
+ }
+
+ @Override
+ public void onOutputUpdate(Paragraph paragraph, int idx, InterpreterResultMessage msg) {
+
+ }
+
+ @Override
+ public void onOutputUpdateAll(Paragraph paragraph, List<InterpreterResultMessage> msgs) {
+
+ }
+
+ @Override
+ public void onProgressUpdate(Job job, int progress) {
+ }
+
+ @Override
+ public void onStatusChange(Job job, Status before, Status after) {
+
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java
new file mode 100644
index 0000000..7f450df
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/git/src/test/java/org/apache/zeppelin/notebook/repo/mock/VFSNotebookRepoMock.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.notebook.repo.mock;
+
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
+
+import java.io.IOException;
+
+public class VFSNotebookRepoMock extends VFSNotebookRepo {
+
+ public VFSNotebookRepoMock() {
+ String secNotebookDir = ZeppelinConfiguration.create().getNotebookDir() + "_secondary";
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), secNotebookDir);
+ }
+
+ public void init(ZeppelinConfiguration conf) throws IOException {
+ super.init(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J1Z/note.json
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J1Z/note.json b/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J1Z/note.json
new file mode 100644
index 0000000..785ccea
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J1Z/note.json
@@ -0,0 +1,341 @@
+{
+ "paragraphs": [
+ {
+ "text": "%md\n## Welcome to Zeppelin.\n##### This is a live tutorial, you can run the code yourself. (Shift-Enter to Run)",
+ "config": {
+ "colWidth": 12.0,
+ "graph": {
+ "mode": "table",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [],
+ "values": [],
+ "groups": [],
+ "scatter": {}
+ },
+ "editorHide": true
+ },
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1423836981412_-1007008116",
+ "id": "20150213-231621_168813393",
+ "result": {
+ "code": "SUCCESS",
+ "type": "HTML",
+ "msg": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a live tutorial, you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n"
+ },
+ "dateCreated": "Feb 13, 2015 11:16:21 PM",
+ "dateStarted": "Apr 1, 2015 9:11:09 PM",
+ "dateFinished": "Apr 1, 2015 9:11:10 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "title": "Load data into table",
+ "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval bankText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n).toDF()\nbank.registerTempTable(\"bank\")",
+ "config": {
+ "colWidth": 12.0,
+ "graph": {
+ "mode": "table",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [],
+ "values": [],
+ "groups": [],
+ "scatter": {}
+ },
+ "title": true
+ },
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1423500779206_-1502780787",
+ "id": "20150210-015259_1403135953",
+ "result": {
+ "code": "SUCCESS",
+ "type": "TEXT",
+ "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[32] at parallelize at \u003cconsole\u003e:65\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string, marital: string, education: string, balance: int]\n"
+ },
+ "dateCreated": "Feb 10, 2015 1:52:59 AM",
+ "dateStarted": "Jul 3, 2015 1:43:40 PM",
+ "dateFinished": "Jul 3, 2015 1:43:45 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "text": "%sql \nselect age, count(1) value\nfrom bank \nwhere age \u003c 30 \ngroup by age \norder by age",
+ "config": {
+ "colWidth": 4.0,
+ "graph": {
+ "mode": "multiBarChart",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [
+ {
+ "name": "age",
+ "index": 0.0,
+ "aggr": "sum"
+ }
+ ],
+ "values": [
+ {
+ "name": "value",
+ "index": 1.0,
+ "aggr": "sum"
+ }
+ ],
+ "groups": [],
+ "scatter": {
+ "xAxis": {
+ "name": "age",
+ "index": 0.0,
+ "aggr": "sum"
+ },
+ "yAxis": {
+ "name": "value",
+ "index": 1.0,
+ "aggr": "sum"
+ }
+ }
+ }
+ },
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1423500782552_-1439281894",
+ "id": "20150210-015302_1492795503",
+ "result": {
+ "code": "SUCCESS",
+ "type": "TABLE",
+ "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n"
+ },
+ "dateCreated": "Feb 10, 2015 1:53:02 AM",
+ "dateStarted": "Jul 3, 2015 1:43:17 PM",
+ "dateFinished": "Jul 3, 2015 1:43:23 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere age \u003c ${maxAge\u003d30} \ngroup by age \norder by age",
+ "config": {
+ "colWidth": 4.0,
+ "graph": {
+ "mode": "multiBarChart",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [
+ {
+ "name": "age",
+ "index": 0.0,
+ "aggr": "sum"
+ }
+ ],
+ "values": [
+ {
+ "name": "value",
+ "index": 1.0,
+ "aggr": "sum"
+ }
+ ],
+ "groups": [],
+ "scatter": {
+ "xAxis": {
+ "name": "age",
+ "index": 0.0,
+ "aggr": "sum"
+ },
+ "yAxis": {
+ "name": "value",
+ "index": 1.0,
+ "aggr": "sum"
+ }
+ }
+ }
+ },
+ "settings": {
+ "params": {
+ "maxAge": "35"
+ },
+ "forms": {
+ "maxAge": {
+ "name": "maxAge",
+ "defaultValue": "30",
+ "hidden": false
+ }
+ }
+ },
+ "jobName": "paragraph_1423720444030_-1424110477",
+ "id": "20150212-145404_867439529",
+ "result": {
+ "code": "SUCCESS",
+ "type": "TABLE",
+ "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n30\t150\n31\t199\n32\t224\n33\t186\n34\t231\n"
+ },
+ "dateCreated": "Feb 12, 2015 2:54:04 PM",
+ "dateStarted": "Jul 3, 2015 1:43:28 PM",
+ "dateFinished": "Jul 3, 2015 1:43:29 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere marital\u003d\"${marital\u003dsingle,single|divorced|married}\" \ngroup by age \norder by age",
+ "config": {
+ "colWidth": 4.0,
+ "graph": {
+ "mode": "multiBarChart",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [
+ {
+ "name": "age",
+ "index": 0.0,
+ "aggr": "sum"
+ }
+ ],
+ "values": [
+ {
+ "name": "value",
+ "index": 1.0,
+ "aggr": "sum"
+ }
+ ],
+ "groups": [],
+ "scatter": {
+ "xAxis": {
+ "name": "age",
+ "index": 0.0,
+ "aggr": "sum"
+ },
+ "yAxis": {
+ "name": "value",
+ "index": 1.0,
+ "aggr": "sum"
+ }
+ }
+ }
+ },
+ "settings": {
+ "params": {
+ "marital": "single"
+ },
+ "forms": {
+ "marital": {
+ "name": "marital",
+ "defaultValue": "single",
+ "options": [
+ {
+ "value": "single"
+ },
+ {
+ "value": "divorced"
+ },
+ {
+ "value": "married"
+ }
+ ],
+ "hidden": false
+ }
+ }
+ },
+ "jobName": "paragraph_1423836262027_-210588283",
+ "id": "20150213-230422_1600658137",
+ "result": {
+ "code": "SUCCESS",
+ "type": "TABLE",
+ "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t17\n24\t13\n25\t33\n26\t56\n27\t64\n28\t78\n29\t56\n30\t92\n31\t86\n32\t105\n33\t61\n34\t75\n35\t46\n36\t50\n37\t43\n38\t44\n39\t30\n40\t25\n41\t19\n42\t23\n43\t21\n44\t20\n45\t15\n46\t14\n47\t12\n48\t12\n49\t11\n50\t8\n51\t6\n52\t9\n53\t4\n55\t3\n56\t3\n57\t2\n58\t7\n59\t2\n60\t5\n66\t2\n69\t1\n"
+ },
+ "dateCreated": "Feb 13, 2015 11:04:22 PM",
+ "dateStarted": "Jul 3, 2015 1:43:33 PM",
+ "dateFinished": "Jul 3, 2015 1:43:34 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!",
+ "config": {
+ "colWidth": 12.0,
+ "graph": {
+ "mode": "table",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [],
+ "values": [],
+ "groups": [],
+ "scatter": {}
+ },
+ "editorHide": true
+ },
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1423836268492_216498320",
+ "id": "20150213-230428_1231780373",
+ "result": {
+ "code": "SUCCESS",
+ "type": "HTML",
+ "msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n"
+ },
+ "dateCreated": "Feb 13, 2015 11:04:28 PM",
+ "dateStarted": "Apr 1, 2015 9:12:18 PM",
+ "dateFinished": "Apr 1, 2015 9:12:18 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```",
+ "config": {
+ "colWidth": 12.0,
+ "graph": {
+ "mode": "table",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [],
+ "values": [],
+ "groups": [],
+ "scatter": {}
+ },
+ "editorHide": true
+ },
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1427420818407_872443482",
+ "id": "20150326-214658_12335843",
+ "result": {
+ "code": "SUCCESS",
+ "type": "HTML",
+ "msg": "\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n"
+ },
+ "dateCreated": "Mar 26, 2015 9:46:58 PM",
+ "dateStarted": "Jul 3, 2015 1:44:56 PM",
+ "dateFinished": "Jul 3, 2015 1:44:56 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "config": {},
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1435955447812_-158639899",
+ "id": "20150703-133047_853701097",
+ "dateCreated": "Jul 3, 2015 1:30:47 PM",
+ "status": "READY",
+ "progressUpdateIntervalMs": 500
+ }
+ ],
+ "name": "Zeppelin Tutorial",
+ "id": "2A94M5J1Z",
+ "angularObjects": {},
+ "config": {
+ "looknfeel": "default"
+ },
+ "info": {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J2Z/note.json
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J2Z/note.json b/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J2Z/note.json
new file mode 100644
index 0000000..79fe35c
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/git/src/test/resources/2A94M5J2Z/note.json
@@ -0,0 +1,87 @@
+{
+ "paragraphs": [
+ {
+ "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!",
+ "config": {
+ "colWidth": 12.0,
+ "graph": {
+ "mode": "table",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [],
+ "values": [],
+ "groups": [],
+ "scatter": {}
+ },
+ "editorHide": true
+ },
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1423836268492_216498320",
+ "id": "20150213-230428_1231780373",
+ "result": {
+ "code": "SUCCESS",
+ "type": "HTML",
+ "msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n"
+ },
+ "dateCreated": "Feb 13, 2015 11:04:28 PM",
+ "dateStarted": "Apr 1, 2015 9:12:18 PM",
+ "dateFinished": "Apr 1, 2015 9:12:18 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```",
+ "config": {
+ "colWidth": 12.0,
+ "graph": {
+ "mode": "table",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [],
+ "values": [],
+ "groups": [],
+ "scatter": {}
+ },
+ "editorHide": true
+ },
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1427420818407_872443482",
+ "id": "20150326-214658_12335843",
+ "result": {
+ "code": "SUCCESS",
+ "type": "HTML",
+ "msg": "\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n"
+ },
+ "dateCreated": "Mar 26, 2015 9:46:58 PM",
+ "dateStarted": "Jul 3, 2015 1:44:56 PM",
+ "dateFinished": "Jul 3, 2015 1:44:56 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "config": {},
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1435955447812_-158639899",
+ "id": "20150703-133047_853701097",
+ "dateCreated": "Jul 3, 2015 1:30:47 PM",
+ "status": "READY",
+ "progressUpdateIntervalMs": 500
+ }
+ ],
+ "name": "Sample note - excerpt from Zeppelin Tutorial",
+ "id": "2A94M5J2Z",
+ "angularObjects": {},
+ "config": {
+ "looknfeel": "default"
+ },
+ "info": {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/git/src/test/resources/log4j.properties b/zeppelin-plugins/notebookrepo/git/src/test/resources/log4j.properties
new file mode 100644
index 0000000..661632b
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/git/src/test/resources/log4j.properties
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %c:%L - %m%n
+#log4j.appender.stdout.layout.ConversionPattern=
+#%5p [%t] (%F:%L) - %m%n
+#%-4r [%t] %-5p %c %x - %m%n
+#
+
+# Root logger option
+log4j.rootLogger=INFO, stdout
+
+log4j.logger.org.apache.zeppelin.notebook.repo=DEBUG
+
+#mute some noisy guys
+log4j.logger.org.apache.hadoop.mapred=WARN
+log4j.logger.org.apache.hadoop.hive.ql=WARN
+log4j.logger.org.apache.hadoop.hive.metastore=WARN
+log4j.logger.org.apache.haadoop.hive.service.HiveServer=WARN
+
+log4j.logger.org.quartz=WARN
+log4j.logger.DataNucleus=WARN
+log4j.logger.DataNucleus.MetaData=ERROR
+log4j.logger.DataNucleus.Datastore=ERROR
+
+# Log all JDBC parameters
+log4j.logger.org.hibernate.type=ALL
+
+log4j.logger.org.apache.zeppelin.interpreter=DEBUG
+log4j.logger.org.apache.zeppelin.scheduler=DEBUG
+log4j.logger.org.apache.zeppelin.plugin=DEBUG
+
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/github/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/github/pom.xml b/zeppelin-plugins/notebookrepo/github/pom.xml
new file mode 100644
index 0000000..3673efb
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/github/pom.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>zengine-plugins-parent</artifactId>
+ <groupId>org.apache.zeppelin</groupId>
+ <version>0.9.0-SNAPSHOT</version>
+ <relativePath>../../../zeppelin-plugins</relativePath>
+ </parent>
+
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>notebookrepo-github</artifactId>
+ <packaging>jar</packaging>
+ <version>0.9.0-SNAPSHOT</version>
+ <name>Zeppelin: Plugin GitHubNotebookRepo</name>
+ <description>NotebookRepo implementation based on GitHub</description>
+
+ <properties>
+ <plugin.name>NotebookRepo/GitHubNotebookRepo</plugin.name>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>notebookrepo-git</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java
new file mode 100644
index 0000000..6052e5f
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepo.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.PullCommand;
+import org.eclipse.jgit.api.PushCommand;
+import org.eclipse.jgit.api.RemoteAddCommand;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.transport.URIish;
+import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+/**
+ * GitHub integration to store notebooks in a GitHub repository.
+ * It uses the same simple logic implemented in @see
+ * {@link org.apache.zeppelin.notebook.repo.GitNotebookRepo}
+ *
+ * The logic for updating the local repository from the remote repository is the following:
+ * - When the <code>GitHubNotebookRepo</code> is initialized
+ * - When pushing the changes to the remote repository
+ *
+ * The logic for updating the remote repository on GitHub from local repository is the following:
+ * - When commit the changes (saving the notebook)
+ */
+public class GitHubNotebookRepo extends GitNotebookRepo {
+ private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepo.class);
+ private ZeppelinConfiguration zeppelinConfiguration;
+ private Git git;
+
+ public GitHubNotebookRepo(ZeppelinConfiguration conf) throws IOException {
+ super(conf);
+
+ this.git = super.getGit();
+ this.zeppelinConfiguration = conf;
+
+ configureRemoteStream();
+ pullFromRemoteStream();
+ }
+
+ @Override
+ public Revision checkpoint(String pattern, String commitMessage, AuthenticationInfo subject) {
+ Revision revision = super.checkpoint(pattern, commitMessage, subject);
+
+ updateRemoteStream();
+
+ return revision;
+ }
+
+ private void configureRemoteStream() {
+ try {
+ LOG.debug("Setting up remote stream");
+ RemoteAddCommand remoteAddCommand = git.remoteAdd();
+ remoteAddCommand.setName(zeppelinConfiguration.getZeppelinNotebookGitRemoteOrigin());
+ remoteAddCommand.setUri(new URIish(zeppelinConfiguration.getZeppelinNotebookGitURL()));
+ remoteAddCommand.call();
+ } catch (GitAPIException e) {
+ LOG.error("Error configuring GitHub", e);
+ } catch (URISyntaxException e) {
+ LOG.error("Error in GitHub URL provided", e);
+ }
+ }
+
+ private void updateRemoteStream() {
+ LOG.debug("Updating remote stream");
+
+ pullFromRemoteStream();
+ pushToRemoteSteam();
+ }
+
+ private void pullFromRemoteStream() {
+ try {
+ LOG.debug("Pull latest changed from remote stream");
+ PullCommand pullCommand = git.pull();
+ pullCommand.setCredentialsProvider(
+ new UsernamePasswordCredentialsProvider(
+ zeppelinConfiguration.getZeppelinNotebookGitUsername(),
+ zeppelinConfiguration.getZeppelinNotebookGitAccessToken()
+ )
+ );
+
+ pullCommand.call();
+
+ } catch (GitAPIException e) {
+ LOG.error("Error when pulling latest changes from remote repository", e);
+ }
+ }
+
+ private void pushToRemoteSteam() {
+ try {
+ LOG.debug("Push latest changed from remote stream");
+ PushCommand pushCommand = git.push();
+ pushCommand.setCredentialsProvider(
+ new UsernamePasswordCredentialsProvider(
+ zeppelinConfiguration.getZeppelinNotebookGitUsername(),
+ zeppelinConfiguration.getZeppelinNotebookGitAccessToken()
+ )
+ );
+
+ pushCommand.call();
+ } catch (GitAPIException e) {
+ LOG.error("Error when pushing latest changes from remote repository", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/github/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/github/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/github/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
new file mode 100644
index 0000000..25d5145
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/github/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.GitHubNotebookRepo
\ No newline at end of file