You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2018/03/22 13:19:54 UTC
[5/7] zeppelin git commit: ZEPPELIN-3196. Plugin framework for
Zeppelin Engine
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java
new file mode 100644
index 0000000..04a59ad
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/github/src/test/java/org/apache/zeppelin/notebook/repo/GitHubNotebookRepoTest.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+
+import com.google.common.base.Joiner;
+import org.apache.commons.io.FileUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.interpreter.InterpreterFactory;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.Paragraph;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.api.errors.GitAPIException;
+import org.eclipse.jgit.internal.storage.file.FileRepository;
+import org.eclipse.jgit.lib.Repository;
+import org.eclipse.jgit.revwalk.RevCommit;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Iterator;
+
+import static org.mockito.Mockito.mock;
+
+/**
+ * This tests the remote Git tracking for notebooks. The tests uses two local Git repositories created locally
+ * to handle the tracking of Git actions (pushes and pulls). The repositories are:
+ * 1. The first repository is considered as a remote that mimics a remote GitHub directory
+ * 2. The second repository is considered as the local notebook repository
+ */
+public class GitHubNotebookRepoTest {
+ private static final Logger LOG = LoggerFactory.getLogger(GitHubNotebookRepoTest.class);
+
+ private static final String TEST_NOTE_ID = "2A94M5J1Z";
+
+ private File remoteZeppelinDir;
+ private File localZeppelinDir;
+ private String localNotebooksDir;
+ private String remoteNotebooksDir;
+ private ZeppelinConfiguration conf;
+ private GitHubNotebookRepo gitHubNotebookRepo;
+ private RevCommit firstCommitRevision;
+ private Git remoteGit;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = ZeppelinConfiguration.create();
+
+ String remoteRepositoryPath = System.getProperty("java.io.tmpdir") + "/ZeppelinTestRemote_" +
+ System.currentTimeMillis();
+ String localRepositoryPath = System.getProperty("java.io.tmpdir") + "/ZeppelinTest_" +
+ System.currentTimeMillis();
+
+ // Create a fake remote notebook Git repository locally in another directory
+ remoteZeppelinDir = new File(remoteRepositoryPath);
+ remoteZeppelinDir.mkdirs();
+
+ // Create a local repository for notebooks
+ localZeppelinDir = new File(localRepositoryPath);
+ localZeppelinDir.mkdirs();
+
+ // Notebooks directory (for both the remote and local directories)
+ localNotebooksDir = Joiner.on(File.separator).join(localRepositoryPath, "notebook");
+ remoteNotebooksDir = Joiner.on(File.separator).join(remoteRepositoryPath, "notebook");
+
+ File notebookDir = new File(localNotebooksDir);
+ notebookDir.mkdirs();
+
+ // Copy the test notebook directory from the test/resources/2A94M5J1Z folder to the fake remote Git directory
+ String remoteTestNoteDir = Joiner.on(File.separator).join(remoteNotebooksDir, TEST_NOTE_ID);
+ FileUtils.copyDirectory(
+ new File(
+ GitHubNotebookRepoTest.class.getResource(
+ Joiner.on(File.separator).join("", TEST_NOTE_ID)
+ ).getFile()
+ ), new File(remoteTestNoteDir)
+ );
+
+ // Create the fake remote Git repository
+ Repository remoteRepository = new FileRepository(Joiner.on(File.separator).join(remoteNotebooksDir, ".git"));
+ remoteRepository.create();
+
+ remoteGit = new Git(remoteRepository);
+ remoteGit.add().addFilepattern(".").call();
+ firstCommitRevision = remoteGit.commit().setMessage("First commit from remote repository").call();
+
+ // Set the Git and Git configurations
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), remoteZeppelinDir.getAbsolutePath());
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath());
+
+ // Set the GitHub configurations
+ System.setProperty(
+ ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(),
+ "org.apache.zeppelin.notebook.repo.GitHubNotebookRepo");
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL.getVarName(),
+ remoteNotebooksDir + File.separator + ".git");
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_USERNAME.getVarName(), "token");
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_ACCESS_TOKEN.getVarName(),
+ "access-token");
+
+ // Create the Notebook repository (configured for the local repository)
+ gitHubNotebookRepo = new GitHubNotebookRepo(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ // Cleanup the temporary folders uses as Git repositories
+ File[] temporaryFolders = { remoteZeppelinDir, localZeppelinDir };
+
+ for(File temporaryFolder : temporaryFolders) {
+ if (!FileUtils.deleteQuietly(temporaryFolder))
+ LOG.error("Failed to delete {} ", temporaryFolder.getName());
+ }
+ }
+
+ @Test
+ /**
+ * Test the case when the Notebook repository is created, it pulls the latest changes from the remote repository
+ */
+ public void pullChangesFromRemoteRepositoryOnLoadingNotebook() throws IOException, GitAPIException {
+ NotebookRepoWithVersionControl.Revision firstHistoryRevision = gitHubNotebookRepo.revisionHistory(TEST_NOTE_ID, null).get(0);
+
+ assert(this.firstCommitRevision.getName().equals(firstHistoryRevision.id));
+ }
+
+ @Test
+ /**
+ * Test the case when the check-pointing (add new files and commit) it also pulls the latest changes from the
+ * remote repository
+ */
+ public void pullChangesFromRemoteRepositoryOnCheckpointing() throws GitAPIException, IOException {
+ // Create a new commit in the remote repository
+ RevCommit secondCommitRevision = remoteGit.commit().setMessage("Second commit from remote repository").call();
+
+ // Add a new paragraph to the local repository
+ addParagraphToNotebook(TEST_NOTE_ID);
+
+ // Commit and push the changes to remote repository
+ NotebookRepoWithVersionControl.Revision thirdCommitRevision = gitHubNotebookRepo.checkpoint(
+ TEST_NOTE_ID, "Third commit from local repository", null);
+
+ // Check all the commits as seen from the local repository. The commits are ordered chronologically. The last
+ // commit is the first in the commit logs.
+ Iterator<RevCommit> revisions = gitHubNotebookRepo.getGit().log().all().call().iterator();
+
+ revisions.next(); // The Merge `master` commit after pushing to the remote repository
+
+ assert(thirdCommitRevision.id.equals(revisions.next().getName())); // The local commit after adding the paragraph
+
+ // The second commit done on the remote repository
+ assert(secondCommitRevision.getName().equals(revisions.next().getName()));
+
+ // The first commit done on the remote repository
+ assert(firstCommitRevision.getName().equals(revisions.next().getName()));
+ }
+
+ @Test
+ /**
+ * Test the case when the check-pointing (add new files and commit) it pushes the local commits to the remote
+ * repository
+ */
+ public void pushLocalChangesToRemoteRepositoryOnCheckpointing() throws IOException, GitAPIException {
+ // Add a new paragraph to the local repository
+ addParagraphToNotebook(TEST_NOTE_ID);
+
+ // Commit and push the changes to remote repository
+ NotebookRepoWithVersionControl.Revision secondCommitRevision = gitHubNotebookRepo.checkpoint(
+ TEST_NOTE_ID, "Second commit from local repository", null);
+
+ // Check all the commits as seen from the remote repository. The commits are ordered chronologically. The last
+ // commit is the first in the commit logs.
+ Iterator<RevCommit> revisions = remoteGit.log().all().call().iterator();
+
+ assert(secondCommitRevision.id.equals(revisions.next().getName())); // The local commit after adding the paragraph
+
+ // The first commit done on the remote repository
+ assert(firstCommitRevision.getName().equals(revisions.next().getName()));
+ }
+
+ private void addParagraphToNotebook(String noteId) throws IOException {
+ Note note = gitHubNotebookRepo.get(TEST_NOTE_ID, null);
+ note.setInterpreterFactory(mock(InterpreterFactory.class));
+ Paragraph paragraph = note.addNewParagraph(AuthenticationInfo.ANONYMOUS);
+ paragraph.setText("%md text");
+ gitHubNotebookRepo.save(note, null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J1Z/note.json
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J1Z/note.json b/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J1Z/note.json
new file mode 100644
index 0000000..785ccea
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J1Z/note.json
@@ -0,0 +1,341 @@
+{
+ "paragraphs": [
+ {
+ "text": "%md\n## Welcome to Zeppelin.\n##### This is a live tutorial, you can run the code yourself. (Shift-Enter to Run)",
+ "config": {
+ "colWidth": 12.0,
+ "graph": {
+ "mode": "table",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [],
+ "values": [],
+ "groups": [],
+ "scatter": {}
+ },
+ "editorHide": true
+ },
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1423836981412_-1007008116",
+ "id": "20150213-231621_168813393",
+ "result": {
+ "code": "SUCCESS",
+ "type": "HTML",
+ "msg": "\u003ch2\u003eWelcome to Zeppelin.\u003c/h2\u003e\n\u003ch5\u003eThis is a live tutorial, you can run the code yourself. (Shift-Enter to Run)\u003c/h5\u003e\n"
+ },
+ "dateCreated": "Feb 13, 2015 11:16:21 PM",
+ "dateStarted": "Apr 1, 2015 9:11:09 PM",
+ "dateFinished": "Apr 1, 2015 9:11:10 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "title": "Load data into table",
+ "text": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\n\n// Zeppelin creates and injects sc (SparkContext) and sqlContext (HiveContext or SqlContext)\n// So you don\u0027t need create them manually\n\n// load bank data\nval bankText \u003d sc.parallelize(\n IOUtils.toString(\n new URL(\"https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv\"),\n Charset.forName(\"utf8\")).split(\"\\n\"))\n\ncase class Bank(age: Integer, job: String, marital: String, education: String, balance: Integer)\n\nval bank \u003d bankText.map(s \u003d\u003e s.split(\";\")).filter(s \u003d\u003e s(0) !\u003d \"\\\"age\\\"\").map(\n s \u003d\u003e Bank(s(0).toInt, \n s(1).replaceAll(\"\\\"\", \"\"),\n s(2).replaceAll(\"\\\"\", \"\"),\n s(3).replaceAll(\"\\\"\", \"\"),\n s(5).replaceAll(\"\\\"\", \"\").toInt\n )\n).toDF()\nbank.registerTempTable(\"bank\")",
+ "config": {
+ "colWidth": 12.0,
+ "graph": {
+ "mode": "table",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [],
+ "values": [],
+ "groups": [],
+ "scatter": {}
+ },
+ "title": true
+ },
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1423500779206_-1502780787",
+ "id": "20150210-015259_1403135953",
+ "result": {
+ "code": "SUCCESS",
+ "type": "TEXT",
+ "msg": "import org.apache.commons.io.IOUtils\nimport java.net.URL\nimport java.nio.charset.Charset\nbankText: org.apache.spark.rdd.RDD[String] \u003d ParallelCollectionRDD[32] at parallelize at \u003cconsole\u003e:65\ndefined class Bank\nbank: org.apache.spark.sql.DataFrame \u003d [age: int, job: string, marital: string, education: string, balance: int]\n"
+ },
+ "dateCreated": "Feb 10, 2015 1:52:59 AM",
+ "dateStarted": "Jul 3, 2015 1:43:40 PM",
+ "dateFinished": "Jul 3, 2015 1:43:45 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "text": "%sql \nselect age, count(1) value\nfrom bank \nwhere age \u003c 30 \ngroup by age \norder by age",
+ "config": {
+ "colWidth": 4.0,
+ "graph": {
+ "mode": "multiBarChart",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [
+ {
+ "name": "age",
+ "index": 0.0,
+ "aggr": "sum"
+ }
+ ],
+ "values": [
+ {
+ "name": "value",
+ "index": 1.0,
+ "aggr": "sum"
+ }
+ ],
+ "groups": [],
+ "scatter": {
+ "xAxis": {
+ "name": "age",
+ "index": 0.0,
+ "aggr": "sum"
+ },
+ "yAxis": {
+ "name": "value",
+ "index": 1.0,
+ "aggr": "sum"
+ }
+ }
+ }
+ },
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1423500782552_-1439281894",
+ "id": "20150210-015302_1492795503",
+ "result": {
+ "code": "SUCCESS",
+ "type": "TABLE",
+ "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n"
+ },
+ "dateCreated": "Feb 10, 2015 1:53:02 AM",
+ "dateStarted": "Jul 3, 2015 1:43:17 PM",
+ "dateFinished": "Jul 3, 2015 1:43:23 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere age \u003c ${maxAge\u003d30} \ngroup by age \norder by age",
+ "config": {
+ "colWidth": 4.0,
+ "graph": {
+ "mode": "multiBarChart",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [
+ {
+ "name": "age",
+ "index": 0.0,
+ "aggr": "sum"
+ }
+ ],
+ "values": [
+ {
+ "name": "value",
+ "index": 1.0,
+ "aggr": "sum"
+ }
+ ],
+ "groups": [],
+ "scatter": {
+ "xAxis": {
+ "name": "age",
+ "index": 0.0,
+ "aggr": "sum"
+ },
+ "yAxis": {
+ "name": "value",
+ "index": 1.0,
+ "aggr": "sum"
+ }
+ }
+ }
+ },
+ "settings": {
+ "params": {
+ "maxAge": "35"
+ },
+ "forms": {
+ "maxAge": {
+ "name": "maxAge",
+ "defaultValue": "30",
+ "hidden": false
+ }
+ }
+ },
+ "jobName": "paragraph_1423720444030_-1424110477",
+ "id": "20150212-145404_867439529",
+ "result": {
+ "code": "SUCCESS",
+ "type": "TABLE",
+ "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t20\n24\t24\n25\t44\n26\t77\n27\t94\n28\t103\n29\t97\n30\t150\n31\t199\n32\t224\n33\t186\n34\t231\n"
+ },
+ "dateCreated": "Feb 12, 2015 2:54:04 PM",
+ "dateStarted": "Jul 3, 2015 1:43:28 PM",
+ "dateFinished": "Jul 3, 2015 1:43:29 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "text": "%sql \nselect age, count(1) value \nfrom bank \nwhere marital\u003d\"${marital\u003dsingle,single|divorced|married}\" \ngroup by age \norder by age",
+ "config": {
+ "colWidth": 4.0,
+ "graph": {
+ "mode": "multiBarChart",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [
+ {
+ "name": "age",
+ "index": 0.0,
+ "aggr": "sum"
+ }
+ ],
+ "values": [
+ {
+ "name": "value",
+ "index": 1.0,
+ "aggr": "sum"
+ }
+ ],
+ "groups": [],
+ "scatter": {
+ "xAxis": {
+ "name": "age",
+ "index": 0.0,
+ "aggr": "sum"
+ },
+ "yAxis": {
+ "name": "value",
+ "index": 1.0,
+ "aggr": "sum"
+ }
+ }
+ }
+ },
+ "settings": {
+ "params": {
+ "marital": "single"
+ },
+ "forms": {
+ "marital": {
+ "name": "marital",
+ "defaultValue": "single",
+ "options": [
+ {
+ "value": "single"
+ },
+ {
+ "value": "divorced"
+ },
+ {
+ "value": "married"
+ }
+ ],
+ "hidden": false
+ }
+ }
+ },
+ "jobName": "paragraph_1423836262027_-210588283",
+ "id": "20150213-230422_1600658137",
+ "result": {
+ "code": "SUCCESS",
+ "type": "TABLE",
+ "msg": "age\tvalue\n19\t4\n20\t3\n21\t7\n22\t9\n23\t17\n24\t13\n25\t33\n26\t56\n27\t64\n28\t78\n29\t56\n30\t92\n31\t86\n32\t105\n33\t61\n34\t75\n35\t46\n36\t50\n37\t43\n38\t44\n39\t30\n40\t25\n41\t19\n42\t23\n43\t21\n44\t20\n45\t15\n46\t14\n47\t12\n48\t12\n49\t11\n50\t8\n51\t6\n52\t9\n53\t4\n55\t3\n56\t3\n57\t2\n58\t7\n59\t2\n60\t5\n66\t2\n69\t1\n"
+ },
+ "dateCreated": "Feb 13, 2015 11:04:22 PM",
+ "dateStarted": "Jul 3, 2015 1:43:33 PM",
+ "dateFinished": "Jul 3, 2015 1:43:34 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!",
+ "config": {
+ "colWidth": 12.0,
+ "graph": {
+ "mode": "table",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [],
+ "values": [],
+ "groups": [],
+ "scatter": {}
+ },
+ "editorHide": true
+ },
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1423836268492_216498320",
+ "id": "20150213-230428_1231780373",
+ "result": {
+ "code": "SUCCESS",
+ "type": "HTML",
+ "msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n"
+ },
+ "dateCreated": "Feb 13, 2015 11:04:28 PM",
+ "dateStarted": "Apr 1, 2015 9:12:18 PM",
+ "dateFinished": "Apr 1, 2015 9:12:18 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```",
+ "config": {
+ "colWidth": 12.0,
+ "graph": {
+ "mode": "table",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [],
+ "values": [],
+ "groups": [],
+ "scatter": {}
+ },
+ "editorHide": true
+ },
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1427420818407_872443482",
+ "id": "20150326-214658_12335843",
+ "result": {
+ "code": "SUCCESS",
+ "type": "HTML",
+ "msg": "\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n"
+ },
+ "dateCreated": "Mar 26, 2015 9:46:58 PM",
+ "dateStarted": "Jul 3, 2015 1:44:56 PM",
+ "dateFinished": "Jul 3, 2015 1:44:56 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "config": {},
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1435955447812_-158639899",
+ "id": "20150703-133047_853701097",
+ "dateCreated": "Jul 3, 2015 1:30:47 PM",
+ "status": "READY",
+ "progressUpdateIntervalMs": 500
+ }
+ ],
+ "name": "Zeppelin Tutorial",
+ "id": "2A94M5J1Z",
+ "angularObjects": {},
+ "config": {
+ "looknfeel": "default"
+ },
+ "info": {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J2Z/note.json
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J2Z/note.json b/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J2Z/note.json
new file mode 100644
index 0000000..79fe35c
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/github/src/test/resources/2A94M5J2Z/note.json
@@ -0,0 +1,87 @@
+{
+ "paragraphs": [
+ {
+ "text": "%md\n## Congratulations, it\u0027s done.\n##### You can create your own notebook in \u0027Notebook\u0027 menu. Good luck!",
+ "config": {
+ "colWidth": 12.0,
+ "graph": {
+ "mode": "table",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [],
+ "values": [],
+ "groups": [],
+ "scatter": {}
+ },
+ "editorHide": true
+ },
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1423836268492_216498320",
+ "id": "20150213-230428_1231780373",
+ "result": {
+ "code": "SUCCESS",
+ "type": "HTML",
+ "msg": "\u003ch2\u003eCongratulations, it\u0027s done.\u003c/h2\u003e\n\u003ch5\u003eYou can create your own notebook in \u0027Notebook\u0027 menu. Good luck!\u003c/h5\u003e\n"
+ },
+ "dateCreated": "Feb 13, 2015 11:04:28 PM",
+ "dateStarted": "Apr 1, 2015 9:12:18 PM",
+ "dateFinished": "Apr 1, 2015 9:12:18 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "text": "%md\n\nAbout bank data\n\n```\nCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n```",
+ "config": {
+ "colWidth": 12.0,
+ "graph": {
+ "mode": "table",
+ "height": 300.0,
+ "optionOpen": false,
+ "keys": [],
+ "values": [],
+ "groups": [],
+ "scatter": {}
+ },
+ "editorHide": true
+ },
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1427420818407_872443482",
+ "id": "20150326-214658_12335843",
+ "result": {
+ "code": "SUCCESS",
+ "type": "HTML",
+ "msg": "\u003cp\u003eAbout bank data\u003c/p\u003e\n\u003cpre\u003e\u003ccode\u003eCitation Request:\n This dataset is public available for research. The details are described in [Moro et al., 2011]. \n Please include this citation if you plan to use this database:\n\n [Moro et al., 2011] S. Moro, R. Laureano and P. Cortez. Using Data Mining for Bank Direct Marketing: An Application of the CRISP-DM Methodology. \n In P. Novais et al. (Eds.), Proceedings of the European Simulation and Modelling Conference - ESM\u00272011, pp. 117-121, Guimarães, Portugal, October, 2011. EUROSIS.\n\n Available at: [pdf] http://hdl.handle.net/1822/14838\n [bib] http://www3.dsi.uminho.pt/pcortez/bib/2011-esm-1.txt\n\u003c/code\u003e\u003c/pre\u003e\n"
+ },
+ "dateCreated": "Mar 26, 2015 9:46:58 PM",
+ "dateStarted": "Jul 3, 2015 1:44:56 PM",
+ "dateFinished": "Jul 3, 2015 1:44:56 PM",
+ "status": "FINISHED",
+ "progressUpdateIntervalMs": 500
+ },
+ {
+ "config": {},
+ "settings": {
+ "params": {},
+ "forms": {}
+ },
+ "jobName": "paragraph_1435955447812_-158639899",
+ "id": "20150703-133047_853701097",
+ "dateCreated": "Jul 3, 2015 1:30:47 PM",
+ "status": "READY",
+ "progressUpdateIntervalMs": 500
+ }
+ ],
+ "name": "Sample note - excerpt from Zeppelin Tutorial",
+ "id": "2A94M5J2Z",
+ "angularObjects": {},
+ "config": {
+ "looknfeel": "default"
+ },
+ "info": {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/mongodb/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/mongodb/pom.xml b/zeppelin-plugins/notebookrepo/mongodb/pom.xml
new file mode 100644
index 0000000..2e0f90f
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/mongodb/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>zengine-plugins-parent</artifactId>
+ <groupId>org.apache.zeppelin</groupId>
+ <version>0.9.0-SNAPSHOT</version>
+ <relativePath>../../../zeppelin-plugins</relativePath>
+ </parent>
+
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>notebookrepo-mongodb</artifactId>
+ <packaging>jar</packaging>
+ <version>0.9.0-SNAPSHOT</version>
+ <name>Zeppelin: Plugin MongoNotebookRepo</name>
+ <description>NotebookRepo implementation based on Mongodb</description>
+
+ <properties>
+ <plugin.name>NotebookRepo/MongoNotebookRepo</plugin.name>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>notebookrepo-vfs</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.mongodb</groupId>
+ <artifactId>mongo-java-driver</artifactId>
+ <version>3.4.1</version>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java b/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java
new file mode 100644
index 0000000..618568d
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/mongodb/src/main/java/org/apache/zeppelin/notebook/repo/MongoNotebookRepo.java
@@ -0,0 +1,222 @@
+package org.apache.zeppelin.notebook.repo;
+
+import static com.mongodb.client.model.Filters.eq;
+import static com.mongodb.client.model.Filters.in;
+import static com.mongodb.client.model.Filters.type;
+
+import com.mongodb.MongoBulkWriteException;
+import com.mongodb.MongoClient;
+import com.mongodb.MongoClientURI;
+import com.mongodb.bulk.BulkWriteError;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoCursor;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.InsertManyOptions;
+import com.mongodb.client.model.UpdateOptions;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.NoteInfo;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.bson.BsonType;
+import org.bson.Document;
+import org.bson.types.ObjectId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Backend for storing Notebook on MongoDB
+ */
+public class MongoNotebookRepo implements NotebookRepo {
+ private static final Logger LOG = LoggerFactory.getLogger(MongoNotebookRepo.class);
+
+ private ZeppelinConfiguration conf;
+ private MongoClient mongo;
+ private MongoDatabase db;
+ private MongoCollection<Document> coll;
+
+ public MongoNotebookRepo() {
+
+ }
+
+ public void init(ZeppelinConfiguration conf) throws IOException {
+ this.conf = conf;
+
+ mongo = new MongoClient(new MongoClientURI(conf.getMongoUri()));
+ db = mongo.getDatabase(conf.getMongoDatabase());
+ coll = db.getCollection(conf.getMongoCollection());
+
+ if (conf.getMongoAutoimport()) {
+ // import local notes into MongoDB
+ insertFileSystemNotes();
+ }
+ }
+
+ /**
+ * If environment variable ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT is true,
+ * this method will insert local notes into MongoDB on startup.
+ * If a note already exists in MongoDB, skip it.
+ */
+ private void insertFileSystemNotes() throws IOException {
+ LinkedList<Document> docs = new LinkedList<>(); // docs to be imported
+ NotebookRepo vfsRepo = new VFSNotebookRepo();
+ vfsRepo.init(conf);
+ List<NoteInfo> infos = vfsRepo.list(null);
+ // collect notes to be imported
+ for (NoteInfo info : infos) {
+ Note note = vfsRepo.get(info.getId(), null);
+ Document doc = noteToDocument(note);
+ docs.add(doc);
+ }
+
+ /*
+ * 'ordered(false)' option allows to proceed bulk inserting even though
+ * there are duplicated documents. The duplicated documents will be skipped
+ * and print a WARN log.
+ */
+ try {
+ coll.insertMany(docs, new InsertManyOptions().ordered(false));
+ } catch (MongoBulkWriteException e) {
+ printDuplicatedException(e); //print duplicated document warning log
+ }
+
+ vfsRepo.close(); // it does nothing for now but maybe in the future...
+ }
+
+ /**
+ * MongoBulkWriteException contains error messages that inform
+ * which documents were duplicated. This method catches those ID and print them.
+ * @param e
+ */
+ private void printDuplicatedException(MongoBulkWriteException e) {
+ List<BulkWriteError> errors = e.getWriteErrors();
+ for (BulkWriteError error : errors) {
+ String msg = error.getMessage();
+ Pattern pattern = Pattern.compile("[A-Z0-9]{9}"); // regex for note ID
+ Matcher matcher = pattern.matcher(msg);
+ if (matcher.find()) { // if there were a note ID
+ String noteId = matcher.group();
+ LOG.warn("Note " + noteId + " not inserted since already exists in MongoDB");
+ }
+ }
+ }
+
+ @Override
+ public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
+ syncId();
+
+ List<NoteInfo> infos = new LinkedList<>();
+ MongoCursor<Document> cursor = coll.find().iterator();
+
+ while (cursor.hasNext()) {
+ Document doc = cursor.next();
+ Note note = documentToNote(doc);
+ NoteInfo info = new NoteInfo(note);
+ infos.add(info);
+ }
+
+ cursor.close();
+
+ return infos;
+ }
+
+ /**
+ * Find documents of which type of _id is object ID, and change it to note ID.
+ * Since updating _id field is not allowed, remove original documents and insert
+ * new ones with string _id(note ID)
+ */
+ private void syncId() {
+ // find documents whose id type is object id
+ MongoCursor<Document> cursor = coll.find(type("_id", BsonType.OBJECT_ID)).iterator();
+ // if there is no such document, exit
+ if (!cursor.hasNext())
+ return;
+
+ List<ObjectId> oldDocIds = new LinkedList<>(); // document ids need to update
+ List<Document> updatedDocs = new LinkedList<>(); // new documents to be inserted
+
+ while (cursor.hasNext()) {
+ Document doc = cursor.next();
+ // store original _id
+ ObjectId oldId = doc.getObjectId("_id");
+ oldDocIds.add(oldId);
+ // store the document with string _id (note id)
+ String noteId = doc.getString("id");
+ doc.put("_id", noteId);
+ updatedDocs.add(doc);
+ }
+
+ coll.insertMany(updatedDocs);
+ coll.deleteMany(in("_id", oldDocIds));
+
+ cursor.close();
+ }
+
+ /**
+ * Convert document to note
+ */
+ private Note documentToNote(Document doc) {
+ // document to JSON
+ String json = doc.toJson();
+ // JSON to note
+ return Note.fromJson(json);
+ }
+
+ /**
+ * Convert note to document
+ */
+ private Document noteToDocument(Note note) {
+ // note to JSON
+ String json = note.toJson();
+ // JSON to document
+ Document doc = Document.parse(json);
+ // set object id as note id
+ doc.put("_id", note.getId());
+ return doc;
+ }
+
+ @Override
+ public Note get(String noteId, AuthenticationInfo subject) throws IOException {
+ Document doc = coll.find(eq("_id", noteId)).first();
+
+ if (doc == null) {
+ throw new IOException("Note " + noteId + "not found");
+ }
+
+ return documentToNote(doc);
+ }
+
+ @Override
+ public void save(Note note, AuthenticationInfo subject) throws IOException {
+ Document doc = noteToDocument(note);
+ coll.replaceOne(eq("_id", note.getId()), doc, new UpdateOptions().upsert(true));
+ }
+
+ @Override
+ public void remove(String noteId, AuthenticationInfo subject) throws IOException {
+ coll.deleteOne(eq("_id", noteId));
+ }
+
+ @Override
+ public void close() {
+ mongo.close();
+ }
+
+ @Override
+ public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
+ LOG.warn("Method not implemented");
+ return Collections.emptyList();
+ }
+
+ @Override
+ public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) {
+ LOG.warn("Method not implemented");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/mongodb/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/mongodb/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/mongodb/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
new file mode 100644
index 0000000..e1943b7
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/mongodb/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.MongoNotebookRepo
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/s3/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/s3/pom.xml b/zeppelin-plugins/notebookrepo/s3/pom.xml
new file mode 100644
index 0000000..ee533f2
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/s3/pom.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>zengine-plugins-parent</artifactId>
+ <groupId>org.apache.zeppelin</groupId>
+ <version>0.9.0-SNAPSHOT</version>
+ <relativePath>../../../zeppelin-plugins</relativePath>
+ </parent>
+
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>notebookrepo-s3</artifactId>
+ <packaging>jar</packaging>
+ <version>0.9.0-SNAPSHOT</version>
+ <name>Zeppelin: Plugin S3NotebookRepo</name>
+ <description>NotebookRepo implementation based on S3</description>
+
+ <properties>
+ <aws.sdk.s3.version>1.10.62</aws.sdk.s3.version>
+ <plugin.name>NotebookRepo/S3NotebookRepo</plugin.name>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-s3</artifactId>
+ <version>${aws.sdk.s3.version}</version>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
new file mode 100644
index 0000000..364943c
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/S3NotebookRepo.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.NoteInfo;
+import org.apache.zeppelin.notebook.Paragraph;
+import org.apache.zeppelin.scheduler.Job.Status;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.ClientConfigurationFactory;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3EncryptionClient;
+import com.amazonaws.services.s3.model.CryptoConfiguration;
+import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.regions.Region;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+/**
+ * Backend for storing Notebooks on S3
+ */
+public class S3NotebookRepo implements NotebookRepo {
+ private static final Logger LOG = LoggerFactory.getLogger(S3NotebookRepo.class);
+
+ // Use a credential provider chain so that instance profiles can be utilized
+ // on an EC2 instance. The order of locations where credentials are searched
+ // is documented here
+ //
+ // http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/
+ // auth/DefaultAWSCredentialsProviderChain.html
+ //
+ // In summary, the order is:
+ //
+ // 1. Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+ // 2. Java System Properties - aws.accessKeyId and aws.secretKey
+ // 3. Credential profiles file at the default location (~/.aws/credentials)
+ // shared by all AWS SDKs and the AWS CLI
+ // 4. Instance profile credentials delivered through the Amazon EC2 metadata service
+ private AmazonS3 s3client;
+ private String bucketName;
+ private String user;
+ private boolean useServerSideEncryption;
+ private ZeppelinConfiguration conf;
+
+ public S3NotebookRepo() {
+
+ }
+
+ public void init(ZeppelinConfiguration conf) throws IOException {
+ this.conf = conf;
+ bucketName = conf.getS3BucketName();
+ user = conf.getS3User();
+ useServerSideEncryption = conf.isS3ServerSideEncryption();
+
+ // always use the default provider chain
+ AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
+ CryptoConfiguration cryptoConf = new CryptoConfiguration();
+ String keyRegion = conf.getS3KMSKeyRegion();
+
+ if (StringUtils.isNotBlank(keyRegion)) {
+ cryptoConf.setAwsKmsRegion(Region.getRegion(Regions.fromName(keyRegion)));
+ }
+
+ ClientConfiguration cliConf = createClientConfiguration();
+
+ // see if we should be encrypting data in S3
+ String kmsKeyID = conf.getS3KMSKeyID();
+ if (kmsKeyID != null) {
+ // use the AWS KMS to encrypt data
+ KMSEncryptionMaterialsProvider emp = new KMSEncryptionMaterialsProvider(kmsKeyID);
+ this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf);
+ }
+ else if (conf.getS3EncryptionMaterialsProviderClass() != null) {
+ // use a custom encryption materials provider class
+ EncryptionMaterialsProvider emp = createCustomProvider(conf);
+ this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf);
+ }
+ else {
+ // regular S3
+ this.s3client = new AmazonS3Client(credentialsProvider, cliConf);
+ }
+
+ // set S3 endpoint to use
+ s3client.setEndpoint(conf.getS3Endpoint());
+ }
+
+ /**
+ * Create an instance of a custom encryption materials provider class
+ * which supplies encryption keys to use when reading/writing data in S3.
+ */
+ private EncryptionMaterialsProvider createCustomProvider(ZeppelinConfiguration conf)
+ throws IOException {
+ // use a custom encryption materials provider class
+ String empClassname = conf.getS3EncryptionMaterialsProviderClass();
+ EncryptionMaterialsProvider emp;
+ try {
+ Object empInstance = Class.forName(empClassname).newInstance();
+ if (empInstance instanceof EncryptionMaterialsProvider) {
+ emp = (EncryptionMaterialsProvider) empInstance;
+ }
+ else {
+ throw new IOException("Class " + empClassname + " does not implement "
+ + EncryptionMaterialsProvider.class.getName());
+ }
+ }
+ catch (Exception e) {
+ throw new IOException("Unable to instantiate encryption materials provider class "
+ + empClassname + ": " + e, e);
+ }
+
+ return emp;
+ }
+
+ /**
+ * Create AWS client configuration and return it.
+ * @return AWS client configuration
+ */
+ private ClientConfiguration createClientConfiguration() {
+ ClientConfigurationFactory configFactory = new ClientConfigurationFactory();
+ ClientConfiguration config = configFactory.getConfig();
+
+ String s3SignerOverride = conf.getS3SignerOverride();
+ if (StringUtils.isNotBlank(s3SignerOverride)) {
+ config.setSignerOverride(s3SignerOverride);
+ }
+
+ return config;
+ }
+
+ @Override
+ public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
+ List<NoteInfo> infos = new LinkedList<>();
+ NoteInfo info;
+ try {
+ ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
+ .withBucketName(bucketName)
+ .withPrefix(user + "/" + "notebook");
+ ObjectListing objectListing;
+ do {
+ objectListing = s3client.listObjects(listObjectsRequest);
+ for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
+ if (objectSummary.getKey().endsWith("note.json")) {
+ info = getNoteInfo(objectSummary.getKey());
+ if (info != null) {
+ infos.add(info);
+ }
+ }
+ }
+ listObjectsRequest.setMarker(objectListing.getNextMarker());
+ } while (objectListing.isTruncated());
+ } catch (AmazonClientException ace) {
+ throw new IOException("Unable to list objects in S3: " + ace, ace);
+ }
+ return infos;
+ }
+
+ private Note getNote(String key) throws IOException {
+ S3Object s3object;
+ try {
+ s3object = s3client.getObject(new GetObjectRequest(bucketName, key));
+ }
+ catch (AmazonClientException ace) {
+ throw new IOException("Unable to retrieve object from S3: " + ace, ace);
+ }
+
+ try (InputStream ins = s3object.getObjectContent()) {
+ String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
+ return Note.fromJson(json);
+ }
+ }
+
+ private NoteInfo getNoteInfo(String key) throws IOException {
+ Note note = getNote(key);
+ return new NoteInfo(note);
+ }
+
+ @Override
+ public Note get(String noteId, AuthenticationInfo subject) throws IOException {
+ return getNote(user + "/" + "notebook" + "/" + noteId + "/" + "note.json");
+ }
+
+ @Override
+ public void save(Note note, AuthenticationInfo subject) throws IOException {
+ String json = note.toJson();
+ String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json";
+
+ File file = File.createTempFile("note", "json");
+ try {
+ Writer writer = new OutputStreamWriter(new FileOutputStream(file));
+ writer.write(json);
+ writer.close();
+
+ PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, file);
+
+ if (useServerSideEncryption) {
+ // Request server-side encryption.
+ ObjectMetadata objectMetadata = new ObjectMetadata();
+ objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
+ putRequest.setMetadata(objectMetadata);
+ }
+
+ s3client.putObject(putRequest);
+ }
+ catch (AmazonClientException ace) {
+ throw new IOException("Unable to store note in S3: " + ace, ace);
+ }
+ finally {
+ FileUtils.deleteQuietly(file);
+ }
+ }
+
+ @Override
+ public void remove(String noteId, AuthenticationInfo subject) throws IOException {
+ String key = user + "/" + "notebook" + "/" + noteId;
+ final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
+ .withBucketName(bucketName).withPrefix(key);
+
+ try {
+ ObjectListing objects = s3client.listObjects(listObjectsRequest);
+ do {
+ for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
+ s3client.deleteObject(bucketName, objectSummary.getKey());
+ }
+ objects = s3client.listNextBatchOfObjects(objects);
+ } while (objects.isTruncated());
+ }
+ catch (AmazonClientException ace) {
+ throw new IOException("Unable to remove note in S3: " + ace, ace);
+ }
+ }
+
+ @Override
+ public void close() {
+ //no-op
+ }
+
+ @Override
+ public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
+ LOG.warn("Method not implemented");
+ return Collections.emptyList();
+ }
+
+ @Override
+ public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) {
+ LOG.warn("Method not implemented");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/s3/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/s3/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/s3/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
new file mode 100644
index 0000000..790bbdf
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/s3/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.S3NotebookRepo
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/vfs/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/vfs/pom.xml b/zeppelin-plugins/notebookrepo/vfs/pom.xml
new file mode 100644
index 0000000..ab414f6
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/vfs/pom.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>zengine-plugins-parent</artifactId>
+ <groupId>org.apache.zeppelin</groupId>
+ <version>0.9.0-SNAPSHOT</version>
+ <relativePath>../../../zeppelin-plugins</relativePath>
+ </parent>
+
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>notebookrepo-vfs</artifactId>
+ <packaging>jar</packaging>
+ <version>0.9.0-SNAPSHOT</version>
+ <name>Zeppelin: Plugin VFSNotebookRepo</name>
+ <description>NotebookRepo implementation based on VFS</description>
+
+ <properties>
+ <commons.vfs2.version>2.2</commons.vfs2.version>
+ <plugin.name>NotebookRepo/VFSNotebookRepo</plugin.name>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-vfs2</artifactId>
+ <version>${commons.vfs2.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+
+</project>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
new file mode 100644
index 0000000..4294b86
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/vfs/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import com.google.common.collect.Lists;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.vfs2.FileContent;
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemManager;
+import org.apache.commons.vfs2.FileType;
+import org.apache.commons.vfs2.NameScope;
+import org.apache.commons.vfs2.Selectors;
+import org.apache.commons.vfs2.VFS;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.NoteInfo;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+*
+*/
+public class VFSNotebookRepo implements NotebookRepo {
+ private static final Logger LOG = LoggerFactory.getLogger(VFSNotebookRepo.class);
+
+ private FileSystemManager fsManager;
+ private URI filesystemRoot;
+ protected ZeppelinConfiguration conf;
+
+ public VFSNotebookRepo() {
+
+ }
+
+ @Override
+ public void init(ZeppelinConfiguration conf) throws IOException {
+ this.conf = conf;
+ setNotebookDirectory(conf.getNotebookDir());
+ }
+
+ protected void setNotebookDirectory(String notebookDirPath) throws IOException {
+ try {
+ LOG.info("Using notebookDir: " + notebookDirPath);
+ if (conf.isWindowsPath(notebookDirPath)) {
+ filesystemRoot = new File(notebookDirPath).toURI();
+ } else {
+ filesystemRoot = new URI(notebookDirPath);
+ }
+ } catch (URISyntaxException e1) {
+ throw new IOException(e1);
+ }
+
+ if (filesystemRoot.getScheme() == null) { // it is local path
+ File f = new File(conf.getRelativeDir(filesystemRoot.getPath()));
+ this.filesystemRoot = f.toURI();
+ }
+
+ fsManager = VFS.getManager();
+ FileObject file = fsManager.resolveFile(filesystemRoot.getPath());
+ if (!file.exists()) {
+ LOG.info("Notebook dir doesn't exist, create on is {}.", file.getName());
+ file.createFolder();
+ }
+ }
+
+ private String getNotebookDirPath() {
+ return filesystemRoot.getPath().toString();
+ }
+
+ private String getPath(String path) {
+ if (path == null || path.trim().length() == 0) {
+ return filesystemRoot.toString();
+ }
+ if (path.startsWith("/")) {
+ return filesystemRoot.toString() + path;
+ } else {
+ return filesystemRoot.toString() + "/" + path;
+ }
+ }
+
+ private boolean isDirectory(FileObject fo) throws IOException {
+ if (fo == null) return false;
+ if (fo.getType() == FileType.FOLDER) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
+ FileObject rootDir = getRootDir();
+
+ FileObject[] children = rootDir.getChildren();
+
+ List<NoteInfo> infos = new LinkedList<>();
+ for (FileObject f : children) {
+ String fileName = f.getName().getBaseName();
+ if (f.isHidden()
+ || fileName.startsWith(".")
+ || fileName.startsWith("#")
+ || fileName.startsWith("~")) {
+ // skip hidden, temporary files
+ continue;
+ }
+
+ if (!isDirectory(f)) {
+ // currently single note is saved like, [NOTE_ID]/note.json.
+ // so it must be a directory
+ continue;
+ }
+
+ NoteInfo info = null;
+
+ try {
+ info = getNoteInfo(f);
+ if (info != null) {
+ infos.add(info);
+ }
+ } catch (Exception e) {
+ LOG.error("Can't read note " + f.getName().toString(), e);
+ }
+ }
+
+ return infos;
+ }
+
+ private Note getNote(FileObject noteDir) throws IOException {
+ if (!isDirectory(noteDir)) {
+ throw new IOException(noteDir.getName().toString() + " is not a directory");
+ }
+
+ FileObject noteJson = noteDir.resolveFile("note.json", NameScope.CHILD);
+ if (!noteJson.exists()) {
+ throw new IOException(noteJson.getName().toString() + " not found");
+ }
+
+ FileContent content = noteJson.getContent();
+ InputStream ins = content.getInputStream();
+ String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
+ ins.close();
+
+ return Note.fromJson(json);
+ }
+
+ private NoteInfo getNoteInfo(FileObject noteDir) throws IOException {
+ Note note = getNote(noteDir);
+ return new NoteInfo(note);
+ }
+
+ @Override
+ public Note get(String noteId, AuthenticationInfo subject) throws IOException {
+ FileObject rootDir = fsManager.resolveFile(getPath("/"));
+ FileObject noteDir = rootDir.resolveFile(noteId, NameScope.CHILD);
+
+ return getNote(noteDir);
+ }
+
+ protected FileObject getRootDir() throws IOException {
+ FileObject rootDir = fsManager.resolveFile(getPath("/"));
+
+ if (!rootDir.exists()) {
+ throw new IOException("Root path does not exists");
+ }
+
+ if (!isDirectory(rootDir)) {
+ throw new IOException("Root path is not a directory");
+ }
+
+ return rootDir;
+ }
+
+ @Override
+ public synchronized void save(Note note, AuthenticationInfo subject) throws IOException {
+ LOG.info("Saving note:" + note.getId());
+ String json = note.toJson();
+
+ FileObject rootDir = getRootDir();
+
+ FileObject noteDir = rootDir.resolveFile(note.getId(), NameScope.CHILD);
+
+ if (!noteDir.exists()) {
+ noteDir.createFolder();
+ }
+ if (!isDirectory(noteDir)) {
+ throw new IOException(noteDir.getName().toString() + " is not a directory");
+ }
+
+ FileObject noteJson = noteDir.resolveFile(".note.json", NameScope.CHILD);
+ // false means not appending. creates file if not exists
+ OutputStream out = noteJson.getContent().getOutputStream(false);
+ out.write(json.getBytes(conf.getString(ConfVars.ZEPPELIN_ENCODING)));
+ out.close();
+ noteJson.moveTo(noteDir.resolveFile("note.json", NameScope.CHILD));
+ }
+
+ @Override
+ public void remove(String noteId, AuthenticationInfo subject) throws IOException {
+ FileObject rootDir = fsManager.resolveFile(getPath("/"));
+ FileObject noteDir = rootDir.resolveFile(noteId, NameScope.CHILD);
+
+ if (!noteDir.exists()) {
+ // nothing to do
+ return;
+ }
+
+ if (!isDirectory(noteDir)) {
+ // it is not look like zeppelin note savings
+ throw new IOException("Can not remove " + noteDir.getName().toString());
+ }
+
+ noteDir.delete(Selectors.SELECT_SELF_AND_CHILDREN);
+ }
+
+ @Override
+ public void close() {
+ //no-op
+ }
+
+ @Override
+ public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
+ NotebookRepoSettingsInfo repoSetting = NotebookRepoSettingsInfo.newInstance();
+ List<NotebookRepoSettingsInfo> settings = new ArrayList<>();
+ repoSetting.name = "Notebook Path";
+ repoSetting.type = NotebookRepoSettingsInfo.Type.INPUT;
+ repoSetting.value = Collections.emptyList();
+ repoSetting.selected = getNotebookDirPath();
+
+ settings.add(repoSetting);
+ return settings;
+ }
+
+ @Override
+ public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) {
+ if (settings == null || settings.isEmpty()) {
+ LOG.error("Cannot update {} with empty settings", this.getClass().getName());
+ return;
+ }
+ String newNotebookDirectotyPath = StringUtils.EMPTY;
+ if (settings.containsKey("Notebook Path")) {
+ newNotebookDirectotyPath = settings.get("Notebook Path");
+ }
+
+ if (StringUtils.isBlank(newNotebookDirectotyPath)) {
+ LOG.error("Notebook path is invalid");
+ return;
+ }
+ LOG.warn("{} will change notebook dir from {} to {}",
+ subject.getUser(), getNotebookDirPath(), newNotebookDirectotyPath);
+ try {
+ setNotebookDirectory(newNotebookDirectotyPath);
+ } catch (IOException e) {
+ LOG.error("Cannot update notebook directory", e);
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/vfs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/vfs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/vfs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
new file mode 100644
index 0000000..ed95232
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/vfs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.VFSNotebookRepo
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/TestVFSNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/TestVFSNotebookRepo.java b/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/TestVFSNotebookRepo.java
new file mode 100644
index 0000000..452adc0
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/vfs/src/test/java/org/apache/zeppelin/notebook/repo/TestVFSNotebookRepo.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.io.FileUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.Paragraph;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestVFSNotebookRepo {
+
+ private ZeppelinConfiguration zConf;
+ private VFSNotebookRepo notebookRepo;
+ private String notebookDir = "/tmp/zeppelin/vfs_notebookrepo/";
+
+ @Before
+ public void setUp() throws IOException {
+ notebookRepo = new VFSNotebookRepo();
+ FileUtils.forceMkdir(new File(notebookDir));
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir);
+ zConf = new ZeppelinConfiguration();
+ notebookRepo.init(zConf);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ FileUtils.deleteDirectory(new File(notebookDir));
+ }
+
+ @Test
+ public void testBasics() throws IOException {
+ assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+
+ Note note1 = new Note();
+ Paragraph p1 = note1.insertNewParagraph(0, AuthenticationInfo.ANONYMOUS);
+ p1.setText("%md hello world");
+ p1.setTitle("my title");
+ notebookRepo.save(note1, AuthenticationInfo.ANONYMOUS);
+
+ assertEquals(1, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+ Note note2 = notebookRepo.get(note1.getId(), AuthenticationInfo.ANONYMOUS);
+ assertEquals(note1.getParagraphCount(), note2.getParagraphCount());
+
+ Paragraph p2 = note2.getParagraph(p1.getId());
+ assertEquals(p1.getText(), p2.getText());
+ assertEquals(p1.getTitle(), p2.getTitle());
+
+ notebookRepo.remove(note1.getId(), AuthenticationInfo.ANONYMOUS);
+ assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+ }
+
+ @Test
+ public void testInvalidJson() throws IOException {
+ assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+
+ // invalid note will be ignored
+ createNewNote("invalid_content", "id_1");
+ assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+
+ // only valid note will be fetched
+ createNewNote("{}", "id_2");
+ assertEquals(1, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+ }
+
+ @Test
+ public void testUpdateSettings() throws IOException {
+ List<NotebookRepoSettingsInfo> repoSettings = notebookRepo.getSettings(AuthenticationInfo.ANONYMOUS);
+ assertEquals(1, repoSettings.size());
+ NotebookRepoSettingsInfo settingInfo = repoSettings.get(0);
+ assertEquals("Notebook Path", settingInfo.name);
+ assertEquals(notebookDir, settingInfo.selected);
+
+ createNewNote("{}", "id_2");
+ assertEquals(1, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+
+ String newNotebookDir = "/tmp/zeppelin/vfs_notebookrepo2";
+ FileUtils.forceMkdir(new File(newNotebookDir));
+ Map<String, String> newSettings = ImmutableMap.of("Notebook Path", newNotebookDir);
+ notebookRepo.updateSettings(newSettings, AuthenticationInfo.ANONYMOUS);
+ assertEquals(0, notebookRepo.list(AuthenticationInfo.ANONYMOUS).size());
+ }
+
+ private void createNewNote(String content, String noteId) throws IOException {
+ FileUtils.writeStringToFile(new File(notebookDir + "/" + noteId, "note.json"), content);
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/zeppelin-hub/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/zeppelin-hub/pom.xml b/zeppelin-plugins/notebookrepo/zeppelin-hub/pom.xml
new file mode 100644
index 0000000..af2c73e
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/zeppelin-hub/pom.xml
@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>zengine-plugins-parent</artifactId>
+ <groupId>org.apache.zeppelin</groupId>
+ <version>0.9.0-SNAPSHOT</version>
+ <relativePath>../../../zeppelin-plugins</relativePath>
+ </parent>
+
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>notebookrepo-zeppelin-hub</artifactId>
+ <packaging>jar</packaging>
+ <version>0.9.0-SNAPSHOT</version>
+ <name>Zeppelin: Plugin ZeppelinHubRepo</name>
+ <description>NotebookRepo implementation based on Zeppelin Hub</description>
+
+ <properties>
+ <jetty.version>9.2.15.v20160210</jetty.version>
+ <google.truth.version>0.27</google.truth.version>
+ <plugin.name>NotebookRepo/ZeppelinHubRepo</plugin.name>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.eclipse.jetty.websocket</groupId>
+ <artifactId>websocket-server</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.truth</groupId>
+ <artifactId>truth</artifactId>
+ <version>${google.truth.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+</project>