You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2018/03/22 13:19:56 UTC
[7/7] zeppelin git commit: ZEPPELIN-3196. Plugin framework for
Zeppelin Engine
ZEPPELIN-3196. Plugin framework for Zeppelin Engine
### What is this PR for?
More details is in the design doc. https://docs.google.com/document/d/1QRgznfDKsi6XxR9FTJtUuiGAvgQvmc12FdIbHrmOZzc/edit?usp=sharing
### What type of PR is it?
[ Improvement | Refactoring]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-3196
### How should this be tested?
* CI pass
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #2760 from zjffdu/ZEPPELIN-3196 and squashes the following commits:
639c599 [Jeff Zhang] ZEPPELIN-3196. Plugin framework for Zeppelin Engine
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/3eea57ab
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/3eea57ab
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/3eea57ab
Branch: refs/heads/master
Commit: 3eea57ab26e8c4a8811bde64854b95f706858f10
Parents: db716c8
Author: Jeff Zhang <zj...@apache.org>
Authored: Wed Jan 31 19:46:44 2018 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Thu Mar 22 21:19:25 2018 +0800
----------------------------------------------------------------------
.travis.yml | 23 +-
pom.xml | 11 +
.../src/assemble/distribution.xml | 3 +
.../zeppelin/conf/ZeppelinConfiguration.java | 9 +
zeppelin-plugins/notebookrepo/azure/pom.xml | 71 +++
.../notebook/repo/AzureNotebookRepo.java | 216 +++++++++
...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 +
.../notebookrepo/filesystem/pom.xml | 236 ++++++++++
.../notebook/repo/FileSystemNotebookRepo.java | 103 +++++
...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 +
.../repo/FileSystemNotebookRepoTest.java | 102 +++++
zeppelin-plugins/notebookrepo/gcs/pom.xml | 141 ++++++
.../zeppelin/notebook/repo/GCSNotebookRepo.java | 214 +++++++++
...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 +
.../notebook/repo/GCSNotebookRepoTest.java | 235 ++++++++++
zeppelin-plugins/notebookrepo/git/pom.xml | 71 +++
.../zeppelin/notebook/repo/GitNotebookRepo.java | 205 +++++++++
...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 +
.../notebook/repo/GitNotebookRepoTest.java | 385 ++++++++++++++++
.../NotebookRepoSyncInitializationTest.java | 161 +++++++
.../notebook/repo/NotebookRepoSyncTest.java | 444 +++++++++++++++++++
.../notebook/repo/mock/VFSNotebookRepoMock.java | 36 ++
.../git/src/test/resources/2A94M5J1Z/note.json | 341 ++++++++++++++
.../git/src/test/resources/2A94M5J2Z/note.json | 87 ++++
.../git/src/test/resources/log4j.properties | 50 +++
zeppelin-plugins/notebookrepo/github/pom.xml | 50 +++
.../notebook/repo/GitHubNotebookRepo.java | 126 ++++++
...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 +
.../notebook/repo/GitHubNotebookRepoTest.java | 207 +++++++++
.../src/test/resources/2A94M5J1Z/note.json | 341 ++++++++++++++
.../src/test/resources/2A94M5J2Z/note.json | 87 ++++
zeppelin-plugins/notebookrepo/mongodb/pom.xml | 55 +++
.../notebook/repo/MongoNotebookRepo.java | 222 ++++++++++
...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 +
zeppelin-plugins/notebookrepo/s3/pom.xml | 51 +++
.../zeppelin/notebook/repo/S3NotebookRepo.java | 294 ++++++++++++
...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 +
zeppelin-plugins/notebookrepo/vfs/pom.xml | 58 +++
.../zeppelin/notebook/repo/VFSNotebookRepo.java | 286 ++++++++++++
...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 +
.../notebook/repo/TestVFSNotebookRepo.java | 113 +++++
.../notebookrepo/zeppelin-hub/pom.xml | 81 ++++
.../repo/zeppelinhub/ZeppelinHubRepo.java | 385 ++++++++++++++++
...g.apache.zeppelin.notebook.repo.NotebookRepo | 18 +
.../repo/zeppelinhub/ZeppelinHubRepoTest.java | 155 +++++++
.../websocket/ZeppelinClientTest.java | 127 ++++++
.../websocket/ZeppelinhubClientTest.java | 72 +++
.../websocket/mock/MockEchoWebsocketServer.java | 46 ++
.../websocket/mock/MockEventServlet.java | 14 +
.../websocket/mock/MockEventSocket.java | 38 ++
.../protocol/ZeppelinhubMessageTest.java | 43 ++
zeppelin-plugins/pom.xml | 124 ++++++
zeppelin-server/pom.xml | 36 --
.../org/apache/zeppelin/realm/PamRealm.java | 2 +-
.../apache/zeppelin/realm/ZeppelinHubRealm.java | 26 +-
.../apache/zeppelin/socket/NotebookServer.java | 5 +-
.../src/test/resources/log4j.properties | 3 +-
zeppelin-zengine/pom.xml | 298 +------------
.../recovery/FileSystemRecoveryStorage.java | 11 -
.../java/org/apache/zeppelin/notebook/Note.java | 5 +
.../org/apache/zeppelin/notebook/Notebook.java | 2 +-
.../notebook/repo/AzureNotebookRepo.java | 209 ---------
.../notebook/repo/FileSystemNotebookRepo.java | 99 -----
.../zeppelin/notebook/repo/GCSNotebookRepo.java | 208 ---------
.../notebook/repo/GitHubNotebookRepo.java | 126 ------
.../zeppelin/notebook/repo/GitNotebookRepo.java | 190 --------
.../notebook/repo/MongoNotebookRepo.java | 217 ---------
.../zeppelin/notebook/repo/NotebookRepo.java | 4 +
.../notebook/repo/NotebookRepoSync.java | 45 +-
.../zeppelin/notebook/repo/S3NotebookRepo.java | 290 ------------
.../zeppelin/notebook/repo/VFSNotebookRepo.java | 280 ------------
.../repo/zeppelinhub/ZeppelinHubRepo.java | 376 ----------------
.../websocket/ZeppelinhubClient.java | 4 +-
.../apache/zeppelin/plugin/PluginManager.java | 95 ++++
.../helium/HeliumApplicationFactoryTest.java | 7 +-
.../apache/zeppelin/notebook/NotebookTest.java | 324 ++++++++++----
.../repo/FileSystemNotebookRepoTest.java | 101 -----
.../notebook/repo/GCSNotebookRepoTest.java | 235 ----------
.../notebook/repo/GitHubNotebookRepoTest.java | 207 ---------
.../notebook/repo/GitNotebookRepoTest.java | 385 ----------------
.../NotebookRepoSyncInitializationTest.java | 159 -------
.../notebook/repo/NotebookRepoSyncTest.java | 437 ------------------
.../notebook/repo/VFSNotebookRepoTest.java | 167 -------
.../notebook/repo/mock/VFSNotebookRepoMock.java | 42 --
.../repo/zeppelinhub/ZeppelinHubRepoTest.java | 154 -------
.../websocket/ZeppelinClientTest.java | 127 ------
.../websocket/ZeppelinhubClientTest.java | 72 ---
.../websocket/mock/MockEchoWebsocketServer.java | 46 --
.../websocket/mock/MockEventServlet.java | 14 -
.../websocket/mock/MockEventSocket.java | 38 --
.../protocol/ZeppelinhubMessageTest.java | 43 --
.../src/test/resources/2A94M5J1Z/note.json | 341 --------------
.../src/test/resources/2A94M5J2Z/note.json | 87 ----
.../src/test/resources/log4j.properties | 3 +-
94 files changed, 6651 insertions(+), 5150 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index e113ae3..5a81441 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -44,7 +44,7 @@ matrix:
# Test License compliance using RAT tool
- jdk: "openjdk7"
dist: trusty
- env: SCALA_VER="2.11" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
+ env: BUILD_PLUGINS="false" SCALA_VER="2.11" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
# Run e2e tests (in zeppelin-web)
# chrome dropped the support for precise (ubuntu 12.04), so need to use trusty
@@ -53,7 +53,7 @@ matrix:
sudo: false
dist: trusty
jdk: "oraclejdk8"
- env: CI="true" WEB_E2E="true" PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Phadoop2 -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e"
+ env: BUILD_PLUGINS="false" CI="true" WEB_E2E="true" PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Phadoop2 -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e"
addons:
apt:
packages:
@@ -66,19 +66,19 @@ matrix:
- sudo: required
jdk: "oraclejdk8"
dist: trusty
- env: PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false"
+ env: BUILD_PLUGINS="true" PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="install -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false"
# Test selenium with spark module for 1.6.3
- jdk: "oraclejdk8"
dist: trusty
addons:
firefox: "31.0"
- env: CI="true" PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Phelium-dev -Pexamples -Pintegration" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-integration -DfailIfNoTests=false"
+ env: BUILD_PLUGINS="true" CI="true" PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop2 -Phadoop-2.6 -Phelium-dev -Pexamples -Pintegration" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-integration -DfailIfNoTests=false"
# Test interpreter modules
- jdk: "openjdk7"
dist: trusty
- env: PYTHON="3" SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
+ env: BUILD_PLUGINS="false" PYTHON="3" SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS=""
# Run ZeppelinSparkClusterTest & SparkIntegrationTest in one build would exceed the time limitation of travis, so running them separately
@@ -86,35 +86,35 @@ matrix:
- sudo: required
jdk: "oraclejdk8"
dist: trusty
- env: PYTHON="2" PROFILE="-Pspark-2.2" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+ env: BUILD_PLUGINS="true" PYTHON="2" PROFILE="-Pspark-2.2" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,zeppelin-server,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Integration test of spark interpreter with different spark versions under python3, only run SparkIntegrationTest. Also run spark unit test of spark 1.6 in this build.
- sudo: required
jdk: "oraclejdk8"
dist: trusty
- env: PYTHON="3" PROFILE="-Pspark-1.6" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=SparkIntegrationTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+ env: BUILD_PLUGINS="true" PYTHON="3" PROFILE="-Pspark-1.6" SPARKR="true" BUILD_FLAG="install -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl zeppelin-zengine,spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=SparkIntegrationTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 2.1.0 with scala 2.11
- jdk: "openjdk7"
dist: trusty
- env: PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.1 -Phadoop2 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+ env: BUILD_PLUGINS="false" PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.1 -Phadoop2 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test spark module for 2.0.2 with scala 2.11
- jdk: "oraclejdk8"
dist: trusty
- env: PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.0 -Phadoop3 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false"
+ env: BUILD_PLUGINS="false" PYTHON="2" SCALA_VER="2.11" PROFILE="-Pspark-2.0 -Phadoop3 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl spark/interpreter,spark/spark-dependencies" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false"
# Test python/pyspark with python 2, livy 0.5
- sudo: required
dist: trusty
jdk: "openjdk7"
- env: PYTHON="2" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS=""
+ env: BUILD_PLUGINS="false" PYTHON="2" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS=""
# Test livy 0.5 with spark 2.2.0 under python3
- sudo: required
dist: trusty
jdk: "openjdk8"
- env: PYTHON="3" SPARK_VER="2.2.0" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS=""
+ env: BUILD_PLUGINS="false" PYTHON="3" SPARK_VER="2.2.0" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy" TEST_PROJECTS=""
before_install:
# check files included in commit range, clear bower_components if a bower.json file has changed.
@@ -136,6 +136,7 @@ before_install:
install:
- echo "mvn $BUILD_FLAG $MODULES $PROFILE -B"
- mvn $BUILD_FLAG $MODULES $PROFILE -B
+ - if [ $BUILD_PLUGINS == "true" ]; then echo "mvn clean package -pl zeppelin-plugins -amd -B"; mvn clean package -pl zeppelin-plugins -amd -B; fi
before_script:
- if [[ -n $SPARK_VER ]]; then travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER; fi
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ca6c409..a2b5553 100644
--- a/pom.xml
+++ b/pom.xml
@@ -81,6 +81,7 @@
<module>zeppelin-web</module>
<module>zeppelin-server</module>
<module>zeppelin-jupyter</module>
+ <module>zeppelin-plugins</module>
<module>zeppelin-distribution</module>
</modules>
@@ -115,6 +116,7 @@
<commons.logging.version>1.1.1</commons.logging.version>
<commons.cli.version>1.3.1</commons.cli.version>
<shiro.version>1.3.2</shiro.version>
+ <joda.version>2.9.9</joda.version>
<!-- test library versions -->
<junit.version>4.12</junit.version>
@@ -247,6 +249,12 @@
<version>${commons.cli.version}</version>
</dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>${joda.version}</version>
+ </dependency>
+
<!-- Apache Shiro -->
<dependency>
<groupId>org.apache.shiro</groupId>
@@ -592,6 +600,9 @@
<version>${plugin.surefire.version}</version>
<configuration combine.children="append">
<argLine>-Xmx2g -Xms1g -Dfile.encoding=UTF-8</argLine>
+ <environmentVariables>
+ <IS_ZEPPELIN_TEST>true</IS_ZEPPELIN_TEST>
+ </environmentVariables>
<excludes>
<exclude>${tests.to.exclude}</exclude>
</excludes>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-distribution/src/assemble/distribution.xml
----------------------------------------------------------------------
diff --git a/zeppelin-distribution/src/assemble/distribution.xml b/zeppelin-distribution/src/assemble/distribution.xml
index 5c369e2..068f85b 100644
--- a/zeppelin-distribution/src/assemble/distribution.xml
+++ b/zeppelin-distribution/src/assemble/distribution.xml
@@ -92,6 +92,9 @@
<directory>../notebook</directory>
</fileSet>
<fileSet>
+ <directory>../plugins</directory>
+ </fileSet>
+ <fileSet>
<outputDirectory>/lib/interpreter</outputDirectory>
<directory>../zeppelin-interpreter/target/lib</directory>
</fileSet>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index db6e150..2960cd0 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -347,10 +347,18 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_DIR);
}
+ public String getPluginsDir() {
+ return getRelativeDir(getString(ConfVars.ZEPPELIN_PLUGINS_DIR));
+ }
+
public String getRecoveryDir() {
return getRelativeDir(ConfVars.ZEPPELIN_RECOVERY_DIR);
}
+ public String getNotebookStorageClass() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE);
+ }
+
public String getRecoveryStorageClass() {
return getString(ConfVars.ZEPPELIN_RECOVERY_STORAGE_CLASS);
}
@@ -711,6 +719,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_RECOVERY_DIR("zeppelin.recovery.dir", "recovery"),
ZEPPELIN_RECOVERY_STORAGE_CLASS("zeppelin.recovery.storage.class",
"org.apache.zeppelin.interpreter.recovery.NullRecoveryStorage"),
+ ZEPPELIN_PLUGINS_DIR("zeppelin.plugins.dir", "plugins"),
// use specified notebook (id) as homescreen
ZEPPELIN_NOTEBOOK_HOMESCREEN("zeppelin.notebook.homescreen", null),
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/azure/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/azure/pom.xml b/zeppelin-plugins/notebookrepo/azure/pom.xml
new file mode 100644
index 0000000..f663de6
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/azure/pom.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>zengine-plugins-parent</artifactId>
+ <groupId>org.apache.zeppelin</groupId>
+ <version>0.9.0-SNAPSHOT</version>
+ <relativePath>../../../zeppelin-plugins</relativePath>
+ </parent>
+
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>notebookrepo-azure</artifactId>
+ <packaging>jar</packaging>
+ <version>0.9.0-SNAPSHOT</version>
+ <name>Zeppelin: Plugin AzureNotebookRepo</name>
+ <description>NotebookRepo implementation based on Azure</description>
+
+ <properties>
+ <adl.sdk.version>2.1.4</adl.sdk.version>
+ <azure.storage.version>4.0.0</azure.storage.version>
+ <plugin.name>NotebookRepo/AzureNotebookRepo</plugin.name>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.microsoft.azure</groupId>
+ <artifactId>azure-data-lake-store-sdk</artifactId>
+ <version>${adl.sdk.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.azure</groupId>
+ <artifactId>azure-storage</artifactId>
+ <version>${azure.storage.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
new file mode 100644
index 0000000..12dbd90
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/AzureNotebookRepo.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import com.microsoft.azure.storage.CloudStorageAccount;
+import com.microsoft.azure.storage.StorageException;
+import com.microsoft.azure.storage.file.CloudFile;
+import com.microsoft.azure.storage.file.CloudFileClient;
+import com.microsoft.azure.storage.file.CloudFileDirectory;
+import com.microsoft.azure.storage.file.CloudFileShare;
+import com.microsoft.azure.storage.file.ListFileItem;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.net.URISyntaxException;
+import java.security.InvalidKeyException;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.NoteInfo;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Azure storage backend for notebooks
+ */
+public class AzureNotebookRepo implements NotebookRepo {
+ private static final Logger LOG = LoggerFactory.getLogger(AzureNotebookRepo.class);
+
+ private ZeppelinConfiguration conf;
+ private String user;
+ private String shareName;
+ private CloudFileDirectory rootDir;
+
+ public AzureNotebookRepo() {
+
+ }
+
+ public void init(ZeppelinConfiguration conf) throws IOException {
+ this.conf = conf;
+ user = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_USER);
+ shareName = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_SHARE);
+
+ try {
+ CloudStorageAccount account = CloudStorageAccount.parse(
+ conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING));
+ CloudFileClient client = account.createCloudFileClient();
+ CloudFileShare share = client.getShareReference(shareName);
+ share.createIfNotExists();
+
+ CloudFileDirectory userDir = StringUtils.isBlank(user) ?
+ share.getRootDirectoryReference() :
+ share.getRootDirectoryReference().getDirectoryReference(user);
+ userDir.createIfNotExists();
+
+ rootDir = userDir.getDirectoryReference("notebook");
+ rootDir.createIfNotExists();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
+ List<NoteInfo> infos = new LinkedList<>();
+ NoteInfo info = null;
+
+ for (ListFileItem item : rootDir.listFilesAndDirectories()) {
+ if (item.getClass() == CloudFileDirectory.class) {
+ CloudFileDirectory dir = (CloudFileDirectory) item;
+
+ try {
+ if (dir.getFileReference("note.json").exists()) {
+ info = new NoteInfo(getNote(dir.getName()));
+
+ if (info != null) {
+ infos.add(info);
+ }
+ }
+ } catch (StorageException | URISyntaxException e) {
+ String msg = "Error enumerating notebooks from Azure storage";
+ LOG.error(msg, e);
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+ }
+
+ return infos;
+ }
+
+ private Note getNote(String noteId) throws IOException {
+ InputStream ins = null;
+
+ try {
+ CloudFileDirectory dir = rootDir.getDirectoryReference(noteId);
+ CloudFile file = dir.getFileReference("note.json");
+
+ ins = file.openRead();
+ } catch (URISyntaxException | StorageException e) {
+ String msg = String.format("Error reading notebook %s from Azure storage", noteId);
+
+ LOG.error(msg, e);
+
+ throw new IOException(msg, e);
+ }
+
+ String json = IOUtils.toString(ins,
+ conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_ENCODING));
+ ins.close();
+ return Note.fromJson(json);
+ }
+
+ @Override
+ public Note get(String noteId, AuthenticationInfo subject) throws IOException {
+ return getNote(noteId);
+ }
+
+ @Override
+ public void save(Note note, AuthenticationInfo subject) throws IOException {
+ String json = note.toJson();
+
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ Writer writer = new OutputStreamWriter(output);
+ writer.write(json);
+ writer.close();
+ output.close();
+
+ byte[] buffer = output.toByteArray();
+
+ try {
+ CloudFileDirectory dir = rootDir.getDirectoryReference(note.getId());
+ dir.createIfNotExists();
+
+ CloudFile cloudFile = dir.getFileReference("note.json");
+ cloudFile.uploadFromByteArray(buffer, 0, buffer.length);
+ } catch (URISyntaxException | StorageException e) {
+ String msg = String.format("Error saving notebook %s to Azure storage", note.getId());
+
+ LOG.error(msg, e);
+
+ throw new IOException(msg, e);
+ }
+ }
+
+ // unfortunately, we need to use a recursive delete here
+ private void delete(ListFileItem item) throws StorageException {
+ if (item.getClass() == CloudFileDirectory.class) {
+ CloudFileDirectory dir = (CloudFileDirectory) item;
+
+ for (ListFileItem subItem : dir.listFilesAndDirectories()) {
+ delete(subItem);
+ }
+
+ dir.deleteIfExists();
+ } else if (item.getClass() == CloudFile.class) {
+ CloudFile file = (CloudFile) item;
+
+ file.deleteIfExists();
+ }
+ }
+
+ @Override
+ public void remove(String noteId, AuthenticationInfo subject) throws IOException {
+ try {
+ CloudFileDirectory dir = rootDir.getDirectoryReference(noteId);
+
+ delete(dir);
+ } catch (URISyntaxException | StorageException e) {
+ String msg = String.format("Error deleting notebook %s from Azure storage", noteId);
+
+ LOG.error(msg, e);
+
+ throw new IOException(msg, e);
+ }
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
+ LOG.warn("Method not implemented");
+ return Collections.emptyList();
+ }
+
+ @Override
+ public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) {
+ LOG.warn("Method not implemented");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/azure/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/azure/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/azure/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
new file mode 100644
index 0000000..b028d54
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/azure/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.AzureNotebookRepo
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/filesystem/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/filesystem/pom.xml b/zeppelin-plugins/notebookrepo/filesystem/pom.xml
new file mode 100644
index 0000000..422b7b6
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/filesystem/pom.xml
@@ -0,0 +1,236 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>zengine-plugins-parent</artifactId>
+ <groupId>org.apache.zeppelin</groupId>
+ <version>0.9.0-SNAPSHOT</version>
+ <relativePath>../../../zeppelin-plugins</relativePath>
+ </parent>
+
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>notebookrepo-filesystem</artifactId>
+ <packaging>jar</packaging>
+ <version>0.9.0-SNAPSHOT</version>
+ <name>Zeppelin: Plugin FileSystemNotebookRepo</name>
+ <description>NotebookRepo implementation based on Hadoop FileSystem</description>
+
+ <properties>
+ <adl.sdk.version>2.1.4</adl.sdk.version>
+ <plugin.name>NotebookRepo/FileSystemNotebookRepo</plugin.name>
+ </properties>
+
+ <profiles>
+
+ <profile>
+ <id>hadoop2-azure</id>
+ <properties>
+ <hadoop.version>2.7.3</hadoop.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-azure</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.jcraf</groupId>
+ <artifactId>jsch</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.microsoft.azure</groupId>
+ <artifactId>azure-data-lake-store-sdk</artifactId>
+ <version>${adl.sdk.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>hadoop2-aws</id>
+ <properties>
+ <hadoop.version>2.7.3</hadoop.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>hadoop3-azure</id>
+ <properties>
+ <hadoop.version>3.0.0</hadoop.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-azure</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.nimbusds</groupId>
+ <artifactId>nimbus-jose-jwt</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-servlet</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-azure-datalake</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>hadoop3-aws</id>
+ <properties>
+ <hadoop.version>3.0.0</hadoop.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aws</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java
new file mode 100644
index 0000000..5d9c85c
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java
@@ -0,0 +1,103 @@
+package org.apache.zeppelin.notebook.repo;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.notebook.FileSystemStorage;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.NoteInfo;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * NotebookRepos for hdfs.
+ *
+ * Assume the notebook directory structure is as following
+ * - notebookdir
+ * - noteId/note.json
+ * - noteId/note.json
+ * - noteId/note.json
+ */
+public class FileSystemNotebookRepo implements NotebookRepo {
+ private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemNotebookRepo.class);
+
+ private FileSystemStorage fs;
+ private Path notebookDir;
+
+ public FileSystemNotebookRepo() {
+
+ }
+
+ public void init(ZeppelinConfiguration zConf) throws IOException {
+ this.fs = new FileSystemStorage(zConf, zConf.getNotebookDir());
+ LOGGER.info("Creating FileSystem: " + this.fs.getFs().getClass().getName() +
+ " for Zeppelin Notebook.");
+ this.notebookDir = this.fs.makeQualified(new Path(zConf.getNotebookDir()));
+ LOGGER.info("Using folder {} to store notebook", notebookDir);
+ this.fs.tryMkDir(notebookDir);
+ }
+
+ @Override
+ public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
+ List<Path> notePaths = fs.list(new Path(notebookDir, "*/note.json"));
+ List<NoteInfo> noteInfos = new ArrayList<>();
+ for (Path path : notePaths) {
+ NoteInfo noteInfo = new NoteInfo(path.getParent().getName(), "", null);
+ noteInfos.add(noteInfo);
+ }
+ return noteInfos;
+ }
+
+ @Override
+ public Note get(final String noteId, AuthenticationInfo subject) throws IOException {
+ String content = this.fs.readFile(
+ new Path(notebookDir.toString() + "/" + noteId + "/note.json"));
+ return Note.fromJson(content);
+ }
+
+ @Override
+ public void save(final Note note, AuthenticationInfo subject) throws IOException {
+ this.fs.writeFile(note.toJson(),
+ new Path(notebookDir.toString() + "/" + note.getId() + "/note.json"),
+ true);
+ }
+
+ @Override
+ public void remove(final String noteId, AuthenticationInfo subject) throws IOException {
+ this.fs.delete(new Path(notebookDir.toString() + "/" + noteId));
+ }
+
+ @Override
+ public void close() {
+ LOGGER.warn("close is not implemented for HdfsNotebookRepo");
+ }
+
+ @Override
+ public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
+ LOGGER.warn("getSettings is not implemented for HdfsNotebookRepo");
+ return null;
+ }
+
+ @Override
+ public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) {
+ LOGGER.warn("updateSettings is not implemented for HdfsNotebookRepo");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/filesystem/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/filesystem/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
new file mode 100644
index 0000000..06ef085
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/filesystem/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java
new file mode 100644
index 0000000..5fd8becc
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/filesystem/src/test/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepoTest.java
@@ -0,0 +1,102 @@
+package org.apache.zeppelin.notebook.repo;
+
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class FileSystemNotebookRepoTest {
+
+ private ZeppelinConfiguration zConf;
+ private Configuration hadoopConf;
+ private FileSystem fs;
+ private FileSystemNotebookRepo hdfsNotebookRepo;
+ private String notebookDir;
+ private AuthenticationInfo authInfo = AuthenticationInfo.ANONYMOUS;
+
+ @Before
+ public void setUp() throws IOException {
+ notebookDir = Files.createTempDirectory("FileSystemNotebookRepoTest").toFile().getAbsolutePath();
+ zConf = new ZeppelinConfiguration();
+ System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir);
+ hadoopConf = new Configuration();
+ fs = FileSystem.get(hadoopConf);
+ hdfsNotebookRepo = new FileSystemNotebookRepo();
+ hdfsNotebookRepo.init(zConf);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ FileUtils.deleteDirectory(new File(notebookDir));
+ }
+
+ @Test
+ public void testBasics() throws IOException {
+ assertEquals(0, hdfsNotebookRepo.list(authInfo).size());
+
+ // create a new note
+ Note note = new Note();
+ note.setName("title_1");
+
+ Map<String, Object> config = new HashMap<>();
+ config.put("config_1", "value_1");
+ note.setConfig(config);
+ hdfsNotebookRepo.save(note, authInfo);
+ assertEquals(1, hdfsNotebookRepo.list(authInfo).size());
+
+ // read this note from hdfs
+ Note note_copy = hdfsNotebookRepo.get(note.getId(), authInfo);
+ assertEquals(note.getName(), note_copy.getName());
+ assertEquals(note.getConfig(), note_copy.getConfig());
+
+ // update this note
+ note.setName("title_2");
+ hdfsNotebookRepo.save(note, authInfo);
+ assertEquals(1, hdfsNotebookRepo.list(authInfo).size());
+ note_copy = hdfsNotebookRepo.get(note.getId(), authInfo);
+ assertEquals(note.getName(), note_copy.getName());
+ assertEquals(note.getConfig(), note_copy.getConfig());
+
+ // delete this note
+ hdfsNotebookRepo.remove(note.getId(), authInfo);
+ assertEquals(0, hdfsNotebookRepo.list(authInfo).size());
+ }
+
+ @Test
+ public void testComplicatedScenarios() throws IOException {
+ // scenario_1: notebook_dir is not clean. There're some unrecognized dir and file under notebook_dir
+ fs.mkdirs(new Path(notebookDir, "1/2"));
+ OutputStream out = fs.create(new Path(notebookDir, "1/a.json"));
+ out.close();
+
+ assertEquals(0, hdfsNotebookRepo.list(authInfo).size());
+
+ // scenario_2: note_folder is existed.
+ // create a new note
+ Note note = new Note();
+ note.setName("title_1");
+ Map<String, Object> config = new HashMap<>();
+ config.put("config_1", "value_1");
+ note.setConfig(config);
+
+ fs.mkdirs(new Path(notebookDir, note.getId()));
+ hdfsNotebookRepo.save(note, authInfo);
+ assertEquals(1, hdfsNotebookRepo.list(authInfo).size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/gcs/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/gcs/pom.xml b/zeppelin-plugins/notebookrepo/gcs/pom.xml
new file mode 100644
index 0000000..15aa7c7
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/gcs/pom.xml
@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>zengine-plugins-parent</artifactId>
+ <groupId>org.apache.zeppelin</groupId>
+ <version>0.9.0-SNAPSHOT</version>
+ <relativePath>../../../zeppelin-plugins</relativePath>
+ </parent>
+
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>notebookrepo-gcs</artifactId>
+ <packaging>jar</packaging>
+ <version>0.9.0-SNAPSHOT</version>
+ <name>Zeppelin: Plugin GCSNotebookRepo</name>
+ <description>NotebookRepo implementation based on Google Cloud Storage</description>
+
+ <properties>
+ <gcs.storage.version>1.14.0</gcs.storage.version>
+ <google.testing.nio.version>0.32.0-alpha</google.testing.nio.version>
+ <google.truth.version>0.27</google.truth.version>
+ <plugin.name>NotebookRepo/GCSNotebookRepo</plugin.name>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-storage</artifactId>
+ <version>${gcs.storage.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.api</groupId>
+ <artifactId>api-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.http-client</groupId>
+ <artifactId>google-http-client-jackson2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.http-client</groupId>
+ <artifactId>google-http-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-nio</artifactId>
+ <version>${google.testing.nio.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.truth</groupId>
+ <artifactId>truth</artifactId>
+ <version>${google.truth.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.api</groupId>
+ <artifactId>api-common</artifactId>
+ <version>1.2.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.http-client</groupId>
+ <artifactId>google-http-client-jackson2</artifactId>
+ <version>1.23.0</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java
new file mode 100644
index 0000000..ad99aba
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import com.google.cloud.storage.Blob;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.Storage.BlobListOption;
+import com.google.cloud.storage.StorageException;
+import com.google.cloud.storage.StorageOptions;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.gson.JsonParseException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.NoteInfo;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A NotebookRepo implementation for storing notebooks in Google Cloud Storage.
+ *
+ * Notes are stored in the GCS "directory" specified by zeppelin.notebook.gcs.dir. This path
+ * must be in the form gs://bucketName/path/to/Dir. The bucket must already exist. N.B: GCS is an
+ * object store, so this "directory" should not itself be an object. Instead, it represents the base
+ * path for the note.json files.
+ *
+ * Authentication is provided by google-auth-library-java.
+ * @see <a href="https://github.com/google/google-auth-library-java">
+ * google-auth-library-java</a>.
+ */
+public class GCSNotebookRepo implements NotebookRepo {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GCSNotebookRepo.class);
+ private String encoding;
+ private String bucketName;
+ private Optional<String> basePath;
+ private Pattern noteNamePattern;
+ private Storage storage;
+
+ public GCSNotebookRepo() {
+ }
+
+ @VisibleForTesting
+ public GCSNotebookRepo(ZeppelinConfiguration zConf, Storage storage) throws IOException {
+ init(zConf);
+ this.storage = storage;
+ }
+
+ @Override
+ public void init(ZeppelinConfiguration zConf) throws IOException {
+ this.encoding = zConf.getString(ConfVars.ZEPPELIN_ENCODING);
+
+ String gcsStorageDir = zConf.getGCSStorageDir();
+ if (gcsStorageDir.isEmpty()) {
+ throw new IOException("GCS storage directory must be set using 'zeppelin.notebook.gcs.dir'");
+ }
+ if (!gcsStorageDir.startsWith("gs://")) {
+ throw new IOException(String.format(
+ "GCS storage directory '%s' must start with 'gs://'.", gcsStorageDir));
+ }
+ String storageDirWithoutScheme = gcsStorageDir.substring("gs://".length());
+
+ // pathComponents excludes empty string if trailing slash is present
+ List<String> pathComponents = Arrays.asList(storageDirWithoutScheme.split("/"));
+ if (pathComponents.size() < 1) {
+ throw new IOException(String.format(
+ "GCS storage directory '%s' must be in the form gs://bucketname/path/to/dir",
+ gcsStorageDir));
+ }
+ this.bucketName = pathComponents.get(0);
+ if (pathComponents.size() > 1) {
+ this.basePath = Optional.of(StringUtils.join(
+ pathComponents.subList(1, pathComponents.size()), "/"));
+ } else {
+ this.basePath = Optional.absent();
+ }
+
+ // Notes are stored at gs://bucketName/basePath/<note-id>/note.json
+ if (basePath.isPresent()) {
+ this.noteNamePattern = Pattern.compile(
+ "^" + Pattern.quote(basePath.get() + "/") + "([^/]+)/note\\.json$");
+ } else {
+ this.noteNamePattern = Pattern.compile("^([^/]+)/note\\.json$");
+ }
+
+ this.storage = StorageOptions.getDefaultInstance().getService();
+ }
+
+ private BlobId makeBlobId(String noteId) {
+ if (basePath.isPresent()) {
+ return BlobId.of(bucketName, basePath.get() + "/" + noteId + "/note.json");
+ } else {
+ return BlobId.of(bucketName, noteId + "/note.json");
+ }
+ }
+
+ @Override
+ public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
+ try {
+ List<NoteInfo> infos = new ArrayList<>();
+ Iterable<Blob> blobsUnderDir;
+ if (basePath.isPresent()) {
+ blobsUnderDir = storage
+ .list(bucketName, BlobListOption.prefix(this.basePath.get() + "/"))
+ .iterateAll();
+ } else {
+ blobsUnderDir = storage
+ .list(bucketName)
+ .iterateAll();
+ }
+ for (Blob b : blobsUnderDir) {
+ Matcher matcher = noteNamePattern.matcher(b.getName());
+ if (matcher.matches()) {
+ // Callers only use the id field, so do not fetch each note
+ // This matches the implementation in FileSystemNoteRepo#list
+ infos.add(new NoteInfo(matcher.group(1), "", null));
+ }
+ }
+ return infos;
+ } catch (StorageException se) {
+ throw new IOException("Could not list GCS directory: " + se.getMessage(), se);
+ }
+ }
+
+ @Override
+ public Note get(String noteId, AuthenticationInfo subject) throws IOException {
+ BlobId blobId = makeBlobId(noteId);
+ byte[] contents;
+ try {
+ contents = storage.readAllBytes(blobId);
+ } catch (StorageException se) {
+ throw new IOException("Could not read " + blobId.toString() + ": " + se.getMessage(), se);
+ }
+
+ try {
+ return Note.fromJson(new String(contents, encoding));
+ } catch (JsonParseException jpe) {
+ throw new IOException(
+ "Could note parse as json " + blobId.toString() + jpe.getMessage(), jpe);
+ }
+ }
+
+ @Override
+ public void save(Note note, AuthenticationInfo subject) throws IOException {
+ BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId()))
+ .setContentType("application/json")
+ .build();
+ try {
+ storage.create(info, note.toJson().getBytes("UTF-8"));
+ } catch (StorageException se) {
+ throw new IOException("Could not write " + info.toString() + ": " + se.getMessage(), se);
+ }
+ }
+
+ @Override
+ public void remove(String noteId, AuthenticationInfo subject) throws IOException {
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(noteId));
+ BlobId blobId = makeBlobId(noteId);
+ try {
+ boolean deleted = storage.delete(blobId);
+ if (!deleted) {
+ throw new IOException("Tried to remove nonexistent blob " + blobId.toString());
+ }
+ } catch (StorageException se) {
+ throw new IOException("Could not remove " + blobId.toString() + ": " + se.getMessage(), se);
+ }
+ }
+
+ @Override
+ public void close() {
+ //no-op
+ }
+
+ @Override
+ public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
+ LOG.warn("getSettings is not implemented for GCSNotebookRepo");
+ return Collections.emptyList();
+ }
+
+ @Override
+ public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) {
+ LOG.warn("updateSettings is not implemented for GCSNotebookRepo");
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/gcs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo b/zeppelin-plugins/notebookrepo/gcs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
new file mode 100644
index 0000000..784027b
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/gcs/src/main/resources/META-INF/services/org.apache.zeppelin.notebook.repo.NotebookRepo
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.zeppelin.notebook.repo.GCSNotebookRepo
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java
new file mode 100644
index 0000000..c1fae67
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import static com.google.common.truth.Truth.assertThat;
+import static junit.framework.TestCase.fail;
+
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.contrib.nio.testing.LocalStorageHelper;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.NoteInfo;
+import org.apache.zeppelin.notebook.Paragraph;
+import org.apache.zeppelin.scheduler.Job.Status;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class GCSNotebookRepoTest {
+ private static final AuthenticationInfo AUTH_INFO = AuthenticationInfo.ANONYMOUS;
+
+ private GCSNotebookRepo notebookRepo;
+ private Storage storage;
+
+ @Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ { "bucketname", Optional.absent(), "gs://bucketname" },
+ { "bucketname-with-slash", Optional.absent(), "gs://bucketname-with-slash/" },
+ { "bucketname", Optional.of("path/to/dir"), "gs://bucketname/path/to/dir" },
+ { "bucketname", Optional.of("trailing/slash"), "gs://bucketname/trailing/slash/" }
+ });
+ }
+
+ @Parameter(0)
+ public String bucketName;
+
+ @Parameter(1)
+ public Optional<String> basePath;
+
+ @Parameter(2)
+ public String uriPath;
+
+ private Note runningNote;
+
+ @Before
+ public void setUp() throws Exception {
+ this.runningNote = makeRunningNote();
+
+ this.storage = LocalStorageHelper.getOptions().getService();
+
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), uriPath);
+ this.notebookRepo = new GCSNotebookRepo(new ZeppelinConfiguration(), storage);
+ }
+
+ private static Note makeRunningNote() {
+ Note note = new Note();
+ note.setConfig(ImmutableMap.<String, Object>of("key", "value"));
+
+ Paragraph p = new Paragraph(note, null, null);
+ p.setText("text");
+ p.setStatus(Status.RUNNING);
+ note.addParagraph(p);
+ return note;
+ }
+
+ @Test
+ public void testList_nonexistent() throws Exception {
+ assertThat(notebookRepo.list(AUTH_INFO)).isEmpty();
+ }
+
+ @Test
+ public void testList() throws Exception {
+ createAt(runningNote, "note.json");
+ createAt(runningNote, "/note.json");
+ createAt(runningNote, "validid/note.json");
+ createAt(runningNote, "validid-2/note.json");
+ createAt(runningNote, "cannot-be-dir/note.json/foo");
+ createAt(runningNote, "cannot/be/nested/note.json");
+
+ List<NoteInfo> infos = notebookRepo.list(AUTH_INFO);
+ List<String> noteIds = new ArrayList<>();
+ for (NoteInfo info : infos) {
+ noteIds.add(info.getId());
+ }
+ // Only valid paths are gs://bucketname/path/<noteid>/note.json
+ assertThat(noteIds).containsExactlyElementsIn(ImmutableList.of("validid", "validid-2"));
+ }
+
+ @Test
+ public void testGet_nonexistent() throws Exception {
+ try {
+ notebookRepo.get("id", AUTH_INFO);
+ fail();
+ } catch (IOException e) {}
+ }
+
+ @Test
+ public void testGet() throws Exception {
+ create(runningNote);
+
+ // Status of saved running note is removed in get()
+ Note got = notebookRepo.get(runningNote.getId(), AUTH_INFO);
+ assertThat(got.getLastParagraph().getStatus()).isEqualTo(Status.ABORT);
+
+ // But otherwise equal
+ got.getLastParagraph().setStatus(Status.RUNNING);
+ assertThat(got).isEqualTo(runningNote);
+ }
+
+ @Test
+ public void testGet_malformed() throws Exception {
+ createMalformed("id");
+ try {
+ notebookRepo.get("id", AUTH_INFO);
+ fail();
+ } catch (IOException e) {}
+ }
+
+ @Test
+ public void testSave_create() throws Exception {
+ notebookRepo.save(runningNote, AUTH_INFO);
+ // Output is saved
+ assertThat(storage.readAllBytes(makeBlobId(runningNote.getId())))
+ .isEqualTo(runningNote.toJson().getBytes("UTF-8"));
+ }
+
+ @Test
+ public void testSave_update() throws Exception {
+ notebookRepo.save(runningNote, AUTH_INFO);
+ // Change name of runningNote
+ runningNote.setName("new-name");
+ notebookRepo.save(runningNote, AUTH_INFO);
+ assertThat(storage.readAllBytes(makeBlobId(runningNote.getId())))
+ .isEqualTo(runningNote.toJson().getBytes("UTF-8"));
+ }
+
+ @Test
+ public void testRemove_nonexistent() throws Exception {
+ try {
+ notebookRepo.remove("id", AUTH_INFO);
+ fail();
+ } catch (IOException e) {}
+ }
+
+ @Test
+ public void testRemove() throws Exception {
+ create(runningNote);
+ notebookRepo.remove(runningNote.getId(), AUTH_INFO);
+ assertThat(storage.get(makeBlobId(runningNote.getId()))).isNull();
+ }
+
+ private String makeName(String relativePath) {
+ if (basePath.isPresent()) {
+ return basePath.get() + "/" + relativePath;
+ } else {
+ return relativePath;
+ }
+ }
+
+ private BlobId makeBlobId(String noteId) {
+ return BlobId.of(bucketName, makeName(noteId + "/note.json"));
+ }
+
+ private void createAt(Note note, String relativePath) throws IOException {
+ BlobId id = BlobId.of(bucketName, makeName(relativePath));
+ BlobInfo info = BlobInfo.newBuilder(id).setContentType("application/json").build();
+ storage.create(info, note.toJson().getBytes("UTF-8"));
+ }
+
+ private void create(Note note) throws IOException {
+ BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId()))
+ .setContentType("application/json")
+ .build();
+ storage.create(info, note.toJson().getBytes("UTF-8"));
+ }
+
+ private void createMalformed(String noteId) throws IOException {
+ BlobInfo info = BlobInfo.newBuilder(makeBlobId(noteId))
+ .setContentType("application/json")
+ .build();
+ storage.create(info, "{ invalid-json }".getBytes("UTF-8"));
+ }
+
+ /* These tests test path parsing for illegal paths, and do not use the parameterized vars */
+
+ @Test
+ public void testInitialization_pathNotSet() throws Exception {
+ try {
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), "");
+ new GCSNotebookRepo(new ZeppelinConfiguration(), storage);
+ fail();
+ } catch (IOException e) {}
+ }
+
+ @Test
+ public void testInitialization_malformedPath() throws Exception {
+ try {
+ System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_GCS_STORAGE_DIR.getVarName(), "foo");
+ new GCSNotebookRepo(new ZeppelinConfiguration(), storage);
+ fail();
+ } catch (IOException e) {}
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3eea57ab/zeppelin-plugins/notebookrepo/git/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-plugins/notebookrepo/git/pom.xml b/zeppelin-plugins/notebookrepo/git/pom.xml
new file mode 100644
index 0000000..6f4ce8c
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/git/pom.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>zengine-plugins-parent</artifactId>
+ <groupId>org.apache.zeppelin</groupId>
+ <version>0.9.0-SNAPSHOT</version>
+ <relativePath>../../../zeppelin-plugins</relativePath>
+ </parent>
+
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>notebookrepo-git</artifactId>
+ <packaging>jar</packaging>
+ <version>0.9.0-SNAPSHOT</version>
+ <name>Zeppelin: Plugin GitNotebookRepo</name>
+ <description>NotebookRepo implementation based on Git</description>
+
+ <properties>
+ <eclipse.jgit.version>4.5.4.201711221230-r</eclipse.jgit.version>
+ <google.truth.version>0.27</google.truth.version>
+ <plugin.name>NotebookRepo/GitNotebookRepo</plugin.name>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>notebookrepo-vfs</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.eclipse.jgit</groupId>
+ <artifactId>org.eclipse.jgit</artifactId>
+ <version>${eclipse.jgit.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.truth</groupId>
+ <artifactId>truth</artifactId>
+ <version>${google.truth.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+</project>