You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2020/03/10 02:11:53 UTC
[zeppelin] branch master updated: [ZEPPELIN-4652]. Support
OSSNotebookRepo of Aliyun
This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new 13374bd [ZEPPELIN-4652]. Support OSSNotebookRepo of Aliyun
13374bd is described below
commit 13374bd6d94751f731d2debcad5d41f341b6f938
Author: Jeff Zhang <zj...@apache.org>
AuthorDate: Sun Mar 1 18:05:55 2020 +0800
[ZEPPELIN-4652]. Support OSSNotebookRepo of Aliyun
### What is this PR for?
This PR is to support Aliyun's OSS as NotebookRepo. It just add one plugin module `oss` and use OSS's SDK for all the note related operation. Unfortunately, I didn't found way to do unit test, I only tested it manually in my local box.
### What type of PR is it?
[ Feature]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-4652
### How should this be tested?
* CI pass
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? no
* Is there breaking changes for older versions? no
* Does this needs documentation? no
Author: Jeff Zhang <zj...@apache.org>
Closes #3673 from zjffdu/ZEPPELIN-4652 and squashes the following commits:
16f57a85b [Jeff Zhang] [ZEPPELIN-4652]. Support OSSNotebookRepo of Aliyun
---
conf/zeppelin-site.xml.template | 36 ++++
docs/_includes/themes/zeppelin/_navigation.html | 1 +
docs/setup/storage/storage.md | 54 ++++++
.../zeppelin/conf/ZeppelinConfiguration.java | 20 ++
zeppelin-plugins/notebookrepo/oss/pom.xml | 58 ++++++
.../zeppelin/notebook/repo/OSSNotebookRepo.java | 215 +++++++++++++++++++++
zeppelin-plugins/pom.xml | 1 +
7 files changed, 385 insertions(+)
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index b7c3728..f5cd379 100755
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -193,6 +193,42 @@
</property>
-->
+<!-- Aliyun OSS notebook storage -->
+<!-- Creates the following directory structure: oss://{bucket}/{notebook_dir}/note_path -->
+<!--
+
+<property>
+ <name>zeppelin.notebook.oss.bucket</name>
+ <value>zeppelin</value>
+ <description>bucket name for notebook storage</description>
+</property>
+
+<property>
+ <name>zeppelin.notebook.oss.endpoint</name>
+ <value>http://oss-cn-hangzhou.aliyuncs.com</value>
+ <description>endpoint for oss bucket</description>
+</property>
+
+<property>
+ <name>zeppelin.notebook.oss.accesskeyid</name>
+ <value></value>
+ <description>Access key id for your OSS account</description>
+</property>
+
+<property>
+ <name>zeppelin.notebook.oss.accesskeysecret</name>
+ <value></value>
+ <description>Access key secret for your OSS account</description>
+</property>
+
+<property>
+ <name>zeppelin.notebook.storage</name>
+ <value>org.apache.zeppelin.notebook.repo.OSSNotebookRepo</value>
+ <description>notebook persistence layer implementation</description>
+</property>
+
+-->
+
<!-- If using Azure for storage use the following settings -->
<!--
<property>
diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html
index 0a2563f..0940863 100644
--- a/docs/_includes/themes/zeppelin/_navigation.html
+++ b/docs/_includes/themes/zeppelin/_navigation.html
@@ -107,6 +107,7 @@
<li><a href="{{BASE_PATH}}/setup/storage/storage.html#notebook-storage-in-local-git-repository">Git Storage</a></li>
<li><a href="{{BASE_PATH}}/setup/storage/storage.html#notebook-storage-in-s3">S3 Storage</a></li>
<li><a href="{{BASE_PATH}}/setup/storage/storage.html#notebook-storage-in-azure">Azure Storage</a></li>
+ <li><a href="{{BASE_PATH}}/setup/storage/storage.html#notebook-storage-in-oss">OSS Storage</a></li>
<li><a href="{{BASE_PATH}}/setup/storage/storage.html#notebook-storage-in-zeppelinhub">ZeppelinHub Storage</a></li>
<li><a href="{{BASE_PATH}}/setup/storage/storage.html#notebook-storage-in-mongodb">MongoDB Storage</a></li>
<li role="separator" class="divider"></li>
diff --git a/docs/setup/storage/storage.md b/docs/setup/storage/storage.md
index 6ef3453..c1e0997 100644
--- a/docs/setup/storage/storage.md
+++ b/docs/setup/storage/storage.md
@@ -34,6 +34,7 @@ There are few notebook storage systems available for a use out of the box:
* storage using Amazon S3 service - `S3NotebookRepo`
* storage using Azure service - `AzureNotebookRepo`
* storage using Google Cloud Storage - `GCSNotebookRepo`
+ * storage using Aliyun OSS - `OSSNotebookRepo`
* storage using MongoDB - `MongoNotebookRepo`
* storage using GitHub - `GitHubNotebookRepo`
@@ -372,6 +373,59 @@ file for authentication with GCS, update the following property :
</br>
+
+## Notebook Storage in OSS <a name="OSS"></a>
+
+Notebooks may be stored in Aliyun OSS.
+
+</br>
+The following folder structure will be created in OSS:
+
+```
+oss://bucket_name/{noteboo_dir}/note_path
+```
+
+
+And you should configure oss related properties in file **zeppelin-site.xml**.
+
+```xml
+<property>
+ <name>zeppelin.notebook.oss.bucket</name>
+ <value>zeppelin</value>
+ <description>bucket name for notebook storage</description>
+</property>
+
+<property>
+ <name>zeppelin.notebook.oss.endpoint</name>
+ <value>http://oss-cn-hangzhou.aliyuncs.com</value>
+ <description>endpoint for oss bucket</description>
+</property>
+
+<property>
+ <name>zeppelin.notebook.oss.accesskeyid</name>
+ <value></value>
+ <description>Access key id for your OSS account</description>
+</property>
+
+<property>
+ <name>zeppelin.notebook.oss.accesskeysecret</name>
+ <value></value>
+ <description>Access key secret for your OSS account</description>
+</property>
+
+```
+
+Uncomment the next property for use OSSNotebookRepo class:
+
+```xml
+<property>
+ <name>zeppelin.notebook.storage</name>
+ <value>org.apache.zeppelin.notebook.repo.OSSNotebookRepo</value>
+ <description>notebook persistence layer implementation</description>
+</property>
+```
+
+</br>
## Notebook Storage in ZeppelinHub <a name="ZeppelinHub"></a>
ZeppelinHub storage layer allows out of the box connection of Zeppelin instance with your ZeppelinHub account. First of all, you need to either comment out the following property in **zeppelin-site.xml**:
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index dbdb614..afeaa4e 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -474,6 +474,22 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE);
}
+ public String getOSSBucketName() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_OSS_BUCKET);
+ }
+
+ public String getOSSEndpoint() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_OSS_ENDPOINT);
+ }
+
+ public String getOSSAccessKeyId() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_OSS_ACCESSKEYID);
+ }
+
+ public String getOSSAccessKeySecret() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_OSS_ACCESSKEYSECRET);
+ }
+
public String getMongoUri() {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_URI);
}
@@ -873,6 +889,10 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION("zeppelin.notebook.s3.kmsKeyRegion", null),
ZEPPELIN_NOTEBOOK_S3_SSE("zeppelin.notebook.s3.sse", false),
ZEPPELIN_NOTEBOOK_S3_SIGNEROVERRIDE("zeppelin.notebook.s3.signerOverride", null),
+ ZEPPELIN_NOTEBOOK_OSS_BUCKET("zeppelin.notebook.oss.bucket", "zeppelin"),
+ ZEPPELIN_NOTEBOOK_OSS_ENDPOINT("zeppelin.notebook.oss.endpoint", "http://oss-cn-hangzhou.aliyuncs.com"),
+ ZEPPELIN_NOTEBOOK_OSS_ACCESSKEYID("zeppelin.notebook.oss.accesskeyid", null),
+ ZEPPELIN_NOTEBOOK_OSS_ACCESSKEYSECRET("zeppelin.notebook.oss.accesskeysecret", null),
ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING("zeppelin.notebook.azure.connectionString", null),
ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"),
ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"),
diff --git a/zeppelin-plugins/notebookrepo/oss/pom.xml b/zeppelin-plugins/notebookrepo/oss/pom.xml
new file mode 100644
index 0000000..9176009
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/oss/pom.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <artifactId>zengine-plugins-parent</artifactId>
+ <groupId>org.apache.zeppelin</groupId>
+ <version>0.9.0-SNAPSHOT</version>
+ <relativePath>../../../zeppelin-plugins</relativePath>
+ </parent>
+
+ <groupId>org.apache.zeppelin</groupId>
+ <artifactId>notebookrepo-oss</artifactId>
+ <packaging>jar</packaging>
+ <version>0.9.0-SNAPSHOT</version>
+ <name>Zeppelin: Plugin OSSNotebookRepo</name>
+ <description>NotebookRepo implementation based on Aliyun OSS</description>
+
+ <properties>
+ <oss.version>3.8.0</oss.version>
+ <plugin.name>NotebookRepo/OSSNotebookRepo</plugin.name>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.aliyun.oss</groupId>
+ <artifactId>aliyun-sdk-oss</artifactId>
+ <version>${oss.version}</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/OSSNotebookRepo.java b/zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/OSSNotebookRepo.java
new file mode 100644
index 0000000..a7f1cb2
--- /dev/null
+++ b/zeppelin-plugins/notebookrepo/oss/src/main/java/org/apache/zeppelin/notebook/repo/OSSNotebookRepo.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.notebook.repo;
+
+import com.aliyun.oss.OSS;
+import com.aliyun.oss.OSSClientBuilder;
+import com.aliyun.oss.model.CopyObjectRequest;
+import com.aliyun.oss.model.DeleteObjectsRequest;
+import com.aliyun.oss.model.ListObjectsRequest;
+import com.aliyun.oss.model.OSSObject;
+import com.aliyun.oss.model.OSSObjectSummary;
+import com.aliyun.oss.model.ObjectListing;
+import com.aliyun.oss.model.PutObjectRequest;
+import org.apache.commons.io.IOUtils;
+import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.notebook.Note;
+import org.apache.zeppelin.notebook.NoteInfo;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * NotebookRepo for Aliyun OSS (https://cn.aliyun.com/product/oss)
+ */
+public class OSSNotebookRepo implements NotebookRepo {
+ private static final Logger LOGGER = LoggerFactory.getLogger(OSSNotebookRepo.class);
+
+ private OSS ossClient;
+ private String bucketName;
+ private String rootFolder;
+
+ public OSSNotebookRepo() {
+ }
+
+ public void init(ZeppelinConfiguration conf) throws IOException {
+ String endpoint = conf.getOSSEndpoint();
+ bucketName = conf.getOSSBucketName();
+ rootFolder = conf.getNotebookDir();
+ // rootFolder is part of OSS key which doesn't start with '/'
+ if (rootFolder.startsWith("/")) {
+ rootFolder = rootFolder.substring(1);
+ }
+ String accessKeyId = conf.getOSSAccessKeyId();
+ String accessKeySecret = conf.getOSSAccessKeySecret();
+ this.ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret);
+ }
+
+ @Override
+ public Map<String, NoteInfo> list(AuthenticationInfo subject) throws IOException {
+ Map<String, NoteInfo> notesInfo = new HashMap<>();
+ final int maxKeys = 200;
+ String nextMarker = null;
+ ObjectListing objectListing = null;
+ do {
+ objectListing = ossClient.listObjects(
+ new ListObjectsRequest(bucketName)
+ .withPrefix(rootFolder + "/")
+ .withMarker(nextMarker)
+ .withMaxKeys(maxKeys));
+ List<OSSObjectSummary> sums = objectListing.getObjectSummaries();
+ for (OSSObjectSummary s : sums) {
+ if (s.getKey().endsWith(".zpln")) {
+ try {
+ String noteId = getNoteId(s.getKey());
+ String notePath = getNotePath(rootFolder, s.getKey());
+ notesInfo.put(noteId, new NoteInfo(noteId, notePath));
+ } catch (IOException e) {
+ LOGGER.warn(e.getMessage());
+ }
+ } else {
+ LOGGER.debug("Skip invalid note file: " + s.getKey());
+ }
+ }
+ nextMarker = objectListing.getNextMarker();
+ } while (objectListing.isTruncated());
+
+ return notesInfo;
+ }
+
+ @Override
+ public Note get(String noteId, String notePath, AuthenticationInfo subject) throws IOException {
+ OSSObject ossObject = ossClient.getObject(bucketName,
+ rootFolder + "/" + buildNoteFileName(noteId, notePath));
+ InputStream in = null;
+ try {
+ in = ossObject.getObjectContent();
+ return Note.fromJson(IOUtils.toString(in));
+ } finally {
+ if (in != null) {
+ in.close();
+ }
+ }
+ }
+
+ @Override
+ public void save(Note note, AuthenticationInfo subject) throws IOException {
+ String content = note.toJson();
+ PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName,
+ rootFolder + "/" + buildNoteFileName(note.getId(), note.getPath()),
+ new ByteArrayInputStream(content.getBytes()));
+ ossClient.putObject(putObjectRequest);
+ }
+
+ @Override
+ public void move(String noteId, String notePath, String newNotePath,
+ AuthenticationInfo subject) throws IOException {
+ String sourceKey = rootFolder + "/" + buildNoteFileName(noteId, notePath);
+ String destKey = rootFolder + "/" + buildNoteFileName(noteId, newNotePath);
+ CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName,
+ sourceKey, bucketName, destKey);
+ ossClient.copyObject(copyObjectRequest);
+ ossClient.deleteObject(bucketName, sourceKey);
+ }
+
+ @Override
+ public void move(String folderPath, String newFolderPath, AuthenticationInfo subject) {
+ final int maxKeys = 200;
+ String nextMarker = null;
+ ObjectListing objectListing = null;
+ do {
+ objectListing = ossClient.listObjects(
+ new ListObjectsRequest(bucketName)
+ .withPrefix(rootFolder + folderPath + "/")
+ .withMarker(nextMarker)
+ .withMaxKeys(maxKeys));
+ List<OSSObjectSummary> sums = objectListing.getObjectSummaries();
+ for (OSSObjectSummary s : sums) {
+ if (s.getKey().endsWith(".zpln")) {
+ try {
+ String noteId = getNoteId(s.getKey());
+ String notePath = getNotePath(rootFolder, s.getKey());
+ String newNotePath = newFolderPath + notePath.substring(folderPath.length());
+ move(noteId, notePath, newNotePath, subject);
+ } catch (IOException e) {
+ LOGGER.warn(e.getMessage());
+ }
+ } else {
+ LOGGER.debug("Skip invalid note file: " + s.getKey());
+ }
+ }
+ nextMarker = objectListing.getNextMarker();
+ } while (objectListing.isTruncated());
+
+ }
+
+ @Override
+ public void remove(String noteId, String notePath, AuthenticationInfo subject)
+ throws IOException {
+ ossClient.deleteObject(bucketName, rootFolder + "/" + buildNoteFileName(noteId, notePath));
+ }
+
+ @Override
+ public void remove(String folderPath, AuthenticationInfo subject) {
+ String nextMarker = null;
+ ObjectListing objectListing = null;
+ do {
+ ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucketName)
+ .withPrefix(rootFolder + folderPath + "/")
+ .withMarker(nextMarker);
+ objectListing = ossClient.listObjects(listObjectsRequest);
+ if (objectListing.getObjectSummaries().size() > 0) {
+ List<String> keys = new ArrayList();
+ for (OSSObjectSummary s : objectListing.getObjectSummaries()) {
+ keys.add(s.getKey());
+ }
+ DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName).withKeys(keys);
+ ossClient.deleteObjects(deleteObjectsRequest);
+ }
+
+ nextMarker = objectListing.getNextMarker();
+ } while (objectListing.isTruncated());
+ }
+
+ @Override
+ public void close() {
+ ossClient.shutdown();
+ }
+
+ @Override
+ public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
+ LOGGER.warn("Method not implemented");
+ return Collections.emptyList();
+ }
+
+ @Override
+ public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) {
+ LOGGER.warn("Method not implemented");
+ }
+
+}
diff --git a/zeppelin-plugins/pom.xml b/zeppelin-plugins/pom.xml
index fcd3957..5629f16 100644
--- a/zeppelin-plugins/pom.xml
+++ b/zeppelin-plugins/pom.xml
@@ -43,6 +43,7 @@
<module>notebookrepo/zeppelin-hub</module>
<module>notebookrepo/filesystem</module>
<module>notebookrepo/mongo</module>
+ <module>notebookrepo/oss</module>
<module>launcher/k8s-standard</module>
<module>launcher/cluster</module>