You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by jo...@apache.org on 2022/01/09 06:05:37 UTC
[zeppelin] branch master updated: [ZEPPELIN-5582] Remove old NotebookRepo
This is an automated email from the ASF dual-hosted git repository.
jongyoul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new bef579d [ZEPPELIN-5582] Remove old NotebookRepo
bef579d is described below
commit bef579d87f7531480052d8e9451752cae1118e36
Author: Jeff Zhang <zj...@apache.org>
AuthorDate: Sun Nov 7 15:23:33 2021 +0800
[ZEPPELIN-5582] Remove old NotebookRepo
### What is this PR for?
This PR is to remove the old NotebookRepo which is not supported after 0.9, it is only there for upgrading from versions before 0.9. It is time to remove this to reduce maintenance effort.
### What type of PR is it?
[ Improvement ]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-5582)
### How should this be tested?
* CI pass
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #4275 from zjffdu/ZEPPELIN-5582 and squashes the following commits:
f72d98bd3 [Jeff Zhang] [ZEPPELIN-5582] Remove old NotebookRepo
---
bin/upgrade-note.sh | 68 -----
.../notebook/repo/OldAzureNotebookRepo.java | 217 ---------------
.../notebook/repo/OldFileSystemNotebookRepo.java | 93 -------
.../zeppelin/notebook/repo/OldGCSNotebookRepo.java | 226 ----------------
.../notebook/repo/OldGitHubNotebookRepo.java | 130 ---------
.../notebook/repo/OldMongoNotebookRepo.java | 235 ----------------
.../zeppelin/notebook/repo/OldS3NotebookRepo.java | 301 ---------------------
.../zeppelin/notebook/repo/NotebookRepoSync.java | 44 ---
.../zeppelin/notebook/repo/OldGitNotebookRepo.java | 205 --------------
.../zeppelin/notebook/repo/OldNotebookRepo.java | 102 -------
.../repo/OldNotebookRepoWithVersionControl.java | 98 -------
.../zeppelin/notebook/repo/OldVFSNotebookRepo.java | 280 -------------------
.../notebook/repo/UpgradeNoteFileTool.java | 44 ---
.../org/apache/zeppelin/plugin/PluginManager.java | 42 ---
.../apache/zeppelin/plugin/PluginManagerTest.java | 6 -
15 files changed, 2091 deletions(-)
diff --git a/bin/upgrade-note.sh b/bin/upgrade-note.sh
deleted file mode 100755
index e68ef78..0000000
--- a/bin/upgrade-note.sh
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Convert note format from 0.9.0 before to 0.9.0 after
-#
-
-USAGE="Usage: bin/upgrade-note.sh [-d]"
-
-bin=$(dirname "${BASH_SOURCE-$0}")
-bin=$(cd "${bin}">/dev/null; pwd)
-
-. "${bin}/common.sh"
-
-JAVA_OPTS="-Dzeppelin.log.file=logs/upgrade-note.log"
-MAIN_CLASS=org.apache.zeppelin.notebook.repo.UpgradeNoteFileTool
-
-# construct classpath
-if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then
- ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes"
-fi
-
-if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then
- ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes"
-fi
-
-if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then
- ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes"
-fi
-
-addJarInDir "${ZEPPELIN_HOME}"
-addJarInDir "${ZEPPELIN_HOME}/lib"
-addJarInDir "${ZEPPELIN_HOME}/lib/interpreter"
-addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
-addJarInDir "${ZEPPELIN_HOME}/zeppelin-zengine/target/lib"
-addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib"
-
-ZEPPELIN_CLASSPATH="$CLASSPATH:$ZEPPELIN_CLASSPATH"
-
-## Add hadoop jars when env USE_HADOOP is true
-if [[ "${USE_HADOOP}" != "false" ]]; then
- if [[ -z "${HADOOP_CONF_DIR}" ]]; then
- echo "Please specify HADOOP_CONF_DIR if USE_HADOOP is true"
- else
- ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}"
- if ! [ -x "$(command -v hadoop)" ]; then
- echo 'hadoop command is not in PATH when HADOOP_CONF_DIR is specified.'
- else
- ZEPPELIN_CLASSPATH+=":`hadoop classpath`"
- fi
- fi
-fi
-
-exec $ZEPPELIN_RUNNER $JAVA_OPTS -cp $ZEPPELIN_CLASSPATH_OVERRIDES:${ZEPPELIN_CLASSPATH} $MAIN_CLASS "$@"
diff --git a/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/OldAzureNotebookRepo.java b/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/OldAzureNotebookRepo.java
deleted file mode 100644
index 26f6631..0000000
--- a/zeppelin-plugins/notebookrepo/azure/src/main/java/org/apache/zeppelin/notebook/repo/OldAzureNotebookRepo.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.notebook.repo;
-
-import com.microsoft.azure.storage.CloudStorageAccount;
-import com.microsoft.azure.storage.StorageException;
-import com.microsoft.azure.storage.file.CloudFile;
-import com.microsoft.azure.storage.file.CloudFileClient;
-import com.microsoft.azure.storage.file.CloudFileDirectory;
-import com.microsoft.azure.storage.file.CloudFileShare;
-import com.microsoft.azure.storage.file.ListFileItem;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.notebook.Note;
-import org.apache.zeppelin.notebook.OldNoteInfo;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.net.URISyntaxException;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Azure storage backend for notebooks
- */
-public class OldAzureNotebookRepo implements OldNotebookRepo {
- private static final Logger LOG = LoggerFactory.getLogger(OldAzureNotebookRepo.class);
-
- private ZeppelinConfiguration conf;
- private String user;
- private String shareName;
- private CloudFileDirectory rootDir;
-
- public OldAzureNotebookRepo() {
-
- }
-
- @Override
- public void init(ZeppelinConfiguration conf) throws IOException {
- this.conf = conf;
- user = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_USER);
- shareName = conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_SHARE);
-
- try {
- CloudStorageAccount account = CloudStorageAccount.parse(
- conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING));
- CloudFileClient client = account.createCloudFileClient();
- CloudFileShare share = client.getShareReference(shareName);
- share.createIfNotExists();
-
- CloudFileDirectory userDir = StringUtils.isBlank(user) ?
- share.getRootDirectoryReference() :
- share.getRootDirectoryReference().getDirectoryReference(user);
- userDir.createIfNotExists();
-
- rootDir = userDir.getDirectoryReference("notebook");
- rootDir.createIfNotExists();
- } catch (Exception e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
- List<OldNoteInfo> infos = new LinkedList<>();
- OldNoteInfo info = null;
-
- for (ListFileItem item : rootDir.listFilesAndDirectories()) {
- if (item.getClass() == CloudFileDirectory.class) {
- CloudFileDirectory dir = (CloudFileDirectory) item;
-
- try {
- if (dir.getFileReference("note.json").exists()) {
- info = new OldNoteInfo(getNote(dir.getName()));
-
- if (info != null) {
- infos.add(info);
- }
- }
- } catch (StorageException | URISyntaxException e) {
- String msg = "Error enumerating notebooks from Azure storage";
- LOG.error(msg, e);
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- }
- }
- }
-
- return infos;
- }
-
- private Note getNote(String noteId) throws IOException {
- InputStream ins = null;
-
- try {
- CloudFileDirectory dir = rootDir.getDirectoryReference(noteId);
- CloudFile file = dir.getFileReference("note.json");
-
- ins = file.openRead();
- } catch (URISyntaxException | StorageException e) {
- String msg = String.format("Error reading notebook %s from Azure storage", noteId);
-
- LOG.error(msg, e);
-
- throw new IOException(msg, e);
- }
-
- String json = IOUtils.toString(ins,
- conf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_ENCODING));
- ins.close();
- return Note.fromJson(noteId, json);
- }
-
- @Override
- public Note get(String noteId, AuthenticationInfo subject) throws IOException {
- return getNote(noteId);
- }
-
- @Override
- public void save(Note note, AuthenticationInfo subject) throws IOException {
- String json = note.toJson();
-
- ByteArrayOutputStream output = new ByteArrayOutputStream();
- Writer writer = new OutputStreamWriter(output);
- writer.write(json);
- writer.close();
- output.close();
-
- byte[] buffer = output.toByteArray();
-
- try {
- CloudFileDirectory dir = rootDir.getDirectoryReference(note.getId());
- dir.createIfNotExists();
-
- CloudFile cloudFile = dir.getFileReference("note.json");
- cloudFile.uploadFromByteArray(buffer, 0, buffer.length);
- } catch (URISyntaxException | StorageException e) {
- String msg = String.format("Error saving notebook %s to Azure storage", note.getId());
-
- LOG.error(msg, e);
-
- throw new IOException(msg, e);
- }
- }
-
- // unfortunately, we need to use a recursive delete here
- private void delete(ListFileItem item) throws StorageException {
- if (item.getClass() == CloudFileDirectory.class) {
- CloudFileDirectory dir = (CloudFileDirectory) item;
-
- for (ListFileItem subItem : dir.listFilesAndDirectories()) {
- delete(subItem);
- }
-
- dir.deleteIfExists();
- } else if (item.getClass() == CloudFile.class) {
- CloudFile file = (CloudFile) item;
-
- file.deleteIfExists();
- }
- }
-
- @Override
- public void remove(String noteId, AuthenticationInfo subject) throws IOException {
- try {
- CloudFileDirectory dir = rootDir.getDirectoryReference(noteId);
-
- delete(dir);
- } catch (URISyntaxException | StorageException e) {
- String msg = String.format("Error deleting notebook %s from Azure storage", noteId);
-
- LOG.error(msg, e);
-
- throw new IOException(msg, e);
- }
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
- LOG.warn("Method not implemented");
- return Collections.emptyList();
- }
-
- @Override
- public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) {
- LOG.warn("Method not implemented");
- }
-
-}
diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/OldFileSystemNotebookRepo.java b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/OldFileSystemNotebookRepo.java
deleted file mode 100644
index 4406cda..0000000
--- a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/OldFileSystemNotebookRepo.java
+++ /dev/null
@@ -1,93 +0,0 @@
-package org.apache.zeppelin.notebook.repo;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.notebook.FileSystemStorage;
-import org.apache.zeppelin.notebook.Note;
-import org.apache.zeppelin.notebook.OldNoteInfo;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-/**
- * NotebookRepos for hdfs.
- *
- * Assume the notebook directory structure is as following
- * - notebookdir
- * - noteId/note.json
- * - noteId/note.json
- * - noteId/note.json
- */
-public class OldFileSystemNotebookRepo implements OldNotebookRepo {
- private static final Logger LOGGER = LoggerFactory.getLogger(FileSystemNotebookRepo.class);
-
- private FileSystemStorage fs;
- private Path notebookDir;
-
- public OldFileSystemNotebookRepo() {
-
- }
-
- @Override
- public void init(ZeppelinConfiguration zConf) throws IOException {
- this.fs = new FileSystemStorage(zConf,
- zConf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR));
- LOGGER.info("Creating FileSystem: {} for Zeppelin Notebook.", this.fs.getFs().getClass().getName());
- this.notebookDir = this.fs.makeQualified(new Path(zConf.getNotebookDir()));
- LOGGER.info("Using folder {} to store notebook", notebookDir);
- this.fs.tryMkDir(notebookDir);
- }
-
- @Override
- public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
- List<Path> notePaths = fs.list(new Path(notebookDir, "*/note.json"));
- List<OldNoteInfo> noteInfos = new ArrayList<>();
- for (Path path : notePaths) {
- OldNoteInfo noteInfo = new OldNoteInfo(path.getParent().getName(), "", null);
- noteInfos.add(noteInfo);
- }
- return noteInfos;
- }
-
- @Override
- public Note get(final String noteId, AuthenticationInfo subject) throws IOException {
- String content = this.fs.readFile(
- new Path(notebookDir.toString() + "/" + noteId + "/note.json"));
- return Note.fromJson(noteId, content);
- }
-
- @Override
- public void save(final Note note, AuthenticationInfo subject) throws IOException {
- this.fs.writeFile(note.toJson(),
- new Path(notebookDir.toString() + "/" + note.getId() + "/note.json"),
- true);
- }
-
- @Override
- public void remove(final String noteId, AuthenticationInfo subject) throws IOException {
- this.fs.delete(new Path(notebookDir.toString() + "/" + noteId));
- }
-
- @Override
- public void close() {
- LOGGER.warn("close is not implemented for HdfsNotebookRepo");
- }
-
- @Override
- public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
- LOGGER.warn("getSettings is not implemented for HdfsNotebookRepo");
- return Collections.emptyList();
- }
-
- @Override
- public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) {
- LOGGER.warn("updateSettings is not implemented for HdfsNotebookRepo");
- }
-
-}
diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/OldGCSNotebookRepo.java b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/OldGCSNotebookRepo.java
deleted file mode 100644
index 765cfa1..0000000
--- a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/OldGCSNotebookRepo.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.notebook.repo;
-
-import com.google.auth.Credentials;
-import com.google.auth.oauth2.GoogleCredentials;
-import com.google.cloud.storage.Blob;
-import com.google.cloud.storage.BlobId;
-import com.google.cloud.storage.BlobInfo;
-import com.google.cloud.storage.Storage;
-import com.google.cloud.storage.Storage.BlobListOption;
-import com.google.cloud.storage.StorageException;
-import com.google.cloud.storage.StorageOptions;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.gson.JsonParseException;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.notebook.Note;
-import org.apache.zeppelin.notebook.OldNoteInfo;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * A NotebookRepo implementation for storing notebooks in Google Cloud Storage.
- *
- * Notes are stored in the GCS "directory" specified by zeppelin.notebook.gcs.dir. This path
- * must be in the form gs://bucketName/path/to/Dir. The bucket must already exist. N.B: GCS is an
- * object store, so this "directory" should not itself be an object. Instead, it represents the base
- * path for the note.json files.
- *
- * Authentication is provided by google-auth-library-java. A custom json key file path
- * can be specified by zeppelin.notebook.google.credentialsJsonFilePath to connect with GCS
- * If not specified the GOOGLE_APPLICATION_CREDENTIALS will be used to connect to GCS.
- * @see <a href="https://github.com/google/google-auth-library-java">
- * google-auth-library-java</a>.
- */
-public class OldGCSNotebookRepo implements OldNotebookRepo {
-
- private static final Logger LOG = LoggerFactory.getLogger(OldGCSNotebookRepo.class);
- private String encoding;
- private String bucketName;
- private Optional<String> basePath;
- private Pattern noteNamePattern;
- private Storage storage;
-
- public OldGCSNotebookRepo() {
- }
-
- @VisibleForTesting
- public OldGCSNotebookRepo(ZeppelinConfiguration zConf, Storage storage) throws IOException {
- init(zConf);
- this.storage = storage;
- }
-
- @Override
- public void init(ZeppelinConfiguration zConf) throws IOException {
- this.encoding = zConf.getString(ConfVars.ZEPPELIN_ENCODING);
-
- String gcsStorageDir = zConf.getGCSStorageDir();
- if (gcsStorageDir.isEmpty()) {
- throw new IOException("GCS storage directory must be set using 'zeppelin.notebook.gcs.dir'");
- }
- if (!gcsStorageDir.startsWith("gs://")) {
- throw new IOException(String.format(
- "GCS storage directory '%s' must start with 'gs://'.", gcsStorageDir));
- }
- String storageDirWithoutScheme = gcsStorageDir.substring("gs://".length());
-
- // pathComponents excludes empty string if trailing slash is present
- List<String> pathComponents = Arrays.asList(storageDirWithoutScheme.split("/"));
- if (pathComponents.isEmpty()) {
- throw new IOException(String.format(
- "GCS storage directory '%s' must be in the form gs://bucketname/path/to/dir",
- gcsStorageDir));
- }
- this.bucketName = pathComponents.get(0);
- if (pathComponents.size() > 1) {
- this.basePath = Optional.of(StringUtils.join(
- pathComponents.subList(1, pathComponents.size()), "/"));
- } else {
- this.basePath = Optional.empty();
- }
-
- // Notes are stored at gs://bucketName/basePath/<note-id>/note.json
- if (basePath.isPresent()) {
- this.noteNamePattern = Pattern.compile(
- "^" + Pattern.quote(basePath.get() + "/") + "([^/]+)/note\\.json$");
- } else {
- this.noteNamePattern = Pattern.compile("^([^/]+)/note\\.json$");
- }
-
-
- Credentials credentials = GoogleCredentials.getApplicationDefault();
- String credentialJsonPath = zConf.getString(ConfVars.ZEPPELIN_NOTEBOOK_GCS_CREDENTIALS_FILE);
- if (credentialJsonPath != null) {
- credentials = GoogleCredentials.fromStream(new FileInputStream(credentialJsonPath));
- }
- this.storage = StorageOptions.newBuilder().setCredentials(credentials).build().getService();
- }
-
- private BlobId makeBlobId(String noteId) {
- if (basePath.isPresent()) {
- return BlobId.of(bucketName, basePath.get() + "/" + noteId + "/note.json");
- } else {
- return BlobId.of(bucketName, noteId + "/note.json");
- }
- }
-
- @Override
- public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
- try {
- List<OldNoteInfo> infos = new ArrayList<>();
- Iterable<Blob> blobsUnderDir;
- if (basePath.isPresent()) {
- blobsUnderDir = storage
- .list(bucketName, BlobListOption.prefix(this.basePath.get() + "/"))
- .iterateAll();
- } else {
- blobsUnderDir = storage
- .list(bucketName)
- .iterateAll();
- }
- for (Blob b : blobsUnderDir) {
- Matcher matcher = noteNamePattern.matcher(b.getName());
- if (matcher.matches()) {
- // Callers only use the id field, so do not fetch each note
- // This matches the implementation in FileSystemNoteRepo#list
- infos.add(new OldNoteInfo(matcher.group(1), "", null));
- }
- }
- return infos;
- } catch (StorageException se) {
- throw new IOException("Could not list GCS directory: " + se.getMessage(), se);
- }
- }
-
- @Override
- public Note get(String noteId, AuthenticationInfo subject) throws IOException {
- BlobId blobId = makeBlobId(noteId);
- byte[] contents;
- try {
- contents = storage.readAllBytes(blobId);
- } catch (StorageException se) {
- throw new IOException("Could not read " + blobId.toString() + ": " + se.getMessage(), se);
- }
-
- try {
- return Note.fromJson(noteId, new String(contents, encoding));
- } catch (JsonParseException jpe) {
- throw new IOException(
- "Could note parse as json " + blobId.toString() + jpe.getMessage(), jpe);
- }
- }
-
- @Override
- public void save(Note note, AuthenticationInfo subject) throws IOException {
- BlobInfo info = BlobInfo.newBuilder(makeBlobId(note.getId()))
- .setContentType("application/json")
- .build();
- try {
- storage.create(info, note.toJson().getBytes(StandardCharsets.UTF_8));
- } catch (StorageException se) {
- throw new IOException("Could not write " + info.toString() + ": " + se.getMessage(), se);
- }
- }
-
- @Override
- public void remove(String noteId, AuthenticationInfo subject) throws IOException {
- Preconditions.checkArgument(StringUtils.isNotEmpty(noteId));
- BlobId blobId = makeBlobId(noteId);
- try {
- boolean deleted = storage.delete(blobId);
- if (!deleted) {
- throw new IOException("Tried to remove nonexistent blob " + blobId.toString());
- }
- } catch (StorageException se) {
- throw new IOException("Could not remove " + blobId.toString() + ": " + se.getMessage(), se);
- }
- }
-
- @Override
- public void close() {
- //no-op
- }
-
- @Override
- public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
- LOG.warn("getSettings is not implemented for GCSNotebookRepo");
- return Collections.emptyList();
- }
-
- @Override
- public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) {
- LOG.warn("updateSettings is not implemented for GCSNotebookRepo");
- }
-}
diff --git a/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/OldGitHubNotebookRepo.java b/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/OldGitHubNotebookRepo.java
deleted file mode 100644
index a6f86ff..0000000
--- a/zeppelin-plugins/notebookrepo/github/src/main/java/org/apache/zeppelin/notebook/repo/OldGitHubNotebookRepo.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.notebook.repo;
-
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.eclipse.jgit.api.Git;
-import org.eclipse.jgit.api.PullCommand;
-import org.eclipse.jgit.api.PushCommand;
-import org.eclipse.jgit.api.RemoteAddCommand;
-import org.eclipse.jgit.api.errors.GitAPIException;
-import org.eclipse.jgit.transport.URIish;
-import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-
-/**
- * GitHub integration to store notebooks in a GitHub repository.
- * It uses the same simple logic implemented in @see
- * {@link org.apache.zeppelin.notebook.repo.GitNotebookRepo}
- *
- * The logic for updating the local repository from the remote repository is the following:
- * - When the <code>GitHubNotebookRepo</code> is initialized
- * - When pushing the changes to the remote repository
- *
- * The logic for updating the remote repository on GitHub from local repository is the following:
- * - When commit the changes (saving the notebook)
- *
- * You should be able to use this integration with all remote git repositories that accept
- * username + password authentication, not just GitHub.
- */
-public class OldGitHubNotebookRepo extends OldGitNotebookRepo {
- private static final Logger LOG = LoggerFactory.getLogger(GitHubNotebookRepo.class);
- private ZeppelinConfiguration zeppelinConfiguration;
- private Git git;
-
- @Override
- public void init(ZeppelinConfiguration conf) throws IOException {
- super.init(conf);
- LOG.debug("initializing GitHubNotebookRepo");
- this.git = super.getGit();
- this.zeppelinConfiguration = conf;
-
- configureRemoteStream();
- pullFromRemoteStream();
- }
-
- @Override
- public Revision checkpoint(String pattern, String commitMessage, AuthenticationInfo subject) {
- Revision revision = super.checkpoint(pattern, commitMessage, subject);
-
- updateRemoteStream();
-
- return revision;
- }
-
- private void configureRemoteStream() {
- try {
- LOG.debug("Setting up remote stream");
- RemoteAddCommand remoteAddCommand = git.remoteAdd();
- remoteAddCommand.setName(zeppelinConfiguration.getZeppelinNotebookGitRemoteOrigin());
- remoteAddCommand.setUri(new URIish(zeppelinConfiguration.getZeppelinNotebookGitURL()));
- remoteAddCommand.call();
- } catch (GitAPIException e) {
- LOG.error("Error configuring GitHub", e);
- } catch (URISyntaxException e) {
- LOG.error("Error in GitHub URL provided", e);
- }
- }
-
- private void updateRemoteStream() {
- LOG.debug("Updating remote stream");
-
- pullFromRemoteStream();
- pushToRemoteSteam();
- }
-
- private void pullFromRemoteStream() {
- try {
- LOG.debug("Pulling latest changes from remote stream");
- PullCommand pullCommand = git.pull();
- pullCommand.setCredentialsProvider(
- new UsernamePasswordCredentialsProvider(
- zeppelinConfiguration.getZeppelinNotebookGitUsername(),
- zeppelinConfiguration.getZeppelinNotebookGitAccessToken()
- )
- );
-
- pullCommand.call();
-
- } catch (GitAPIException e) {
- LOG.error("Error when pulling latest changes from remote repository", e);
- }
- }
-
- private void pushToRemoteSteam() {
- try {
- LOG.debug("Pushing latest changes to remote stream");
- PushCommand pushCommand = git.push();
- pushCommand.setCredentialsProvider(
- new UsernamePasswordCredentialsProvider(
- zeppelinConfiguration.getZeppelinNotebookGitUsername(),
- zeppelinConfiguration.getZeppelinNotebookGitAccessToken()
- )
- );
-
- pushCommand.call();
- } catch (GitAPIException e) {
- LOG.error("Error when pushing latest changes to remote repository", e);
- }
- }
-}
diff --git a/zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/OldMongoNotebookRepo.java b/zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/OldMongoNotebookRepo.java
deleted file mode 100644
index dbeac44..0000000
--- a/zeppelin-plugins/notebookrepo/mongo/src/main/java/org/apache/zeppelin/notebook/repo/OldMongoNotebookRepo.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.notebook.repo;
-
-import static com.mongodb.client.model.Filters.eq;
-import static com.mongodb.client.model.Filters.in;
-import static com.mongodb.client.model.Filters.type;
-import org.bson.BsonType;
-import org.bson.Document;
-import org.bson.types.ObjectId;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import com.mongodb.MongoBulkWriteException;
-import com.mongodb.MongoClient;
-import com.mongodb.MongoClientURI;
-import com.mongodb.bulk.BulkWriteError;
-import com.mongodb.client.MongoCollection;
-import com.mongodb.client.MongoCursor;
-import com.mongodb.client.MongoDatabase;
-import com.mongodb.client.model.InsertManyOptions;
-import com.mongodb.client.model.UpdateOptions;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.notebook.Note;
-import org.apache.zeppelin.notebook.OldNoteInfo;
-import org.apache.zeppelin.user.AuthenticationInfo;
-
-/**
- * Backend for storing Notebook on MongoDB
- */
-public class OldMongoNotebookRepo implements OldNotebookRepo {
- private static final Logger LOG = LoggerFactory.getLogger(MongoNotebookRepo.class);
-
- private ZeppelinConfiguration conf;
- private MongoClient mongo;
- private MongoDatabase db;
- private MongoCollection<Document> coll;
-
- @Override
- public void init(ZeppelinConfiguration zConf) throws IOException {
- this.conf = zConf;
-
- mongo = new MongoClient(new MongoClientURI(conf.getMongoUri()));
- db = mongo.getDatabase(conf.getMongoDatabase());
- coll = db.getCollection(conf.getMongoCollection());
-
- if (conf.getMongoAutoimport()) {
- // import local notes into MongoDB
- insertFileSystemNotes();
- }
- }
-
- /**
- * If environment variable ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT is true,
- * this method will insert local notes into MongoDB on startup.
- * If a note already exists in MongoDB, skip it.
- */
- private void insertFileSystemNotes() throws IOException {
- LinkedList<Document> docs = new LinkedList<>(); // docs to be imported
- OldNotebookRepo vfsRepo = new OldVFSNotebookRepo();
- vfsRepo.init(this.conf);
- List<OldNoteInfo> infos = vfsRepo.list(null);
- // collect notes to be imported
- for (OldNoteInfo info : infos) {
- Note note = vfsRepo.get(info.getId(), null);
- Document doc = noteToDocument(note);
- docs.add(doc);
- }
-
- /*
- * 'ordered(false)' option allows to proceed bulk inserting even though
- * there are duplicated documents. The duplicated documents will be skipped
- * and print a WARN log.
- */
- try {
- coll.insertMany(docs, new InsertManyOptions().ordered(false));
- } catch (MongoBulkWriteException e) {
- printDuplicatedException(e); //print duplicated document warning log
- }
-
- vfsRepo.close(); // it does nothing for now but maybe in the future...
- }
-
- /**
- * MongoBulkWriteException contains error messages that inform
- * which documents were duplicated. This method catches those ID and print them.
- * @param e
- */
- private void printDuplicatedException(MongoBulkWriteException e) {
- List<BulkWriteError> errors = e.getWriteErrors();
- for (BulkWriteError error : errors) {
- String msg = error.getMessage();
- Pattern pattern = Pattern.compile("[A-Z0-9]{9}"); // regex for note ID
- Matcher matcher = pattern.matcher(msg);
- if (matcher.find()) { // if there were a note ID
- String noteId = matcher.group();
- LOG.warn("Note " + noteId + " not inserted since already exists in MongoDB");
- }
- }
- }
-
- @Override
- public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
- syncId();
-
- List<OldNoteInfo> infos = new LinkedList<>();
- MongoCursor<Document> cursor = coll.find().iterator();
-
- while (cursor.hasNext()) {
- Document doc = cursor.next();
- Note note = documentToNote(null, doc);
- OldNoteInfo info = new OldNoteInfo(note);
- infos.add(info);
- }
-
- cursor.close();
-
- return infos;
- }
-
- /**
- * Find documents of which type of _id is object ID, and change it to note ID.
- * Since updating _id field is not allowed, remove original documents and insert
- * new ones with string _id(note ID)
- */
- private void syncId() {
- // find documents whose id type is object id
- MongoCursor<Document> cursor = coll.find(type("_id", BsonType.OBJECT_ID)).iterator();
- // if there is no such document, exit
- if (!cursor.hasNext())
- return;
-
- List<ObjectId> oldDocIds = new LinkedList<>(); // document ids need to update
- List<Document> updatedDocs = new LinkedList<>(); // new documents to be inserted
-
- while (cursor.hasNext()) {
- Document doc = cursor.next();
- // store original _id
- ObjectId oldId = doc.getObjectId("_id");
- oldDocIds.add(oldId);
- // store the document with string _id (note id)
- String noteId = doc.getString("id");
- doc.put("_id", noteId);
- updatedDocs.add(doc);
- }
-
- coll.insertMany(updatedDocs);
- coll.deleteMany(in("_id", oldDocIds));
-
- cursor.close();
- }
-
- /**
- * Convert document to note
- */
- private Note documentToNote(String noteId, Document doc) throws IOException {
- // document to JSON
- String json = doc.toJson();
- // JSON to note
- return Note.fromJson(noteId, json);
- }
-
- /**
- * Convert note to document
- */
- private Document noteToDocument(Note note) {
- // note to JSON
- String json = note.toJson();
- // JSON to document
- Document doc = Document.parse(json);
- // set object id as note id
- doc.put("_id", note.getId());
- return doc;
- }
-
- @Override
- public Note get(String noteId, AuthenticationInfo subject) throws IOException {
- Document doc = coll.find(eq("_id", noteId)).first();
-
- if (doc == null) {
- throw new IOException("Note " + noteId + "not found");
- }
-
- return documentToNote(noteId, doc);
- }
-
- @Override
- public void save(Note note, AuthenticationInfo subject) throws IOException {
- Document doc = noteToDocument(note);
- coll.replaceOne(eq("_id", note.getId()), doc, new UpdateOptions().upsert(true));
- }
-
- @Override
- public void remove(String noteId, AuthenticationInfo subject) throws IOException {
- coll.deleteOne(eq("_id", noteId));
- }
-
- @Override
- public void close() {
- mongo.close();
- }
-
- @Override
- public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
- LOG.warn("Method not implemented");
- return Collections.emptyList();
- }
-
- @Override
- public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) {
- LOG.warn("Method not implemented");
- }
-
-}
diff --git a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/OldS3NotebookRepo.java b/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/OldS3NotebookRepo.java
deleted file mode 100644
index cae6d0b..0000000
--- a/zeppelin-plugins/notebookrepo/s3/src/main/java/org/apache/zeppelin/notebook/repo/OldS3NotebookRepo.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.notebook.repo;
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.ClientConfigurationFactory;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
-import com.amazonaws.regions.Region;
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.AmazonS3EncryptionClient;
-import com.amazonaws.services.s3.model.CannedAccessControlList;
-import com.amazonaws.services.s3.model.CryptoConfiguration;
-import com.amazonaws.services.s3.model.EncryptionMaterialsProvider;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.KMSEncryptionMaterialsProvider;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PutObjectRequest;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.notebook.Note;
-import org.apache.zeppelin.notebook.NoteInfo;
-import org.apache.zeppelin.notebook.OldNoteInfo;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Backend for storing Notebooks on S3
- */
-public class OldS3NotebookRepo implements OldNotebookRepo {
- private static final Logger LOG = LoggerFactory.getLogger(S3NotebookRepo.class);
-
- // Use a credential provider chain so that instance profiles can be utilized
- // on an EC2 instance. The order of locations where credentials are searched
- // is documented here
- //
- // http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/
- // auth/DefaultAWSCredentialsProviderChain.html
- //
- // In summary, the order is:
- //
- // 1. Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
- // 2. Java System Properties - aws.accessKeyId and aws.secretKey
- // 3. Credential profiles file at the default location (~/.aws/credentials)
- // shared by all AWS SDKs and the AWS CLI
- // 4. Instance profile credentials delivered through the Amazon EC2 metadata service
- private AmazonS3 s3client;
- private String bucketName;
- private String user;
- private boolean useServerSideEncryption;
- private CannedAccessControlList objectCannedAcl;
- private ZeppelinConfiguration conf;
-
- public OldS3NotebookRepo() {
-
- }
-
- public void init(ZeppelinConfiguration conf) throws IOException {
- this.conf = conf;
- bucketName = conf.getS3BucketName();
- user = conf.getS3User();
- useServerSideEncryption = conf.isS3ServerSideEncryption();
- if (StringUtils.isNotBlank(conf.getS3CannedAcl())) {
- objectCannedAcl = CannedAccessControlList.valueOf(conf.getS3CannedAcl());
- }
-
- // always use the default provider chain
- AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();
- CryptoConfiguration cryptoConf = new CryptoConfiguration();
- String keyRegion = conf.getS3KMSKeyRegion();
-
- if (StringUtils.isNotBlank(keyRegion)) {
- cryptoConf.setAwsKmsRegion(Region.getRegion(Regions.fromName(keyRegion)));
- }
-
- ClientConfiguration cliConf = createClientConfiguration();
-
- // see if we should be encrypting data in S3
- String kmsKeyID = conf.getS3KMSKeyID();
- if (kmsKeyID != null) {
- // use the AWS KMS to encrypt data
- KMSEncryptionMaterialsProvider emp = new KMSEncryptionMaterialsProvider(kmsKeyID);
- this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf);
- }
- else if (conf.getS3EncryptionMaterialsProviderClass() != null) {
- // use a custom encryption materials provider class
- EncryptionMaterialsProvider emp = createCustomProvider(conf);
- this.s3client = new AmazonS3EncryptionClient(credentialsProvider, emp, cliConf, cryptoConf);
- }
- else {
- // regular S3
- this.s3client = new AmazonS3Client(credentialsProvider, cliConf);
- }
-
- // set S3 endpoint to use
- s3client.setEndpoint(conf.getS3Endpoint());
- }
-
- /**
- * Create an instance of a custom encryption materials provider class
- * which supplies encryption keys to use when reading/writing data in S3.
- */
- private EncryptionMaterialsProvider createCustomProvider(ZeppelinConfiguration conf)
- throws IOException {
- // use a custom encryption materials provider class
- String empClassname = conf.getS3EncryptionMaterialsProviderClass();
- EncryptionMaterialsProvider emp;
- try {
- Object empInstance = Class.forName(empClassname).newInstance();
- if (empInstance instanceof EncryptionMaterialsProvider) {
- emp = (EncryptionMaterialsProvider) empInstance;
- }
- else {
- throw new IOException("Class " + empClassname + " does not implement "
- + EncryptionMaterialsProvider.class.getName());
- }
- }
- catch (Exception e) {
- throw new IOException("Unable to instantiate encryption materials provider class "
- + empClassname + ": " + e, e);
- }
-
- return emp;
- }
-
- /**
- * Create AWS client configuration and return it.
- * @return AWS client configuration
- */
- private ClientConfiguration createClientConfiguration() {
- ClientConfigurationFactory configFactory = new ClientConfigurationFactory();
- ClientConfiguration config = configFactory.getConfig();
-
- String s3SignerOverride = conf.getS3SignerOverride();
- if (StringUtils.isNotBlank(s3SignerOverride)) {
- config.setSignerOverride(s3SignerOverride);
- }
-
- return config;
- }
-
- @Override
- public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
- List<OldNoteInfo> infos = new LinkedList<>();
- OldNoteInfo info;
- try {
- ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
- .withBucketName(bucketName)
- .withPrefix(user + "/" + "notebook");
- ObjectListing objectListing;
- do {
- objectListing = s3client.listObjects(listObjectsRequest);
- for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
- if (objectSummary.getKey().endsWith("note.json")) {
- info = getNoteInfo(objectSummary.getKey());
- if (info != null) {
- infos.add(info);
- } else {
- LOG.debug("Unable to get notebook info for key: " + objectSummary.getKey());
- }
- }
- }
- listObjectsRequest.setMarker(objectListing.getNextMarker());
- } while (objectListing.isTruncated());
- } catch (AmazonClientException ace) {
- throw new IOException("Unable to list objects in S3: " + ace, ace);
- }
- return infos;
- }
-
- private Note getNote(String key) throws IOException {
- S3Object s3object;
- try {
- s3object = s3client.getObject(new GetObjectRequest(bucketName, key));
- }
- catch (AmazonClientException ace) {
- throw new IOException("Unable to retrieve object from S3: " + ace, ace);
- }
-
- try (InputStream ins = s3object.getObjectContent()) {
- String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
- return Note.fromJson(null, json);
- }
- }
-
- private OldNoteInfo getNoteInfo(String key) throws IOException {
- Note note = getNote(key);
- return note != null ? new OldNoteInfo(note) : null;
- }
-
- @Override
- public Note get(String noteId, AuthenticationInfo subject) throws IOException {
- return getNote(user + "/" + "notebook" + "/" + noteId + "/" + "note.json");
- }
-
- @Override
- public void save(Note note, AuthenticationInfo subject) throws IOException {
- String json = note.toJson();
- String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json";
-
- File file = File.createTempFile("note", "json");
- try {
- Writer writer = new OutputStreamWriter(new FileOutputStream(file));
- writer.write(json);
- writer.close();
-
- PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, file);
-
- if (useServerSideEncryption) {
- // Request server-side encryption.
- ObjectMetadata objectMetadata = new ObjectMetadata();
- objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
- putRequest.setMetadata(objectMetadata);
- }
- if (objectCannedAcl != null) {
- putRequest.withCannedAcl(objectCannedAcl);
- }
- s3client.putObject(putRequest);
- }
- catch (AmazonClientException ace) {
- throw new IOException("Unable to store note in S3: " + ace, ace);
- }
- finally {
- FileUtils.deleteQuietly(file);
- }
- }
-
- @Override
- public void remove(String noteId, AuthenticationInfo subject) throws IOException {
- String key = user + "/" + "notebook" + "/" + noteId;
- final ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
- .withBucketName(bucketName).withPrefix(key);
-
- try {
- ObjectListing objects = s3client.listObjects(listObjectsRequest);
- do {
- for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
- s3client.deleteObject(bucketName, objectSummary.getKey());
- }
- objects = s3client.listNextBatchOfObjects(objects);
- } while (objects.isTruncated());
- }
- catch (AmazonClientException ace) {
- throw new IOException("Unable to remove note in S3: " + ace, ace);
- }
- }
-
- @Override
- public void close() {
- //no-op
- }
-
- @Override
- public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
- LOG.warn("Method not implemented");
- return Collections.emptyList();
- }
-
- @Override
- public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) {
- LOG.warn("Method not implemented");
- }
-
-}
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
index 21cef25..186aa61 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/NotebookRepoSync.java
@@ -102,50 +102,6 @@ public class NotebookRepoSync implements NotebookRepoWithVersionControl {
}
}
- // Zeppelin change its note file name structure in 0.9.0, this is called when upgrading
- // from 0.9.0 before to 0.9.0 after
- public void convertNoteFiles(ZeppelinConfiguration conf, boolean deleteOld) throws IOException {
- // convert old note file (noteId/note.json) to new note file (note_name_note_id.zpln)
- for (int i = 0; i < repos.size(); ++i) {
- NotebookRepo newNotebookRepo = repos.get(i);
- OldNotebookRepo oldNotebookRepo =
- PluginManager.get().loadOldNotebookRepo(newNotebookRepo.getClass().getCanonicalName());
- oldNotebookRepo.init(conf);
- List<OldNoteInfo> oldNotesInfo = oldNotebookRepo.list(AuthenticationInfo.ANONYMOUS);
- LOGGER.info("Convert old note file to new style, note count: {}", oldNotesInfo.size());
- LOGGER.info("Delete old note: {}", deleteOld);
- for (OldNoteInfo oldNoteInfo : oldNotesInfo) {
- LOGGER.info("Converting note, id: {}", oldNoteInfo.getId());
- Note note = oldNotebookRepo.get(oldNoteInfo.getId(), AuthenticationInfo.ANONYMOUS);
- note.setPath(note.getName());
- note.setVersion(Util.getVersion());
- newNotebookRepo.save(note, AuthenticationInfo.ANONYMOUS);
- if (newNotebookRepo instanceof NotebookRepoWithVersionControl) {
- ((NotebookRepoWithVersionControl) newNotebookRepo).checkpoint(
- note.getId(),
- note.getPath(),
- "Upgrade note '" + note.getName() + "' to " + Util.getVersion(),
- AuthenticationInfo.ANONYMOUS);
- }
- if (deleteOld) {
- oldNotebookRepo.remove(note.getId(), AuthenticationInfo.ANONYMOUS);
- LOGGER.info("Remote old note: {}", note.getId());
- // TODO(zjffdu) no commit when deleting note, This is an issue of
- // NotebookRepoWithVersionControl
- /**
- if (oldNotebookRepo instanceof NotebookRepoWithVersionControl) {
- ((NotebookRepoWithVersionControl) oldNotebookRepo).checkpoint(
- note.getId(),
- note.getName(),
- "Delete note '" + note.getName() + "' during note upgrade",
- AuthenticationInfo.ANONYMOUS);
- }
- **/
- }
- }
- }
- }
-
public List<NotebookRepoWithSettings> getNotebookRepos(AuthenticationInfo subject) {
List<NotebookRepoWithSettings> reposSetting = new ArrayList<>();
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldGitNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldGitNotebookRepo.java
deleted file mode 100644
index ae0503a..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldGitNotebookRepo.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.notebook.repo;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.notebook.Note;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.eclipse.jgit.api.Git;
-import org.eclipse.jgit.api.errors.GitAPIException;
-import org.eclipse.jgit.api.errors.NoHeadException;
-import org.eclipse.jgit.diff.DiffEntry;
-import org.eclipse.jgit.dircache.DirCache;
-import org.eclipse.jgit.internal.storage.file.FileRepository;
-import org.eclipse.jgit.lib.Constants;
-import org.eclipse.jgit.lib.ObjectId;
-import org.eclipse.jgit.lib.Repository;
-import org.eclipse.jgit.revwalk.RevCommit;
-import org.eclipse.jgit.treewalk.filter.PathFilter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-/**
- * NotebookRepo that hosts all the notebook FS in a single Git repo
- *
- * This impl intended to be simple and straightforward:
- * - does not handle branches
- * - only basic local git file repo, no remote Github push\pull. GitHub integration is
- * implemented in @see {@link org.apache.zeppelin.notebook.repo.GitNotebookRepo}
- *
- * TODO(bzz): add default .gitignore
- */
-public class OldGitNotebookRepo extends OldVFSNotebookRepo
- implements OldNotebookRepoWithVersionControl {
- private static final Logger LOG = LoggerFactory.getLogger(GitNotebookRepo.class);
-
- private String localPath;
- private Git git;
-
- public OldGitNotebookRepo() {
- super();
- }
-
- @VisibleForTesting
- public OldGitNotebookRepo(ZeppelinConfiguration conf) throws IOException {
- this();
- init(conf);
- }
-
- @Override
- public void init(ZeppelinConfiguration conf) throws IOException {
- //TODO(zjffdu), it is weird that I can not call super.init directly here, as it would cause
- //AbstractMethodError
- this.conf = conf;
- setNotebookDirectory(conf.getNotebookDir());
-
- localPath = getRootDir().getName().getPath();
- LOG.info("Opening a git repo at '{}'", localPath);
- Repository localRepo = new FileRepository(String.join(File.separator, localPath, ".git"));
- if (!localRepo.getDirectory().exists()) {
- LOG.info("Git repo {} does not exist, creating a new one", localRepo.getDirectory());
- localRepo.create();
- }
- git = new Git(localRepo);
- }
-
- @Override
- public synchronized void save(Note note, AuthenticationInfo subject) throws IOException {
- super.save(note, subject);
- }
-
- /* implemented as git add+commit
- * @param pattern is the noteId
- * @param commitMessage is a commit message (checkpoint message)
- * (non-Javadoc)
- * @see org.apache.zeppelin.notebook.repo.VFSNotebookRepo#checkpoint(String, String)
- */
- @Override
- public Revision checkpoint(String pattern, String commitMessage, AuthenticationInfo subject) {
- Revision revision = Revision.EMPTY;
- try {
- List<DiffEntry> gitDiff = git.diff().call();
- if (!gitDiff.isEmpty()) {
- LOG.debug("Changes found for pattern '{}': {}", pattern, gitDiff);
- DirCache added = git.add().addFilepattern(pattern).call();
- LOG.debug("{} changes are about to be commited", added.getEntryCount());
- RevCommit commit = git.commit().setMessage(commitMessage).call();
- revision = new Revision(commit.getName(), commit.getShortMessage(), commit.getCommitTime());
- } else {
- LOG.debug("No changes found {}", pattern);
- }
- } catch (GitAPIException e) {
- LOG.error("Failed to add+commit {} to Git", pattern, e);
- }
- return revision;
- }
-
- /**
- * the idea is to:
- * 1. stash current changes
- * 2. remember head commit and checkout to the desired revision
- * 3. get note and checkout back to the head
- * 4. apply stash on top and remove it
- */
- @Override
- public synchronized Note get(String noteId, String revId, AuthenticationInfo subject)
- throws IOException {
- Note note = null;
- RevCommit stash = null;
- try {
- List<DiffEntry> gitDiff = git.diff().setPathFilter(PathFilter.create(noteId)).call();
- boolean modified = !gitDiff.isEmpty();
- if (modified) {
- // stash changes
- stash = git.stashCreate().call();
- Collection<RevCommit> stashes = git.stashList().call();
- LOG.debug("Created stash : {}, stash size : {}", stash, stashes.size());
- }
- ObjectId head = git.getRepository().resolve(Constants.HEAD);
- // checkout to target revision
- git.checkout().setStartPoint(revId).addPath(noteId).call();
- // get the note
- note = super.get(noteId, subject);
- // checkout back to head
- git.checkout().setStartPoint(head.getName()).addPath(noteId).call();
- if (modified && stash != null) {
- // unstash changes
- ObjectId applied = git.stashApply().setStashRef(stash.getName()).call();
- ObjectId dropped = git.stashDrop().setStashRef(0).call();
- Collection<RevCommit> stashes = git.stashList().call();
- LOG.debug("Stash applied as : {}, and dropped : {}, stash size: {}", applied, dropped,
- stashes.size());
- }
- } catch (GitAPIException e) {
- LOG.error("Failed to return note from revision \"{}\"", revId, e);
- }
- return note;
- }
-
- @Override
- public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject) {
- List<Revision> history = new ArrayList<>();
- LOG.debug("Listing history for {}:", noteId);
- try {
- Iterable<RevCommit> logs = git.log().addPath(noteId).call();
- for (RevCommit log: logs) {
- history.add(new Revision(log.getName(), log.getShortMessage(), log.getCommitTime()));
- LOG.debug(" - ({},{},{})", log.getName(), log.getCommitTime(), log.getFullMessage());
- }
- } catch (NoHeadException e) {
- //when no initial commit exists
- LOG.warn("No Head found for {}, {}", noteId, e.getMessage());
- } catch (GitAPIException e) {
- LOG.error("Failed to get logs for {}", noteId, e);
- }
- return history;
- }
-
- @Override
- public Note setNoteRevision(String noteId, String revId, AuthenticationInfo subject)
- throws IOException {
- Note revisionNote = get(noteId, revId, subject);
- if (revisionNote != null) {
- save(revisionNote, subject);
- }
- return revisionNote;
- }
-
- @Override
- public void close() {
- git.getRepository().close();
- }
-
- //DI replacements for Tests
- protected Git getGit() {
- return git;
- }
-
- void setGit(Git git) {
- this.git = git;
- }
-
-}
-
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldNotebookRepo.java
deleted file mode 100644
index 38c3f90..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldNotebookRepo.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.notebook.repo;
-
-import org.apache.zeppelin.annotation.ZeppelinApi;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.notebook.Note;
-import org.apache.zeppelin.notebook.OldNoteInfo;
-import org.apache.zeppelin.user.AuthenticationInfo;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Notebook repository (persistence layer) abstraction
- */
-public interface OldNotebookRepo {
-
- void init(ZeppelinConfiguration zConf) throws IOException;
-
- /**
- * Lists notebook information about all notebooks in storage.
- * @param subject contains user information.
- * @return
- * @throws IOException
- */
- @ZeppelinApi
- public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException;
-
- /**
- * Get the notebook with the given id.
- * @param noteId is note id.
- * @param subject contains user information.
- * @return
- * @throws IOException
- */
- @ZeppelinApi
- public Note get(String noteId, AuthenticationInfo subject) throws IOException;
-
- /**
- * Save given note in storage
- * @param note is the note itself.
- * @param subject contains user information.
- * @throws IOException
- */
- @ZeppelinApi
- public void save(Note note, AuthenticationInfo subject) throws IOException;
-
- /**
- * Remove note with given id.
- * @param noteId is the note id.
- * @param subject contains user information.
- * @throws IOException
- */
- @ZeppelinApi
- public void remove(String noteId, AuthenticationInfo subject) throws IOException;
-
- /**
- * Release any underlying resources
- */
- @ZeppelinApi
- public void close();
-
- /**
- * Versioning API (optional, preferred to have).
- */
-
- /**
- * Get NotebookRepo settings got the given user.
- *
- * @param subject
- * @return
- */
- @ZeppelinApi
- public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject);
-
- /**
- * update notebook repo settings.
- *
- * @param settings
- * @param subject
- */
- @ZeppelinApi
- public void updateSettings(Map<String, String> settings, AuthenticationInfo subject);
-
-}
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldNotebookRepoWithVersionControl.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldNotebookRepoWithVersionControl.java
deleted file mode 100644
index 62fdc09..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldNotebookRepoWithVersionControl.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.notebook.repo;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.zeppelin.annotation.ZeppelinApi;
-import org.apache.zeppelin.notebook.Note;
-import org.apache.zeppelin.user.AuthenticationInfo;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Notebook repository (persistence layer) abstraction
- */
-public interface OldNotebookRepoWithVersionControl extends OldNotebookRepo {
-
- /**
- * chekpoint (set revision) for notebook.
- * @param noteId Id of the Notebook
- * @param checkpointMsg message description of the checkpoint
- * @return Rev
- * @throws IOException
- */
- @ZeppelinApi
- public Revision checkpoint(String noteId, String checkpointMsg,
- AuthenticationInfo subject) throws IOException;
-
- /**
- * Get particular revision of the Notebook.
- *
- * @param noteId Id of the Notebook
- * @param revId revision of the Notebook
- * @return a Notebook
- * @throws IOException
- */
- @ZeppelinApi
- public Note get(String noteId, String revId, AuthenticationInfo subject)
- throws IOException;
-
- /**
- * List of revisions of the given Notebook.
- *
- * @param noteId id of the Notebook
- * @return list of revisions
- */
- @ZeppelinApi
- public List<Revision> revisionHistory(String noteId, AuthenticationInfo subject);
-
- /**
- * Set note to particular revision.
- *
- * @param noteId Id of the Notebook
- * @param revId revision of the Notebook
- * @return a Notebook
- * @throws IOException
- */
- @ZeppelinApi
- public Note setNoteRevision(String noteId, String revId, AuthenticationInfo subject)
- throws IOException;
-
- /**
- * Represents the 'Revision' a point in life of the notebook
- */
- static class Revision {
- public static final Revision EMPTY = new Revision(StringUtils.EMPTY, StringUtils.EMPTY, 0);
-
- public String id;
- public String message;
- public int time;
-
- public Revision(String revId, String message, int time) {
- this.id = revId;
- this.message = message;
- this.time = time;
- }
-
- public static boolean isEmpty(Revision revision) {
- return revision == null || EMPTY.equals(revision);
- }
- }
-
-}
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldVFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldVFSNotebookRepo.java
deleted file mode 100644
index aefe2d2..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldVFSNotebookRepo.java
+++ /dev/null
@@ -1,280 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.notebook.repo;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.commons.vfs2.FileContent;
-import org.apache.commons.vfs2.FileObject;
-import org.apache.commons.vfs2.FileSystemManager;
-import org.apache.commons.vfs2.FileType;
-import org.apache.commons.vfs2.NameScope;
-import org.apache.commons.vfs2.Selectors;
-import org.apache.commons.vfs2.VFS;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
-import org.apache.zeppelin.notebook.Note;
-import org.apache.zeppelin.notebook.OldNoteInfo;
-import org.apache.zeppelin.user.AuthenticationInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-/**
-*
-*/
-public class OldVFSNotebookRepo implements OldNotebookRepo {
- private static final Logger LOG = LoggerFactory.getLogger(OldVFSNotebookRepo.class);
-
- private FileSystemManager fsManager;
- private URI filesystemRoot;
- protected ZeppelinConfiguration conf;
-
- @Override
- public void init(ZeppelinConfiguration conf) throws IOException {
- this.conf = conf;
- setNotebookDirectory(conf.getNotebookDir());
- }
-
- protected void setNotebookDirectory(String notebookDirPath) throws IOException {
- try {
- LOG.info("Using notebookDir: {}", notebookDirPath);
- if (conf.isWindowsPath(notebookDirPath)) {
- filesystemRoot = new File(notebookDirPath).toURI();
- } else {
- filesystemRoot = new URI(notebookDirPath);
- }
- } catch (URISyntaxException e1) {
- throw new IOException(e1);
- }
-
- if (filesystemRoot.getScheme() == null) { // it is local path
- File f = new File(conf.getAbsoluteDir(filesystemRoot.getPath()));
- this.filesystemRoot = f.toURI();
- }
-
- fsManager = VFS.getManager();
- FileObject file = fsManager.resolveFile(filesystemRoot.getPath());
- if (!file.exists()) {
- LOG.info("Notebook dir doesn't exist, create on is {}.", file.getName());
- file.createFolder();
- }
- }
-
- private String getNotebookDirPath() {
- return filesystemRoot.getPath();
- }
-
- private String getPath(String path) {
- if (path == null || path.trim().length() == 0) {
- return filesystemRoot.toString();
- }
- if (path.startsWith("/")) {
- return filesystemRoot.toString() + path;
- } else {
- return filesystemRoot.toString() + "/" + path;
- }
- }
-
- private boolean isDirectory(FileObject fo) throws IOException {
- if (fo == null) {
- return false;
- }
- return fo.getType() == FileType.FOLDER;
- }
-
- @Override
- public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
- FileObject rootDir = getRootDir();
-
- FileObject[] children = rootDir.getChildren();
-
- List<OldNoteInfo> infos = new LinkedList<>();
- for (FileObject f : children) {
- String fileName = f.getName().getBaseName();
- if (f.isHidden()
- || fileName.startsWith(".")
- || fileName.startsWith("#")
- || fileName.startsWith("~")) {
- // skip hidden, temporary files
- continue;
- }
-
- if (!isDirectory(f)) {
- // currently single note is saved like, [NOTE_ID]/note.json.
- // so it must be a directory
- continue;
- }
-
- OldNoteInfo info = null;
-
- try {
- info = getNoteInfo(f);
- if (info != null) {
- infos.add(info);
- }
- } catch (Exception e) {
- LOG.error("Can't read note {}", f.getName());
- }
- }
-
- return infos;
- }
-
- private Note getNote(FileObject noteDir) throws IOException {
- if (!isDirectory(noteDir)) {
- throw new IOException(noteDir.getName().toString() + " is not a directory");
- }
-
- FileObject noteJson = noteDir.resolveFile("note.json", NameScope.CHILD);
- if (!noteJson.exists()) {
- throw new IOException(noteJson.getName().toString() + " not found");
- }
-
- FileContent content = noteJson.getContent();
- InputStream ins = content.getInputStream();
- String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
- ins.close();
-
- return Note.fromJson(null, json);
- }
-
- private OldNoteInfo getNoteInfo(FileObject noteDir) throws IOException {
- Note note = getNote(noteDir);
- return new OldNoteInfo(note);
- }
-
- @Override
- public Note get(String noteId, AuthenticationInfo subject) throws IOException {
- FileObject rootDir = fsManager.resolveFile(getPath("/"));
- FileObject noteDir = rootDir.resolveFile(noteId, NameScope.CHILD);
-
- return getNote(noteDir);
- }
-
- protected FileObject getRootDir() throws IOException {
- FileObject rootDir = fsManager.resolveFile(getPath("/"));
-
- if (!rootDir.exists()) {
- throw new IOException("Root path does not exists");
- }
-
- if (!isDirectory(rootDir)) {
- throw new IOException("Root path is not a directory");
- }
-
- return rootDir;
- }
-
- @Override
- public synchronized void save(Note note, AuthenticationInfo subject) throws IOException {
- LOG.info("Saving note: {}", note.getId());
- String json = note.toJson();
-
- FileObject rootDir = getRootDir();
-
- FileObject noteDir = rootDir.resolveFile(note.getId(), NameScope.CHILD);
-
- if (!noteDir.exists()) {
- noteDir.createFolder();
- }
- if (!isDirectory(noteDir)) {
- throw new IOException(noteDir.getName().toString() + " is not a directory");
- }
-
- FileObject noteJson = noteDir.resolveFile(".note.json", NameScope.CHILD);
- // false means not appending. creates file if not exists
- OutputStream out = noteJson.getContent().getOutputStream(false);
- out.write(json.getBytes(conf.getString(ConfVars.ZEPPELIN_ENCODING)));
- out.close();
- noteJson.moveTo(noteDir.resolveFile("note.json", NameScope.CHILD));
- }
-
- @Override
- public void remove(String noteId, AuthenticationInfo subject) throws IOException {
- FileObject rootDir = fsManager.resolveFile(getPath("/"));
- FileObject noteDir = rootDir.resolveFile(noteId, NameScope.CHILD);
-
- if (!noteDir.exists()) {
- // nothing to do
- return;
- }
-
- if (!isDirectory(noteDir)) {
- // it is not look like zeppelin note savings
- throw new IOException("Can not remove " + noteDir.getName().toString());
- }
-
- noteDir.delete(Selectors.SELECT_SELF_AND_CHILDREN);
- }
-
- @Override
- public void close() {
- //no-op
- }
-
- @Override
- public List<NotebookRepoSettingsInfo> getSettings(AuthenticationInfo subject) {
- NotebookRepoSettingsInfo repoSetting = NotebookRepoSettingsInfo.newInstance();
- List<NotebookRepoSettingsInfo> settings = new ArrayList<>();
- repoSetting.name = "Notebook Path";
- repoSetting.type = NotebookRepoSettingsInfo.Type.INPUT;
- repoSetting.value = Collections.emptyList();
- repoSetting.selected = getNotebookDirPath();
-
- settings.add(repoSetting);
- return settings;
- }
-
- @Override
- public void updateSettings(Map<String, String> settings, AuthenticationInfo subject) {
- if (settings == null || settings.isEmpty()) {
- LOG.error("Cannot update {} with empty settings", this.getClass().getName());
- return;
- }
- String newNotebookDirectotyPath = StringUtils.EMPTY;
- if (settings.containsKey("Notebook Path")) {
- newNotebookDirectotyPath = settings.get("Notebook Path");
- }
-
- if (StringUtils.isBlank(newNotebookDirectotyPath)) {
- LOG.error("Notebook path is invalid");
- return;
- }
- LOG.warn("{} will change notebook dir from {} to {}",
- subject.getUser(), getNotebookDirPath(), newNotebookDirectotyPath);
- try {
- setNotebookDirectory(newNotebookDirectotyPath);
- } catch (IOException e) {
- LOG.error("Cannot update notebook directory", e);
- }
- }
-
-}
-
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/UpgradeNoteFileTool.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/UpgradeNoteFileTool.java
deleted file mode 100644
index c39b116..0000000
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/UpgradeNoteFileTool.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.zeppelin.notebook.repo;
-
-import org.apache.commons.cli.*;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
-
-import java.io.IOException;
-
-public class UpgradeNoteFileTool {
-
- public static void main(String[] args) throws IOException {
- Options options = new Options();
- Option input = new Option("d", "deleteOld", false, "Whether delete old note file");
- options.addOption(input);
- CommandLineParser parser = new DefaultParser();
- CommandLine cmd = null;
- try {
- cmd = parser.parse(options, args);
- } catch (ParseException e) {
- System.out.println(e);
- System.exit(1);
- }
-
- ZeppelinConfiguration conf = ZeppelinConfiguration.create();
- NotebookRepoSync notebookRepoSync = new NotebookRepoSync(conf);
- notebookRepoSync.convertNoteFiles(conf, cmd.hasOption("d"));
- }
-}
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java
index c24c8fb..7cdc915 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java
@@ -25,9 +25,6 @@ import org.apache.zeppelin.interpreter.launcher.StandardInterpreterLauncher;
import org.apache.zeppelin.interpreter.recovery.RecoveryStorage;
import org.apache.zeppelin.notebook.repo.GitNotebookRepo;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
-import org.apache.zeppelin.notebook.repo.OldGitNotebookRepo;
-import org.apache.zeppelin.notebook.repo.OldNotebookRepo;
-import org.apache.zeppelin.notebook.repo.OldVFSNotebookRepo;
import org.apache.zeppelin.notebook.repo.VFSNotebookRepo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,9 +60,6 @@ public class PluginManager {
private List<String> builtinNotebookRepoClassNames = Arrays.asList(
VFSNotebookRepo.class.getName(),
GitNotebookRepo.class.getName());
- private List<String> builtinOldNotebookRepoClassNames = Arrays.asList(
- OldVFSNotebookRepo.class.getName(),
- OldGitNotebookRepo.class.getName());
public static synchronized PluginManager get() {
if (instance == null) {
@@ -107,42 +101,6 @@ public class PluginManager {
return notebookRepoClassName.substring(0, pos) + ".Old" + notebookRepoClassName.substring(pos + 1);
}
- /**
- * This is a temporary class which is used for loading old implementation of NotebookRepo.
- *
- * @param notebookRepoClassName
- * @return
- * @throws IOException
- */
- public OldNotebookRepo loadOldNotebookRepo(String notebookRepoClassName) throws IOException {
- String oldNotebookRepoClassName = getOldNotebookRepoClassName(notebookRepoClassName);
- LOGGER.info("Loading OldNotebookRepo Plugin: {}", oldNotebookRepoClassName);
- if (builtinOldNotebookRepoClassNames.contains(oldNotebookRepoClassName) ||
- Boolean.parseBoolean(System.getProperty("zeppelin.isTest", "false"))) {
- try {
- return (OldNotebookRepo) (Class.forName(oldNotebookRepoClassName).newInstance());
- } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
- throw new IOException("Fail to instantiate notebookrepo from classpath directly:"
- + oldNotebookRepoClassName);
- }
- }
-
- String simpleClassName = notebookRepoClassName.substring(notebookRepoClassName.lastIndexOf(".") + 1);
- URLClassLoader pluginClassLoader = getPluginClassLoader(pluginsDir, "NotebookRepo", simpleClassName);
- if (pluginClassLoader == null) {
- return null;
- }
- OldNotebookRepo notebookRepo = null;
- try {
- notebookRepo = (OldNotebookRepo) (Class.forName(oldNotebookRepoClassName, true, pluginClassLoader)).newInstance();
- } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
- throw new IOException("Fail to instantiate notebookrepo " + oldNotebookRepoClassName +
- " from plugin classpath:" + pluginsDir, e);
- }
-
- return notebookRepo;
- }
-
public synchronized InterpreterLauncher loadInterpreterLauncher(String launcherPlugin,
RecoveryStorage recoveryStorage)
throws IOException {
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/plugin/PluginManagerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/plugin/PluginManagerTest.java
index bd900f2..d2d54a9 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/plugin/PluginManagerTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/plugin/PluginManagerTest.java
@@ -19,8 +19,6 @@ package org.apache.zeppelin.plugin;
import org.apache.zeppelin.notebook.repo.GitNotebookRepo;
import org.apache.zeppelin.notebook.repo.NotebookRepo;
-import org.apache.zeppelin.notebook.repo.OldGitNotebookRepo;
-import org.apache.zeppelin.notebook.repo.OldNotebookRepo;
import org.junit.Test;
import java.io.IOException;
@@ -35,9 +33,5 @@ public class PluginManagerTest {
NotebookRepo notebookRepo = PluginManager.get()
.loadNotebookRepo("org.apache.zeppelin.notebook.repo.GitNotebookRepo");
assertTrue(notebookRepo instanceof GitNotebookRepo);
-
- OldNotebookRepo oldNotebookRepo = PluginManager.get()
- .loadOldNotebookRepo("org.apache.zeppelin.notebook.repo.GitNotebookRepo");
- assertTrue(oldNotebookRepo instanceof OldGitNotebookRepo);
}
}