You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2020/10/10 05:52:41 UTC
[zeppelin] branch branch-0.9 updated: [ZEPPELIN-4910]. Don‘t convert relative path to local absolute path when using hadoop filesystem
This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/branch-0.9 by this push:
new 7a55839 [ZEPPELIN-4910]. Don‘t convert relative path to local absolute path when using hadoop filesystem
7a55839 is described below
commit 7a55839529c64007b4c0494ae5986e1d77dc83e3
Author: Jeff Zhang <zj...@apache.org>
AuthorDate: Thu Jun 25 00:27:11 2020 +0800
[ZEPPELIN-4910]. Don‘t convert relative path to local absolute path when using hadoop filesystem
### What is this PR for?
`ZeppelinConfiguration#getRelativeDir` actually get the absolute path intead of relative path, this PR just rename this method.
### What type of PR is it?
[Improvement ]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-4910
### How should this be tested?
* CI pass
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #3833 from zjffdu/ZEPPELIN-4910 and squashes the following commits:
c3962b499 [Jeff Zhang] save
ee86b2ca7 [Jeff Zhang] [ZEPPELIN-4910]. Don convert relative path to local absolute path when using hadoop filesystem
(cherry picked from commit 6a3e84bd1fc93763aafc0fdc527c325cd2c2b322)
Signed-off-by: Jeff Zhang <zj...@apache.org>
---
.../org/apache/zeppelin/jdbc/JDBCInterpreter.java | 1 +
.../zeppelin/integration/AuthenticationIT.java | 2 +-
.../integration/InterpreterModeActionsIT.java | 5 +-
.../zeppelin/integration/PersonalizeActionsIT.java | 5 +-
.../zeppelin/conf/ZeppelinConfiguration.java | 58 ++++++++++++----------
.../notebook/repo/FileSystemNotebookRepo.java | 2 +-
.../notebook/repo/OldFileSystemNotebookRepo.java | 3 +-
.../org/apache/zeppelin/server/ZeppelinServer.java | 2 +-
.../java/org/apache/zeppelin/helium/Helium.java | 3 +-
.../zeppelin/helium/HeliumBundleFactory.java | 17 +++----
.../zeppelin/interpreter/InterpreterSetting.java | 2 +-
.../interpreter/InterpreterSettingManager.java | 2 +-
.../interpreter/RemoteInterpreterEventServer.java | 15 ++++--
.../recovery/FileSystemRecoveryStorage.java | 5 +-
.../zeppelin/notebook/repo/OldVFSNotebookRepo.java | 3 +-
.../zeppelin/notebook/repo/VFSNotebookRepo.java | 3 +-
.../zeppelin/storage/FileSystemConfigStorage.java | 11 ++--
.../zeppelin/storage/LocalConfigStorage.java | 6 +--
.../zeppelin/conf/ZeppelinConfigurationTest.java | 4 +-
.../zeppelin/helium/HeliumBundleFactoryTest.java | 3 +-
20 files changed, 80 insertions(+), 72 deletions(-)
diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
index d147d21..40fcd1d 100644
--- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
+++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
@@ -427,6 +427,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
boolean testWhileIdle = "true".equalsIgnoreCase(properties.getProperty("testWhileIdle"));
long timeBetweenEvictionRunsMillis = PropertiesUtil.getLong(
properties, "timeBetweenEvictionRunsMillis", -1L);
+
long maxWaitMillis = PropertiesUtil.getLong(properties, "maxWaitMillis", -1L);
int maxIdle = PropertiesUtil.getInt(properties, "maxIdle", 8);
int minIdle = PropertiesUtil.getInt(properties, "minIdle", 0);
diff --git a/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java b/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java
index 2e34fd4..7032c05 100644
--- a/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java
+++ b/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/AuthenticationIT.java
@@ -82,7 +82,7 @@ public class AuthenticationIT extends AbstractZeppelinIT {
try {
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), new File("../").getAbsolutePath());
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
- shiroPath = conf.getRelativeDir(String.format("%s/shiro.ini", conf.getConfDir()));
+ shiroPath = conf.getAbsoluteDir(String.format("%s/shiro.ini", conf.getConfDir()));
File file = new File(shiroPath);
if (file.exists()) {
originalShiro = StringUtils.join(FileUtils.readLines(file, "UTF-8"), "\n");
diff --git a/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/InterpreterModeActionsIT.java b/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/InterpreterModeActionsIT.java
index c1f5ab2..200a4b2 100644
--- a/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/InterpreterModeActionsIT.java
+++ b/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/InterpreterModeActionsIT.java
@@ -43,7 +43,6 @@ import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
-import java.util.List;
import static org.junit.Assert.assertTrue;
@@ -82,8 +81,8 @@ public class InterpreterModeActionsIT extends AbstractZeppelinIT {
try {
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), new File("../").getAbsolutePath());
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
- shiroPath = conf.getRelativeDir(String.format("%s/shiro.ini", conf.getConfDir()));
- interpreterOptionPath = conf.getRelativeDir(String.format("%s/interpreter.json", conf.getConfDir()));
+ shiroPath = conf.getAbsoluteDir(String.format("%s/shiro.ini", conf.getConfDir()));
+ interpreterOptionPath = conf.getAbsoluteDir(String.format("%s/interpreter.json", conf.getConfDir()));
File shiroFile = new File(shiroPath);
if (shiroFile.exists()) {
originalShiro = StringUtils.join(FileUtils.readLines(shiroFile, "UTF-8"), "\n");
diff --git a/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/PersonalizeActionsIT.java b/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/PersonalizeActionsIT.java
index 8a7138c..2f48c27 100644
--- a/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/PersonalizeActionsIT.java
+++ b/zeppelin-integration/src/test/java/org/apache/zeppelin/integration/PersonalizeActionsIT.java
@@ -18,7 +18,6 @@ package org.apache.zeppelin.integration;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.AbstractZeppelinIT;
-import org.apache.zeppelin.integration.AuthenticationIT;
import org.apache.zeppelin.WebDriverManager;
import org.apache.zeppelin.ZeppelinITUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
@@ -40,8 +39,6 @@ import org.slf4j.LoggerFactory;
import org.apache.commons.io.FileUtils;
import java.io.File;
import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
import static org.junit.Assert.assertTrue;
@@ -74,7 +71,7 @@ public class PersonalizeActionsIT extends AbstractZeppelinIT {
try {
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), new File("../").getAbsolutePath());
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
- shiroPath = conf.getRelativeDir(String.format("%s/shiro.ini", conf.getConfDir()));
+ shiroPath = conf.getAbsoluteDir(String.format("%s/shiro.ini", conf.getConfDir()));
File file = new File(shiroPath);
if (file.exists()) {
originalShiro = StringUtils.join(FileUtils.readLines(file, "UTF-8"), "\n");
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index b1443c3..d4b92de 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -355,7 +355,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
if (path != null && path.startsWith("/") || isWindowsPath(path)) {
return path;
} else {
- return getRelativeDir(
+ return getAbsoluteDir(
String.format("%s/%s",
getConfDir(),
path));
@@ -387,7 +387,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
if (path != null && path.startsWith("/") || isWindowsPath(path)) {
return path;
} else {
- return getRelativeDir(
+ return getAbsoluteDir(
String.format("%s/%s",
getConfDir(),
path));
@@ -429,7 +429,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public String getNotebookDir() {
- return getRelativeDir(ConfVars.ZEPPELIN_NOTEBOOK_DIR);
+ return getAbsoluteDir(ConfVars.ZEPPELIN_NOTEBOOK_DIR);
}
public String getNotebookRunId() {
@@ -449,11 +449,11 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public String getPluginsDir() {
- return getRelativeDir(getString(ConfVars.ZEPPELIN_PLUGINS_DIR));
+ return getAbsoluteDir(getString(ConfVars.ZEPPELIN_PLUGINS_DIR));
}
public String getRecoveryDir() {
- return getRelativeDir(ConfVars.ZEPPELIN_RECOVERY_DIR);
+ return getAbsoluteDir(ConfVars.ZEPPELIN_RECOVERY_DIR);
}
public String getNotebookStorageClass() {
@@ -549,27 +549,27 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public String getInterpreterListPath() {
- return getRelativeDir(String.format("%s/interpreter-list", getConfDir()));
+ return getAbsoluteDir(String.format("%s/interpreter-list", getConfDir()));
}
public String getInterpreterDir() {
- return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_DIR);
+ return getAbsoluteDir(ConfVars.ZEPPELIN_INTERPRETER_DIR);
}
public String getInterpreterJson() {
return getString(ConfVars.ZEPPELIN_INTERPRETER_JSON);
}
- public String getInterpreterSettingPath() {
- return getConfigFSDir() + "/interpreter.json";
+ public String getInterpreterSettingPath(boolean absolute) {
+ return getConfigFSDir(absolute) + "/interpreter.json";
}
public String getHeliumConfPath() {
- return getRelativeDir(String.format("%s/helium.json", getConfDir()));
+ return getAbsoluteDir(String.format("%s/helium.json", getConfDir()));
}
public String getHeliumRegistry() {
- return getRelativeDir(ConfVars.ZEPPELIN_HELIUM_REGISTRY);
+ return getAbsoluteDir(ConfVars.ZEPPELIN_HELIUM_REGISTRY);
}
public String getHeliumNodeInstallerUrl() {
@@ -584,8 +584,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL);
}
- public String getNotebookAuthorizationPath() {
- return getConfigFSDir() + "/notebook-authorization.json";
+ public String getNotebookAuthorizationPath(boolean absolute) {
+ return getConfigFSDir(absolute) + "/notebook-authorization.json";
}
public Boolean credentialsPersist() {
@@ -596,32 +596,32 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_CREDENTIALS_ENCRYPT_KEY);
}
- public String getCredentialsPath() {
- return getConfigFSDir() + "/credentials.json";
+ public String getCredentialsPath(boolean absolute) {
+ return getConfigFSDir(absolute) + "/credentials.json";
}
public String getShiroPath() {
- String shiroPath = getRelativeDir(String.format("%s/shiro.ini", getConfDir()));
+ String shiroPath = getAbsoluteDir(String.format("%s/shiro.ini", getConfDir()));
return new File(shiroPath).exists() ? shiroPath : StringUtils.EMPTY;
}
public String getInterpreterRemoteRunnerPath() {
- return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER);
+ return getAbsoluteDir(ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER);
}
public String getInterpreterLocalRepoPath() {
- return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO);
+ return getAbsoluteDir(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO);
}
public String getInterpreterMvnRepoPath() {
return getString(ConfVars.ZEPPELIN_INTERPRETER_DEP_MVNREPO);
}
- public String getRelativeDir(ConfVars c) {
- return getRelativeDir(getString(c));
+ public String getAbsoluteDir(ConfVars c) {
+ return getAbsoluteDir(getString(c));
}
- public String getRelativeDir(String path) {
+ public String getAbsoluteDir(String path) {
if (path != null && (path.startsWith(File.separator) || isWindowsPath(path) || isPathWithScheme(path))) {
return path;
} else {
@@ -669,20 +669,24 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public String getConfDir() {
- return getRelativeDir(ConfVars.ZEPPELIN_CONF_DIR);
+ return getAbsoluteDir(ConfVars.ZEPPELIN_CONF_DIR);
}
- public String getConfigFSDir() {
+ public String getConfigFSDir(boolean absolute) {
String fsConfigDir = getString(ConfVars.ZEPPELIN_CONFIG_FS_DIR);
if (StringUtils.isBlank(fsConfigDir)) {
LOG.warn(ConfVars.ZEPPELIN_CONFIG_FS_DIR.varName + " is not specified, fall back to local " +
"conf directory " + ConfVars.ZEPPELIN_CONF_DIR.varName);
- return getConfDir();
+ if (absolute) {
+ return getConfDir();
+ } else {
+ return getString(ConfVars.ZEPPELIN_CONF_DIR);
+ }
}
if (getString(ConfVars.ZEPPELIN_CONFIG_STORAGE_CLASS)
.equals("org.apache.zeppelin.storage.LocalConfigStorage")) {
// only apply getRelativeDir when it is LocalConfigStorage
- return getRelativeDir(fsConfigDir);
+ return getAbsoluteDir(fsConfigDir);
} else {
return fsConfigDir;
}
@@ -799,7 +803,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public String getZeppelinSearchIndexPath() {
- return getRelativeDir(ConfVars.ZEPPELIN_SEARCH_INDEX_PATH);
+ return getAbsoluteDir(ConfVars.ZEPPELIN_SEARCH_INDEX_PATH);
}
public Boolean isOnlyYarnCluster() {
@@ -861,7 +865,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public String getK8sTemplatesDir() {
- return getRelativeDir(ConfVars.ZEPPELIN_K8S_TEMPLATE_DIR);
+ return getAbsoluteDir(ConfVars.ZEPPELIN_K8S_TEMPLATE_DIR);
}
public String getK8sServiceName() {
diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java
index d2c5cdd..e74f286 100644
--- a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java
+++ b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java
@@ -46,7 +46,7 @@ public class FileSystemNotebookRepo implements NotebookRepo {
}
public void init(ZeppelinConfiguration zConf) throws IOException {
- this.fs = new FileSystemStorage(zConf, zConf.getNotebookDir());
+ this.fs = new FileSystemStorage(zConf, zConf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR));
LOGGER.info("Creating FileSystem: " + this.fs.getFs().getClass().getName());
this.notebookDir = this.fs.makeQualified(new Path(zConf.getNotebookDir()));
LOGGER.info("Using folder {} to store notebook", notebookDir);
diff --git a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/OldFileSystemNotebookRepo.java b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/OldFileSystemNotebookRepo.java
index 8b9605e..86f3728 100644
--- a/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/OldFileSystemNotebookRepo.java
+++ b/zeppelin-plugins/notebookrepo/filesystem/src/main/java/org/apache/zeppelin/notebook/repo/OldFileSystemNotebookRepo.java
@@ -35,7 +35,8 @@ public class OldFileSystemNotebookRepo implements OldNotebookRepo {
}
public void init(ZeppelinConfiguration zConf) throws IOException {
- this.fs = new FileSystemStorage(zConf, zConf.getNotebookDir());
+ this.fs = new FileSystemStorage(zConf,
+ zConf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR));
LOGGER.info("Creating FileSystem: " + this.fs.getFs().getClass().getName() +
" for Zeppelin Notebook.");
this.notebookDir = this.fs.makeQualified(new Path(zConf.getNotebookDir()));
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
index 146fec0..446c4d6 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -549,7 +549,7 @@ public class ZeppelinServer extends ResourceConfig {
// use packaged WAR
webApp.setWar(warFile.getAbsolutePath());
webApp.setExtractWAR(false);
- File warTempDirectory = new File(conf.getRelativeDir(ConfVars.ZEPPELIN_WAR_TEMPDIR) + contextPath);
+ File warTempDirectory = new File(conf.getAbsoluteDir(ConfVars.ZEPPELIN_WAR_TEMPDIR) + contextPath);
warTempDirectory.mkdir();
LOG.info("ZeppelinServer Webapp path: {}", warTempDirectory.getPath());
webApp.setTempDirectory(warTempDirectory);
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
index d2a704d..ab02222 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java
@@ -39,7 +39,6 @@ import org.apache.zeppelin.interpreter.InterpreterNotFoundException;
import org.apache.zeppelin.interpreter.InterpreterSettingManager;
import org.apache.zeppelin.interpreter.ManagedInterpreterGroup;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
-import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.resource.DistributedResourcePool;
import org.apache.zeppelin.resource.Resource;
@@ -75,7 +74,7 @@ public class Helium {
this(
conf.getHeliumConfPath(),
conf.getHeliumRegistry(),
- new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO), "helium-registry-cache"),
+ new File(conf.getAbsoluteDir(ConfVars.ZEPPELIN_DEP_LOCALREPO), "helium-registry-cache"),
heliumBundleFactory,
heliumApplicationFactory,
interpreterSettingManager);
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
index 3beba56..4009c37 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumBundleFactory.java
@@ -42,7 +42,6 @@ import java.io.IOException;
import java.io.OutputStream;
import java.io.StringReader;
import java.net.URI;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashMap;
@@ -109,7 +108,7 @@ public class HeliumBundleFactory {
@Inject
public HeliumBundleFactory(ZeppelinConfiguration conf) {
this.heliumLocalRepoDirectory =
- new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO), HELIUM_LOCAL_REPO).getAbsoluteFile();
+ new File(conf.getAbsoluteDir(ConfVars.ZEPPELIN_DEP_LOCALREPO), HELIUM_LOCAL_REPO);
this.heliumBundleDirectory = new File(heliumLocalRepoDirectory, HELIUM_BUNDLES_DIR);
this.heliumLocalModuleDirectory = new File(heliumLocalRepoDirectory, HELIUM_LOCAL_MODULE_DIR);
this.yarnCacheDir = new File(heliumLocalRepoDirectory, YARN_CACHE_DIR);
@@ -123,17 +122,17 @@ public class HeliumBundleFactory {
this.gson = new Gson();
- File zeppelinWebPath = new File(conf.getRelativeDir("zeppelin-web"));
+ File zeppelinWebPath = new File(conf.getAbsoluteDir("zeppelin-web"));
if (!zeppelinWebPath.isDirectory()) {
this.tabledataModulePath =
- new File(conf.getRelativeDir("lib/node_modules/zeppelin-tabledata"));
- this.visualizationModulePath = new File(conf.getRelativeDir("lib/node_modules/zeppelin-vis"));
- this.spellModulePath = new File(conf.getRelativeDir("lib/node_modules/zeppelin-spell"));
+ new File(conf.getAbsoluteDir("lib/node_modules/zeppelin-tabledata"));
+ this.visualizationModulePath = new File(conf.getAbsoluteDir("lib/node_modules/zeppelin-vis"));
+ this.spellModulePath = new File(conf.getAbsoluteDir("lib/node_modules/zeppelin-spell"));
} else {
- this.tabledataModulePath = new File(conf.getRelativeDir("zeppelin-web/src/app/tabledata"));
+ this.tabledataModulePath = new File(conf.getAbsoluteDir("zeppelin-web/src/app/tabledata"));
this.visualizationModulePath =
- new File(conf.getRelativeDir("zeppelin-web/src/app/visualization"));
- this.spellModulePath = new File(conf.getRelativeDir("zeppelin-web/src/app/spell"));
+ new File(conf.getAbsoluteDir("zeppelin-web/src/app/visualization"));
+ this.spellModulePath = new File(conf.getAbsoluteDir("zeppelin-web/src/app/spell"));
}
}
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
index de829c6..989fb80 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
@@ -975,7 +975,7 @@ public class InterpreterSetting {
LOGGER.info("Start to download dependencies for interpreter: {}", name);
for (Dependency d : deps) {
File destDir = new File(
- conf.getRelativeDir(ZeppelinConfiguration.ConfVars.ZEPPELIN_DEP_LOCALREPO));
+ conf.getAbsoluteDir(ZeppelinConfiguration.ConfVars.ZEPPELIN_DEP_LOCALREPO));
if (d.getExclusions() != null) {
dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(),
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
index 377b525..2affa5e 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
@@ -745,7 +745,7 @@ public class InterpreterSettingManager implements NoteEventListener, ClusterEven
LOGGER.info("Start to copy dependencies for interpreter: {}", setting.getName());
for (Dependency d : deps) {
File destDir = new File(
- conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
+ conf.getAbsoluteDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
int numSplits = d.getGroupArtifactVersion().split(":").length;
if (!(numSplits >= 3 && numSplits <= 6)) {
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/RemoteInterpreterEventServer.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/RemoteInterpreterEventServer.java
index 8b8fc1f..aacccc8 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/RemoteInterpreterEventServer.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/RemoteInterpreterEventServer.java
@@ -285,12 +285,16 @@ public class RemoteInterpreterEventServer implements RemoteInterpreterEventServi
}
interpreterGroup.getAngularObjectRegistry().add(angularObject.getName(),
angularObject.get(), angularObject.getNoteId(), angularObject.getParagraphId());
+
if (angularObject.getNoteId() != null) {
try {
Note note = interpreterSettingManager.getNotebook().getNote(angularObject.getNoteId());
- note.addOrUpdateAngularObject(intpGroupId, angularObject);
+ if (note != null) {
+ note.addOrUpdateAngularObject(intpGroupId, angularObject);
+ interpreterSettingManager.getNotebook().saveNote(note, AuthenticationInfo.ANONYMOUS);
+ }
} catch (IOException e) {
- LOGGER.warn("Fail to get note: " + angularObject.getNoteId(), e);
+ LOGGER.error("Fail to get note: {}", angularObject.getNoteId());
}
}
}
@@ -316,9 +320,12 @@ public class RemoteInterpreterEventServer implements RemoteInterpreterEventServi
if (angularObject.getNoteId() != null) {
try {
Note note = interpreterSettingManager.getNotebook().getNote(angularObject.getNoteId());
- note.addOrUpdateAngularObject(intpGroupId, angularObject);
+ if (note != null) {
+ note.addOrUpdateAngularObject(intpGroupId, angularObject);
+ interpreterSettingManager.getNotebook().saveNote(note, AuthenticationInfo.ANONYMOUS);
+ }
} catch (IOException e) {
- LOGGER.warn("Fail to get note: " + angularObject.getNoteId(), e);
+ LOGGER.error("Fail to get note: {}", angularObject.getNoteId());
}
}
}
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/recovery/FileSystemRecoveryStorage.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/recovery/FileSystemRecoveryStorage.java
index d43d9f9..26cecb2 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/recovery/FileSystemRecoveryStorage.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/recovery/FileSystemRecoveryStorage.java
@@ -53,10 +53,11 @@ public class FileSystemRecoveryStorage extends RecoveryStorage {
throws IOException {
super(zConf);
this.interpreterSettingManager = interpreterSettingManager;
- this.fs = new FileSystemStorage(zConf, zConf.getRecoveryDir());
+ String recoveryDirProperty = zConf.getString(ZeppelinConfiguration.ConfVars.ZEPPELIN_RECOVERY_DIR);
+ this.fs = new FileSystemStorage(zConf, recoveryDirProperty);
LOGGER.info("Creating FileSystem: " + this.fs.getFs().getClass().getName() +
" for Zeppelin Recovery.");
- this.recoveryDir = this.fs.makeQualified(new Path(zConf.getRecoveryDir()));
+ this.recoveryDir = this.fs.makeQualified(new Path(recoveryDirProperty));
LOGGER.info("Using folder {} to store recovery data", recoveryDir);
this.fs.tryMkDir(recoveryDir);
}
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldVFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldVFSNotebookRepo.java
index 7a1436a..9c4e1f0 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldVFSNotebookRepo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/OldVFSNotebookRepo.java
@@ -29,7 +29,6 @@ import org.apache.commons.vfs2.VFS;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.notebook.Note;
-import org.apache.zeppelin.notebook.NoteInfo;
import org.apache.zeppelin.notebook.OldNoteInfo;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.slf4j.Logger;
@@ -76,7 +75,7 @@ public class OldVFSNotebookRepo implements OldNotebookRepo {
}
if (filesystemRoot.getScheme() == null) { // it is local path
- File f = new File(conf.getRelativeDir(filesystemRoot.getPath()));
+ File f = new File(conf.getAbsoluteDir(filesystemRoot.getPath()));
this.filesystemRoot = f.toURI();
}
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
index 2d2fbc2..018d192 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java
@@ -38,7 +38,6 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.NoteInfo;
-import org.apache.zeppelin.notebook.NoteAuth;
import org.apache.zeppelin.user.AuthenticationInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -78,7 +77,7 @@ public class VFSNotebookRepo implements NotebookRepo {
}
if (filesystemRoot.getScheme() == null) { // it is local path
- File f = new File(conf.getRelativeDir(filesystemRoot.getPath()));
+ File f = new File(conf.getAbsoluteDir(filesystemRoot.getPath()));
filesystemRoot = f.toURI();
}
this.fsManager = VFS.getManager();
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/FileSystemConfigStorage.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/FileSystemConfigStorage.java
index 03883d1..005923e 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/FileSystemConfigStorage.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/FileSystemConfigStorage.java
@@ -47,14 +47,15 @@ public class FileSystemConfigStorage extends ConfigStorage {
public FileSystemConfigStorage(ZeppelinConfiguration zConf) throws IOException {
super(zConf);
- this.fs = new FileSystemStorage(zConf, zConf.getConfigFSDir());
+ String configDir = zConf.getConfigFSDir(false);
+ this.fs = new FileSystemStorage(zConf, configDir);
LOGGER.info("Creating FileSystem: {} for Zeppelin Config", this.fs.getFs().getClass().getName());
- Path configPath = this.fs.makeQualified(new Path(zConf.getConfigFSDir()));
+ Path configPath = this.fs.makeQualified(new Path(configDir));
this.fs.tryMkDir(configPath);
LOGGER.info("Using folder {} to store Zeppelin Config", configPath);
- this.interpreterSettingPath = fs.makeQualified(new Path(zConf.getInterpreterSettingPath()));
- this.authorizationPath = fs.makeQualified(new Path(zConf.getNotebookAuthorizationPath()));
- this.credentialPath = fs.makeQualified(new Path(zConf.getCredentialsPath()));
+ this.interpreterSettingPath = fs.makeQualified(new Path(zConf.getInterpreterSettingPath(false)));
+ this.authorizationPath = fs.makeQualified(new Path(zConf.getNotebookAuthorizationPath(false)));
+ this.credentialPath = fs.makeQualified(new Path(zConf.getCredentialsPath(false)));
}
@Override
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/LocalConfigStorage.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/LocalConfigStorage.java
index d7c3f35..97be0da 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/LocalConfigStorage.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/storage/LocalConfigStorage.java
@@ -43,9 +43,9 @@ public class LocalConfigStorage extends ConfigStorage {
public LocalConfigStorage(ZeppelinConfiguration zConf) {
super(zConf);
- this.interpreterSettingPath = new File(zConf.getInterpreterSettingPath());
- this.authorizationPath = new File(zConf.getNotebookAuthorizationPath());
- this.credentialPath = new File(zConf.getCredentialsPath());
+ this.interpreterSettingPath = new File(zConf.getInterpreterSettingPath(true));
+ this.authorizationPath = new File(zConf.getNotebookAuthorizationPath(true));
+ this.credentialPath = new File(zConf.getCredentialsPath(true));
}
@Override
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/conf/ZeppelinConfigurationTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/conf/ZeppelinConfigurationTest.java
index 08e8255..2ce9ada 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/conf/ZeppelinConfigurationTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/conf/ZeppelinConfigurationTest.java
@@ -131,9 +131,9 @@ public class ZeppelinConfigurationTest {
System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), "/usr/lib/zeppelin");
System.setProperty(ConfVars.ZEPPELIN_CONFIG_FS_DIR.getVarName(), "conf");
ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-test-site.xml"));
- assertEquals("/usr/lib/zeppelin/conf", conf.getConfigFSDir());
+ assertEquals("/usr/lib/zeppelin/conf", conf.getConfigFSDir(true));
System.setProperty(ConfVars.ZEPPELIN_CONFIG_STORAGE_CLASS.getVarName(), "org.apache.zeppelin.storage.FileSystemConfigStorage");
- assertEquals("conf", conf.getConfigFSDir());
+ assertEquals("conf", conf.getConfigFSDir(false));
}
}
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
index d47fead..5e8d435 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumBundleFactoryTest.java
@@ -48,7 +48,8 @@ public class HeliumBundleFactoryTest {
ZeppelinConfiguration conf = ZeppelinConfiguration.create();
nodeInstallationDir =
- new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO), HELIUM_LOCAL_REPO).getAbsoluteFile();
+ new File(conf.getAbsoluteDir(ConfVars.ZEPPELIN_DEP_LOCALREPO), HELIUM_LOCAL_REPO);
+
hbf = new HeliumBundleFactory(conf);
hbf.installNodeAndNpm();
hbf.copyFrameworkModulesToInstallPath(true);