You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by pd...@apache.org on 2021/01/25 10:04:13 UTC
[zeppelin] branch branch-0.9 updated: [ZEPPELIN-5210] Upgrade
Configuration library to version 2
This is an automated email from the ASF dual-hosted git repository.
pdallig pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/branch-0.9 by this push:
new 0c2810c [ZEPPELIN-5210] Upgrade Configuration library to version 2
0c2810c is described below
commit 0c2810c247452912591fc0882caf8092e3d579df
Author: Philipp Dallig <ph...@gmail.com>
AuthorDate: Tue Jan 19 09:03:19 2021 +0100
[ZEPPELIN-5210] Upgrade Configuration library to version 2
### What is this PR for?
This PR migrates the configuration library to version 2
### What type of PR is it?
- Improvement
### Todos
* [x] - Add CI for for parseExceptions
* [x] - Update license file
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-5210
### How should this be tested?
* via CI
### Questions:
* Does the licenses files need update? Yes
* Is there breaking changes for older versions? No
* Does this needs documentation? Yes
Author: Philipp Dallig <ph...@gmail.com>
Closes #4028 from Reamer/configuration_version_2 and squashes the following commits:
3cc9b89cc [Philipp Dallig] conf cleanup
a733adc82 [Philipp Dallig] Use ConfVars to get config values
4ff798d31 [Philipp Dallig] return primitive type
ac210e0df [Philipp Dallig] cleanup
088652555 [Philipp Dallig] Update configuration lib to version 2
(cherry picked from commit d63289a47a9ed26098ad93cb62ae1660bb937182)
Signed-off-by: Philipp Dallig <ph...@gmail.com>
---
docs/setup/operation/configuration.md | 10 +-
pom.xml | 27 +-
zeppelin-distribution/src/bin_license/LICENSE | 6 +-
zeppelin-interpreter/pom.xml | 18 +-
.../zeppelin/conf/ZeppelinConfiguration.java | 387 ++++++++-------------
.../zeppelin/conf/ZeppelinLocationStrategy.java | 62 ++++
.../zeppelin/cluster/ClusterSingleNodeTest.java | 7 +-
.../org/apache/zeppelin/server/ZeppelinServer.java | 1 -
.../zeppelin/cluster/ZeppelinServerMock.java | 26 --
.../service/ShiroAuthenticationServiceTest.java | 10 +-
.../org/apache/zeppelin/utils/CorsUtilsTest.java | 35 +-
zeppelin-zengine/pom.xml | 14 -
.../zeppelin/interpreter/YarnAppMonitor.java | 5 +-
.../remote/RemoteInterpreterProcess.java | 2 +-
.../repo/zeppelinhub/security/Authentication.java | 7 +-
.../java/org/apache/zeppelin/user/Credentials.java | 2 +-
.../zeppelin/conf/ZeppelinConfigurationTest.java | 86 +++--
17 files changed, 332 insertions(+), 373 deletions(-)
diff --git a/docs/setup/operation/configuration.md b/docs/setup/operation/configuration.md
index c745531..32d5015 100644
--- a/docs/setup/operation/configuration.md
+++ b/docs/setup/operation/configuration.md
@@ -24,12 +24,14 @@ limitations under the License.
<div id="toc"></div>
## Zeppelin Properties
-There are two locations you can configure Apache Zeppelin.
-* **Environment variables** can be defined `conf/zeppelin-env.sh`(`conf\zeppelin-env.cmd` for Windows).
-* **Java properties** can be defined in `conf/zeppelin-site.xml`.
+Zeppelin can be configured via several sources.
+
+Sources descending by priority:
+ - environment variables can be defined `conf/zeppelin-env.sh`(`conf\zeppelin-env.cmd` for Windows).
+ - system properties
+ - configuration file can be defined in `conf/zeppelin-site.xml`
-If both are defined, then the **environment variables** will take priority.
> Mouse hover on each property and click <i class="fa fa-link fa-flip-horizontal"></i> then you can get a link for that.
<table class="table-configuration">
diff --git a/pom.xml b/pom.xml
index b94620f..24b85e7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -130,7 +130,8 @@
<httpcomponents.asyncclient.version>4.0.2</httpcomponents.asyncclient.version>
<commons.compress.version>1.20</commons.compress.version>
<commons.lang3.version>3.10</commons.lang3.version>
- <commons.configuration.version>1.9</commons.configuration.version>
+ <commons.text.version>1.8</commons.text.version>
+ <commons.configuration2.version>2.7</commons.configuration2.version>
<commons.exec.version>1.3</commons.exec.version>
<commons.codec.version>1.14</commons.codec.version>
<commons.io.version>2.6</commons.io.version>
@@ -290,6 +291,12 @@
<dependency>
<groupId>org.apache.commons</groupId>
+ <artifactId>commons-text</artifactId>
+ <version>${commons.text.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
<artifactId>commons-exec</artifactId>
<version>${commons.exec.version}</version>
</dependency>
@@ -307,9 +314,17 @@
</dependency>
<dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- <version>${commons.configuration.version}</version>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-configuration2</artifactId>
+ <version>${commons.configuration2.version}</version>
+ </dependency>
+
+ <!-- force the latest commons-lang version
+ hadoop 2.7 has an older version as transitive dependency -->
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.6</version>
</dependency>
<dependency>
@@ -484,6 +499,10 @@
<artifactId>commons-configuration2</artifactId>
</exclusion>
<exclusion>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils-core</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-webapp</artifactId>
</exclusion>
diff --git a/zeppelin-distribution/src/bin_license/LICENSE b/zeppelin-distribution/src/bin_license/LICENSE
index 532c765..878ac0c 100644
--- a/zeppelin-distribution/src/bin_license/LICENSE
+++ b/zeppelin-distribution/src/bin_license/LICENSE
@@ -8,14 +8,14 @@ The following components are provided under Apache License.
(Apache 2.0) Apache Commons Codec (commons-codec:commons-codec:1.5 - http://commons.apache.org/proper/commons-codec/)
(Apache 2.0) Apache Commons Collections (commons-collections:commons-collections:3.2.1 - http://commons.apache.org/proper/commons-configuration/)
(Apache 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.9 - http://commons.apache.org/proper/commons-compress/)
- (Apache 2.0) Apache Commons Configuration (commons-configuration:commons-configuration:1.9 - http://commons.apache.org/configuration/)
+ (Apache 2.0) Apache Commons Configuration (org.apache.commons:commons-configuration2:2.7 - http://commons.apache.org/configuration/)
(Apache 2.0) Apache Commons CLI (commons-cli:commons-cli:1.2 - http://commons.apache.org/cli/)
(Apache 2.0) Apache Commons Exec (commons-exec:commons-exec:1.3 - http://commons.apache.org/exec/)
(Apache 2.0) Http Components (org.apache.httpcomponents:httpcore:4.3.3 - https://github.com/apache/httpclient)
(Apache 2.0) Http Components (org.apache.httpcomponents:httpclient:4.3.6 - https://github.com/apache/httpclient)
(Apache 2.0) Http Components (org.apache.httpcomponents:httpasyncclient:4.0.2 - https://github.com/apache/httpclient)
- (Apache 2.0) Apache Commons Lang (org.apache.commons:commons-lang:2.5 - http://commons.apache.org/proper/commons-lang/)
- (Apache 2.0) Apache Commons Lang 3 (org.apache.commons:commons-lang3:3.4 - http://commons.apache.org/proper/commons-lang/)
+ (Apache 2.0) Apache Commons Lang (org.apache.commons:commons-lang:2.6 - http://commons.apache.org/proper/commons-lang/)
+ (Apache 2.0) Apache Commons Lang 3 (org.apache.commons:commons-lang3:3.10 - http://commons.apache.org/proper/commons-lang/)
(Apache 2.0) Apache Commons Math 3 (org.apache.commons:commons-math3:3.6.1 - http://commons.apache.org/proper/commons-math/)
(Apache 2.0) Apache Commons Net (commons-net:commons-net:2.2 - http://commons.apache.org/proper/commons-net/)
(Apache 2.0) Apache Commons Pool2 (commons-exec:commons-pool2:2.3 - https://commons.apache.org/proper/commons-pool/)
diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml
index 32aa5d2..303787f 100644
--- a/zeppelin-interpreter/pom.xml
+++ b/zeppelin-interpreter/pom.xml
@@ -102,9 +102,22 @@
</dependency>
<dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-configuration2</artifactId>
<exclusions>
+ <!-- using jcl-over-slf4j instead -->
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- Used in conjunction with commons-configuration2 for the Fluent Builder -->
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ <version>1.9.4</version>
+ <exclusions>
<!-- using jcl-over-slf4j instead -->
<exclusion>
<groupId>commons-logging</groupId>
@@ -112,7 +125,6 @@
</exclusion>
</exclusions>
</dependency>
-
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-exec</artifactId>
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index 3eb8e41..c170a6b 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -19,20 +19,30 @@ package org.apache.zeppelin.conf;
import com.google.common.annotations.VisibleForTesting;
import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
-import java.io.Writer;
import java.net.URI;
import java.net.URISyntaxException;
-import java.net.URL;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.XMLConfiguration;
-import org.apache.commons.configuration.tree.ConfigurationNode;
-import org.apache.commons.exec.environment.EnvironmentUtils;
+import java.util.stream.Collectors;
+
+import javax.annotation.Nullable;
+
+import org.apache.commons.configuration2.EnvironmentConfiguration;
+import org.apache.commons.configuration2.SystemConfiguration;
+import org.apache.commons.configuration2.XMLConfiguration;
+import org.apache.commons.configuration2.builder.FileBasedConfigurationBuilder;
+import org.apache.commons.configuration2.builder.fluent.Parameters;
+import org.apache.commons.configuration2.ex.ConfigurationException;
+import org.apache.commons.configuration2.io.ClasspathLocationStrategy;
+import org.apache.commons.configuration2.io.CombinedLocationStrategy;
+import org.apache.commons.configuration2.io.FileLocationStrategy;
+import org.apache.commons.configuration2.tree.ImmutableNode;
import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.interpreter.lifecycle.NullLifecycleManager;
import org.apache.zeppelin.util.Util;
@@ -42,19 +52,24 @@ import org.slf4j.LoggerFactory;
/**
* Zeppelin configuration.
*
+ * Sources descending by priority:
+ * - system properties
+ * - environment variables
+ * - configuration file
*/
-public class ZeppelinConfiguration extends XMLConfiguration {
+public class ZeppelinConfiguration {
+
private static final String ZEPPELIN_SITE_XML = "zeppelin-site.xml";
- private static final long serialVersionUID = 4749305895693848035L;
- private static final Logger LOG = LoggerFactory.getLogger(ZeppelinConfiguration.class);
+ private static final Logger LOGGER = LoggerFactory.getLogger(ZeppelinConfiguration.class);
private Boolean anonymousAllowed;
- private static final String HELIUM_PACKAGE_DEFAULT_URL =
- "https://s3.amazonaws.com/helium-package/helium.json";
private static ZeppelinConfiguration conf;
- private Map<String, String> properties = new HashMap<>();
+ private static final EnvironmentConfiguration envConfig = new EnvironmentConfiguration();
+ private static final SystemConfiguration sysConfig = new SystemConfiguration();
+
+ private final Map<String, String> properties = new HashMap<>();
public enum RUN_MODE {
LOCAL,
@@ -62,143 +77,78 @@ public class ZeppelinConfiguration extends XMLConfiguration {
DOCKER
}
- public ZeppelinConfiguration(URL url) throws ConfigurationException {
- setDelimiterParsingDisabled(true);
- load(url);
- initProperties();
+ // private constructor, so that it is singleton.
+ private ZeppelinConfiguration(@Nullable String filename) {
+ try {
+ loadXMLConfig(filename);
+ } catch (ConfigurationException e) {
+ LOGGER.warn("Failed to load configuration, proceeding with a default", e);
+ }
}
- private void initProperties() {
- List<ConfigurationNode> nodes = getRootNode().getChildren();
- if (nodes == null || nodes.isEmpty()) {
- return;
+ private void loadXMLConfig(@Nullable String filename) throws ConfigurationException {
+ if (StringUtils.isBlank(filename)) {
+ filename = ZEPPELIN_SITE_XML;
}
- for (ConfigurationNode p : nodes) {
- String name = (String) p.getChildren("name").get(0).getValue();
- String value = (String) p.getChildren("value").get(0).getValue();
- if (StringUtils.isNotBlank(name) && StringUtils.isNotBlank(value)) {
- properties.put(name, value);
+ List<FileLocationStrategy> subs = Arrays.asList(
+ new ZeppelinLocationStrategy(),
+ new ClasspathLocationStrategy());
+ FileLocationStrategy strategy = new CombinedLocationStrategy(subs);
+ Parameters params = new Parameters();
+ FileBasedConfigurationBuilder<XMLConfiguration> xmlbuilder =
+ new FileBasedConfigurationBuilder<XMLConfiguration>(XMLConfiguration.class)
+ .configure(params.xml()
+ .setLocationStrategy(strategy)
+ .setFileName(filename)
+ .setBasePath(File.separator + "conf" + File.separator));
+ XMLConfiguration xmlConfig = xmlbuilder.getConfiguration();
+ List<ImmutableNode> nodes = xmlConfig.getNodeModel().getRootNode().getChildren();
+ if (nodes != null && !nodes.isEmpty()) {
+ for (ImmutableNode p : nodes) {
+ String name = String.valueOf(p.getChildren("name").get(0).getValue());
+ String value = String.valueOf(p.getChildren("value").get(0).getValue());
+ if (StringUtils.isNotBlank(name) && StringUtils.isNotBlank(value)) {
+ setProperty(name, value);
+ }
}
}
}
-
- // private constructor, so that it is singleton.
- private ZeppelinConfiguration() {
- ConfVars[] vars = ConfVars.values();
- for (ConfVars v : vars) {
- // set property if env is set, so that the configuration can be passed to
- // interpreter process properly.
- if (StringUtils.isNotBlank(System.getenv(v.name()))) {
- this.setProperty(v.getVarName(), System.getenv(v.name()));
- } else if (v.getType() == ConfVars.VarType.BOOLEAN) {
- this.setProperty(v.getVarName(), v.getBooleanValue());
- } else if (v.getType() == ConfVars.VarType.LONG) {
- this.setProperty(v.getVarName(), v.getLongValue());
- } else if (v.getType() == ConfVars.VarType.INT) {
- this.setProperty(v.getVarName(), v.getIntValue());
- } else if (v.getType() == ConfVars.VarType.FLOAT) {
- this.setProperty(v.getVarName(), v.getFloatValue());
- } else if (v.getType() == ConfVars.VarType.STRING) {
- this.setProperty(v.getVarName(), v.getStringValue());
- } else {
- throw new RuntimeException("Unsupported VarType");
- }
+ public static ZeppelinConfiguration create() {
+ if (conf != null) {
+ return conf;
}
-
+ return ZeppelinConfiguration.create(null);
}
-
-
/**
- * Load from resource.
- *url = ZeppelinConfiguration.class.getResource(ZEPPELIN_SITE_XML);
- * @throws ConfigurationException
+ * Load from via filename.
*/
- public static synchronized ZeppelinConfiguration create() {
+ public static synchronized ZeppelinConfiguration create(@Nullable String filename) {
if (conf != null) {
return conf;
}
- ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
- URL url;
-
- url = ZeppelinConfiguration.class.getResource(ZEPPELIN_SITE_XML);
- if (url == null) {
- ClassLoader cl = ZeppelinConfiguration.class.getClassLoader();
- if (cl != null) {
- url = cl.getResource(ZEPPELIN_SITE_XML);
- }
- }
- if (url == null) {
- url = classLoader.getResource(ZEPPELIN_SITE_XML);
- }
-
- if (url == null) {
- try {
- Map procEnv = EnvironmentUtils.getProcEnvironment();
- if (procEnv.containsKey("ZEPPELIN_HOME")) {
- String zconfDir = (String) procEnv.get("ZEPPELIN_HOME");
- File file = new File(zconfDir + File.separator
- + "conf" + File.separator + ZEPPELIN_SITE_XML);
- if (file.exists()) {
- url = file.toURL();
- }
- }
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- }
- }
-
- if (url == null) {
- try {
- Map procEnv = EnvironmentUtils.getProcEnvironment();
- if (procEnv.containsKey("ZEPPELIN_CONF_DIR")) {
- String zconfDir = (String) procEnv.get("ZEPPELIN_CONF_DIR");
- File file = new File(zconfDir + File.separator + ZEPPELIN_SITE_XML);
- if (file.exists()) {
- url = file.toURL();
- }
- }
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- }
- }
+ conf = new ZeppelinConfiguration(filename);
- if (url == null) {
- LOG.warn("Failed to load configuration, proceeding with a default");
- conf = new ZeppelinConfiguration();
- } else {
- try {
- LOG.info("Load configuration from " + url);
- conf = new ZeppelinConfiguration(url);
- } catch (ConfigurationException e) {
- LOG.warn("Failed to load configuration from " + url + " proceeding with a default", e);
- conf = new ZeppelinConfiguration();
- }
- }
- LOG.info("Server Host: " + conf.getServerAddress());
- if (conf.useSsl() == false) {
- LOG.info("Server Port: " + conf.getServerPort());
+ LOGGER.info("Server Host: {}", conf.getServerAddress());
+ if (conf.useSsl()) {
+ LOGGER.info("Server SSL Port: {}", conf.getServerSslPort());
} else {
- LOG.info("Server SSL Port: " + conf.getServerSslPort());
+ LOGGER.info("Server Port: {}", conf.getServerPort());
}
- LOG.info("Context Path: " + conf.getServerContextPath());
- LOG.info("Zeppelin Version: " + Util.getVersion());
+ LOGGER.info("Context Path: {}", conf.getServerContextPath());
+ LOGGER.info("Zeppelin Version: {}", Util.getVersion());
return conf;
}
- public Map<String, String> getProperties() {
- return this.properties;
- }
-
public static void reset() {
conf = null;
}
public void setProperty(String name, String value) {
- if (StringUtils.isNotBlank(name) && StringUtils.isNotBlank(value)) {
+ if (StringUtils.isNoneBlank(name, value)) {
this.properties.put(name, value);
}
}
@@ -214,7 +164,11 @@ public class ZeppelinConfiguration extends XMLConfiguration {
private int getIntValue(String name, int d) {
String value = this.properties.get(name);
if (value != null) {
- return Integer.parseInt(value);
+ try {
+ return Integer.parseInt(value);
+ } catch (NumberFormatException e) {
+ LOGGER.warn("Can not parse the property {} with the value \"{}\" to an int value", name, value, e);
+ }
}
return d;
}
@@ -222,7 +176,11 @@ public class ZeppelinConfiguration extends XMLConfiguration {
private long getLongValue(String name, long d) {
String value = this.properties.get(name);
if (value != null) {
- return Long.parseLong(value);
+ try {
+ return Long.parseLong(value);
+ } catch (NumberFormatException e) {
+ LOGGER.warn("Can not parse the property {} with the value \"{}\" to a long value", name, value, e);
+ }
}
return d;
}
@@ -230,7 +188,11 @@ public class ZeppelinConfiguration extends XMLConfiguration {
private float getFloatValue(String name, float d) {
String value = this.properties.get(name);
if (value != null) {
- return Float.parseFloat(value);
+ try {
+ return Float.parseFloat(value);
+ } catch (NumberFormatException e) {
+ LOGGER.warn("Can not parse the property {} with the value \"{}\" to a float value", name, value, e);
+ }
}
return d;
}
@@ -248,13 +210,12 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public String getString(String envName, String propertyName, String defaultValue) {
- if (System.getenv(envName) != null) {
- return System.getenv(envName);
+ if (envConfig.containsKey(envName)) {
+ return envConfig.getString(envName);
}
- if (System.getProperty(propertyName) != null) {
- return System.getProperty(propertyName);
+ if (sysConfig.containsKey(propertyName)) {
+ return sysConfig.getString(propertyName);
}
-
return getStringValue(propertyName, defaultValue);
}
@@ -263,12 +224,11 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public int getInt(String envName, String propertyName, int defaultValue) {
- if (System.getenv(envName) != null) {
- return Integer.parseInt(System.getenv(envName));
+ if (envConfig.containsKey(envName)) {
+ return envConfig.getInt(envName);
}
-
- if (System.getProperty(propertyName) != null) {
- return Integer.parseInt(System.getProperty(propertyName));
+ if (sysConfig.containsKey(propertyName)) {
+ return sysConfig.getInt(propertyName);
}
return getIntValue(propertyName, defaultValue);
}
@@ -278,12 +238,11 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public long getLong(String envName, String propertyName, long defaultValue) {
- if (System.getenv(envName) != null) {
- return Long.parseLong(System.getenv(envName));
+ if (envConfig.containsKey(envName)) {
+ return envConfig.getLong(envName);
}
-
- if (System.getProperty(propertyName) != null) {
- return Long.parseLong(System.getProperty(propertyName));
+ if (sysConfig.containsKey(propertyName)) {
+ return sysConfig.getLong(propertyName);
}
return getLongValue(propertyName, defaultValue);
}
@@ -293,11 +252,11 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public float getFloat(String envName, String propertyName, float defaultValue) {
- if (System.getenv(envName) != null) {
- return Float.parseFloat(System.getenv(envName));
+ if (envConfig.containsKey(envName)) {
+ return envConfig.getFloat(envName);
}
- if (System.getProperty(propertyName) != null) {
- return Float.parseFloat(System.getProperty(propertyName));
+ if (sysConfig.containsKey(propertyName)) {
+ return sysConfig.getFloat(propertyName);
}
return getFloatValue(propertyName, defaultValue);
}
@@ -307,12 +266,11 @@ public class ZeppelinConfiguration extends XMLConfiguration {
}
public boolean getBoolean(String envName, String propertyName, boolean defaultValue) {
- if (System.getenv(envName) != null) {
- return Boolean.parseBoolean(System.getenv(envName));
+ if (envConfig.containsKey(envName)) {
+ return envConfig.getBoolean(envName);
}
-
- if (System.getProperty(propertyName) != null) {
- return Boolean.parseBoolean(System.getProperty(propertyName));
+ if (sysConfig.containsKey(propertyName)) {
+ return sysConfig.getBoolean(propertyName);
}
return getBooleanValue(propertyName, defaultValue);
}
@@ -600,7 +558,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getConfigFSDir(absolute) + "/notebook-authorization.json";
}
- public Boolean credentialsPersist() {
+ public boolean credentialsPersist() {
return getBoolean(ConfVars.ZEPPELIN_CREDENTIALS_PERSIST);
}
@@ -691,7 +649,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
public String getConfigFSDir(boolean absolute) {
String fsConfigDir = getString(ConfVars.ZEPPELIN_CONFIG_FS_DIR);
if (StringUtils.isBlank(fsConfigDir)) {
- LOG.warn("{} is not specified, fall back to local conf directory {}",
+ LOGGER.warn("{} is not specified, fall back to local conf directory {}",
ConfVars.ZEPPELIN_CONFIG_FS_DIR.varName, ConfVars.ZEPPELIN_CONF_DIR.varName);
if (absolute) {
return getConfDir();
@@ -711,7 +669,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
public List<String> getAllowedOrigins()
{
if (getString(ConfVars.ZEPPELIN_ALLOWED_ORIGINS).isEmpty()) {
- return Arrays.asList(new String[0]);
+ return Collections.emptyList();
}
return Arrays.asList(getString(ConfVars.ZEPPELIN_ALLOWED_ORIGINS).toLowerCase().split(","));
@@ -733,7 +691,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getInt(ConfVars.ZEPPELIN_SERVER_JETTY_REQUEST_HEADER_SIZE);
}
- public Boolean isAuthorizationHeaderClear() {
+ public boolean isAuthorizationHeaderClear() {
return getBoolean(ConfVars.ZEPPELIN_SERVER_AUTHORIZATION_HEADER_CLEAR);
}
@@ -786,7 +744,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_NOTEBOOK_GIT_REMOTE_ORIGIN);
}
- public Boolean isZeppelinNotebookCronEnable() {
+ public boolean isZeppelinNotebookCronEnable() {
return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_CRON_ENABLE);
}
@@ -810,11 +768,11 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getString(ConfVars.ZEPPELIN_PROXY_PASSWORD);
}
- public Boolean isIndexRebuild() {
+ public boolean isIndexRebuild() {
return getBoolean(ConfVars.ZEPPELIN_SEARCH_INDEX_REBUILD);
}
- public Boolean isZeppelinSearchUseDisk() {
+ public boolean isZeppelinSearchUseDisk() {
return getBoolean(ConfVars.ZEPPELIN_SEARCH_USE_DISK);
}
@@ -822,7 +780,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getAbsoluteDir(ConfVars.ZEPPELIN_SEARCH_INDEX_PATH);
}
- public Boolean isOnlyYarnCluster() {
+ public boolean isOnlyYarnCluster() {
return getBoolean(ConfVars.ZEPPELIN_SPARK_ONLY_YARN_CLUSTER);
}
@@ -900,40 +858,31 @@ public class ZeppelinConfiguration extends XMLConfiguration {
return getBoolean(ConfVars.ZEPPELIN_METRIC_ENABLE_PROMETHEUS);
}
- public Map<String, String> dumpConfigurations(Predicate<String> predicate) {
- Map<String, String> properties = new HashMap<>();
-
- for (ConfVars v : ConfVars.values()) {
- String key = v.getVarName();
-
- if (!predicate.test(key)) {
- continue;
- }
-
- ConfVars.VarType type = v.getType();
- Object value = null;
- if (type == ConfVars.VarType.BOOLEAN) {
- value = getBoolean(v);
- } else if (type == ConfVars.VarType.LONG) {
- value = getLong(v);
- } else if (type == ConfVars.VarType.INT) {
- value = getInt(v);
- } else if (type == ConfVars.VarType.FLOAT) {
- value = getFloat(v);
- } else if (type == ConfVars.VarType.STRING) {
- value = getString(v);
- }
-
- if (value != null) {
- properties.put(key, value.toString());
+ /**
+ * This method return the complete configuration map
+ * @return
+ */
+ public Map<String, String> getCompleteConfiguration() {
+ Map<String, String> completeConfiguration = new HashMap<>(properties);
+ // Is it possible that we overwrite properties
+ for (ConfVars c : ConfVars.values()) {
+ if (sysConfig.containsKey(c.getVarName())) {
+ completeConfiguration.put(c.getVarName(), sysConfig.getString(c.getVarName()));
+ } else if (envConfig.containsKey(c.name())) {
+ completeConfiguration.put(c.getVarName(), envConfig.getString(c.name()));
}
}
- return properties;
+ return completeConfiguration;
}
- @Override
- public void save(Writer writer) throws ConfigurationException {
- try {
+ public Map<String, String> dumpConfigurations(Predicate<String> predicate) {
+ return getCompleteConfiguration().entrySet().stream()
+ .filter(e -> predicate.test(e.getKey()))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ }
+
+ public void save(String location) throws ConfigurationException {
+ try (FileWriter writer = new FileWriter(location)){
writer.write("<configuration>\n");
for (Map.Entry<String, String> entry : properties.entrySet()) {
writer.write("<property>\n");
@@ -942,11 +891,9 @@ public class ZeppelinConfiguration extends XMLConfiguration {
writer.write("</property>\n");
}
writer.write("</configuration>");
- writer.close();
} catch (IOException e) {
throw new ConfigurationException(e);
}
-
}
/**
@@ -1115,6 +1062,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_DOCKER_CONTAINER_IMAGE("zeppelin.docker.container.image", "apache/zeppelin:" + Util.getVersion()),
ZEPPELIN_METRIC_ENABLE_PROMETHEUS("zeppelin.metric.enable.prometheus", false),
+ ZEPPELINHUB_USER_KEY("zeppelinhub.user.key",""),
ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER("zeppelin.impersonate.spark.proxy.user", true),
ZEPPELIN_NOTEBOOK_GIT_REMOTE_URL("zeppelin.notebook.git.remote.url", ""),
@@ -1138,10 +1086,8 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_NOTE_FILE_EXCLUDE_FIELDS("zeppelin.note.file.exclude.fields", "");
private String varName;
- @SuppressWarnings("rawtypes")
- private Class varClass;
+ private Class<?> varClass;
private String stringValue;
- private VarType type;
private int intValue;
private float floatValue;
private boolean booleanValue;
@@ -1156,7 +1102,6 @@ public class ZeppelinConfiguration extends XMLConfiguration {
this.floatValue = -1;
this.longValue = -1;
this.booleanValue = false;
- this.type = VarType.STRING;
}
ConfVars(String varName, int intValue) {
@@ -1167,7 +1112,6 @@ public class ZeppelinConfiguration extends XMLConfiguration {
this.floatValue = -1;
this.longValue = -1;
this.booleanValue = false;
- this.type = VarType.INT;
}
ConfVars(String varName, long longValue) {
@@ -1178,7 +1122,6 @@ public class ZeppelinConfiguration extends XMLConfiguration {
this.floatValue = -1;
this.longValue = longValue;
this.booleanValue = false;
- this.type = VarType.LONG;
}
ConfVars(String varName, float floatValue) {
@@ -1189,7 +1132,6 @@ public class ZeppelinConfiguration extends XMLConfiguration {
this.longValue = -1;
this.floatValue = floatValue;
this.booleanValue = false;
- this.type = VarType.FLOAT;
}
ConfVars(String varName, boolean booleanValue) {
@@ -1200,15 +1142,13 @@ public class ZeppelinConfiguration extends XMLConfiguration {
this.longValue = -1;
this.floatValue = -1;
this.booleanValue = booleanValue;
- this.type = VarType.BOOLEAN;
}
public String getVarName() {
return varName;
}
- @SuppressWarnings("rawtypes")
- public Class getVarClass() {
+ public Class<?> getVarClass() {
return varClass;
}
@@ -1231,56 +1171,5 @@ public class ZeppelinConfiguration extends XMLConfiguration {
public boolean getBooleanValue() {
return booleanValue;
}
-
- public VarType getType() {
- return type;
- }
-
- enum VarType {
- STRING {
- @Override
- void checkType(String value) throws Exception {}
- },
- INT {
- @Override
- void checkType(String value) throws Exception {
- Integer.valueOf(value);
- }
- },
- LONG {
- @Override
- void checkType(String value) throws Exception {
- Long.valueOf(value);
- }
- },
- FLOAT {
- @Override
- void checkType(String value) throws Exception {
- Float.valueOf(value);
- }
- },
- BOOLEAN {
- @Override
- void checkType(String value) throws Exception {
- Boolean.valueOf(value);
- }
- };
-
- boolean isType(String value) {
- try {
- checkType(value);
- } catch (Exception e) {
- LOG.error("Exception in ZeppelinConfiguration while isType", e);
- return false;
- }
- return true;
- }
-
- String typeString() {
- return name().toUpperCase();
- }
-
- abstract void checkType(String value) throws Exception;
- }
}
}
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinLocationStrategy.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinLocationStrategy.java
new file mode 100644
index 0000000..57bd034
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinLocationStrategy.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zeppelin.conf;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Map;
+
+import org.apache.commons.configuration2.io.FileLocationStrategy;
+import org.apache.commons.configuration2.io.FileLocator;
+import org.apache.commons.configuration2.io.FileSystem;
+import org.apache.commons.exec.environment.EnvironmentUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZeppelinLocationStrategy implements FileLocationStrategy {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ZeppelinLocationStrategy.class);
+
+ @Override
+ public URL locate(FileSystem fileSystem, FileLocator locator) {
+ try {
+ Map<String, String> procEnv = EnvironmentUtils.getProcEnvironment();
+ if (procEnv.containsKey("ZEPPELIN_HOME")) {
+ String zconfDir = procEnv.get("ZEPPELIN_HOME");
+ File file = new File(zconfDir + File.separator
+ + "conf" + File.separator + locator.getFileName());
+ if (file.isFile()) {
+ LOGGER.info("Load configuration from {}", file);
+ return file.toURI().toURL();
+ }
+ }
+
+ if (procEnv.containsKey("ZEPPELIN_CONF_DIR")) {
+ String zconfDir = procEnv.get("ZEPPELIN_CONF_DIR");
+ File file = new File(zconfDir + File.separator + locator.getFileName());
+ if (file.isFile()) {
+ LOGGER.info("Load configuration from {}", file);
+ return file.toURI().toURL();
+ }
+ }
+ } catch (IOException e) {
+ LOGGER.error(e.getMessage(), e);
+ }
+ return null;
+ }
+}
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/cluster/ClusterSingleNodeTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/cluster/ClusterSingleNodeTest.java
index 7f78b6e..b6bb921 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/cluster/ClusterSingleNodeTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/cluster/ClusterSingleNodeTest.java
@@ -16,7 +16,6 @@
*/
package org.apache.zeppelin.cluster;
-import org.apache.commons.configuration.ConfigurationException;
import org.apache.zeppelin.cluster.meta.ClusterMeta;
import org.apache.zeppelin.cluster.meta.ClusterMetaType;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
@@ -45,18 +44,16 @@ public class ClusterSingleNodeTest {
static final String metaKey = "ClusterSingleNodeTestKey";
@BeforeClass
- public static void startCluster() throws IOException, InterruptedException, ConfigurationException {
+ public static void startCluster() throws IOException, InterruptedException {
LOGGER.info("startCluster >>>");
- zconf = ZeppelinConfiguration.create();
+ zconf = ZeppelinConfiguration.create("zeppelin-site-test.xml");
// Set the cluster IP and port
zServerHost = RemoteInterpreterUtils.findAvailableHostAddress();
zServerPort = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces();
zconf.setClusterAddress(zServerHost + ":" + zServerPort);
- // mock cluster manager server
- zconf.load(ClusterSingleNodeTest.class.getResource("/zeppelin-site-test.xml"));
clusterServer = ClusterManagerServer.getInstance(zconf);
clusterServer.start();
diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
index c3f225e..76d0f82 100644
--- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
+++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java
@@ -154,7 +154,6 @@ public class ZeppelinServer extends ResourceConfig {
public static void main(String[] args) throws InterruptedException, IOException {
ZeppelinServer.conf = ZeppelinConfiguration.create();
- conf.setProperty("args", args);
jettyWebServer = setupJettyServer(conf);
initMetrics(conf);
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/cluster/ZeppelinServerMock.java b/zeppelin-server/src/test/java/org/apache/zeppelin/cluster/ZeppelinServerMock.java
index 6ebfadc..fa6d859 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/cluster/ZeppelinServerMock.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/cluster/ZeppelinServerMock.java
@@ -17,24 +17,6 @@
package org.apache.zeppelin.cluster;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.http.Header;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpStatus;
-import org.apache.http.NameValuePair;
-import org.apache.http.client.config.CookieSpecs;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.entity.UrlEncodedFormEntity;
-import org.apache.http.client.methods.CloseableHttpResponse;
-import org.apache.http.client.methods.HttpDelete;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.message.BasicNameValuePair;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.interpreter.InterpreterSetting;
import org.apache.zeppelin.notebook.Notebook;
@@ -42,21 +24,13 @@ import org.apache.zeppelin.plugin.PluginManager;
import org.apache.zeppelin.rest.AbstractTestRestApi;
import org.apache.zeppelin.server.ZeppelinServer;
import org.apache.zeppelin.utils.TestUtils;
-import org.hamcrest.Description;
-import org.hamcrest.Matcher;
-import org.hamcrest.TypeSafeMatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.IOException;
-import java.lang.ref.WeakReference;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.regex.Pattern;
public class ZeppelinServerMock {
protected static final Logger LOG = LoggerFactory.getLogger(ZeppelinServerMock.class);
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/service/ShiroAuthenticationServiceTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/service/ShiroAuthenticationServiceTest.java
index ef0e679..1311b2e 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/service/ShiroAuthenticationServiceTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/service/ShiroAuthenticationServiceTest.java
@@ -22,7 +22,6 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.security.Principal;
-import org.apache.commons.configuration.ConfigurationException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.notebook.Notebook;
import org.junit.Before;
@@ -74,12 +73,9 @@ public class ShiroAuthenticationServiceTest {
when(subject.getPrincipal()).thenReturn(new TestPrincipal(expectedName));
Notebook notebook = Mockito.mock(Notebook.class);
- try {
- when(notebook.getConf())
- .thenReturn(new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml")));
- } catch (ConfigurationException e) {
- e.printStackTrace();
- }
+ when(notebook.getConf())
+ .thenReturn(ZeppelinConfiguration.create("zeppelin-site.xml"));
+
}
public class TestPrincipal implements Principal {
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/utils/CorsUtilsTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/utils/CorsUtilsTest.java
index 8fb934d..fe4b6a0 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/utils/CorsUtilsTest.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/utils/CorsUtilsTest.java
@@ -22,11 +22,16 @@ import static org.junit.Assert.assertTrue;
import java.net.InetAddress;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
-import org.apache.commons.configuration.ConfigurationException;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.junit.After;
import org.junit.Test;
public class CorsUtilsTest {
+
+ @After
+ public void cleanup() {
+ ZeppelinConfiguration.reset();
+ }
@Test
public void isInvalid() throws URISyntaxException, UnknownHostException {
assertFalse(CorsUtils.isValidOrigin("http://127.0.1.1", ZeppelinConfiguration.create()));
@@ -34,9 +39,9 @@ public class CorsUtilsTest {
@Test
public void isInvalidFromConfig()
- throws URISyntaxException, UnknownHostException, ConfigurationException {
+ throws URISyntaxException, UnknownHostException {
assertFalse(CorsUtils.isValidOrigin("http://otherinvalidhost.com",
- new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"))));
+ ZeppelinConfiguration.create("zeppelin-site.xml")));
}
@Test
@@ -53,43 +58,43 @@ public class CorsUtilsTest {
@Test
public void isValidFromConfig()
- throws URISyntaxException, UnknownHostException, ConfigurationException {
+ throws URISyntaxException, UnknownHostException {
assertTrue(CorsUtils.isValidOrigin("http://otherhost.com",
- new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"))));
+ ZeppelinConfiguration.create("zeppelin-site.xml")));
}
@Test
public void isValidFromStar()
- throws URISyntaxException, UnknownHostException, ConfigurationException {
+ throws URISyntaxException, UnknownHostException {
assertTrue(CorsUtils.isValidOrigin("http://anyhost.com",
- new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site-star.xml"))));
+ ZeppelinConfiguration.create("zeppelin-site-star.xml")));
}
@Test
public void nullOrigin()
- throws URISyntaxException, UnknownHostException, ConfigurationException {
+ throws URISyntaxException, UnknownHostException {
assertFalse(CorsUtils.isValidOrigin(null,
- new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"))));
+ ZeppelinConfiguration.create("zeppelin-site.xml")));
}
@Test
public void nullOriginWithStar()
- throws URISyntaxException, UnknownHostException, ConfigurationException {
+ throws URISyntaxException, UnknownHostException {
assertTrue(CorsUtils.isValidOrigin(null,
- new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site-star.xml"))));
+ ZeppelinConfiguration.create("zeppelin-site-star.xml")));
}
@Test
public void emptyOrigin()
- throws URISyntaxException, UnknownHostException, ConfigurationException {
+ throws URISyntaxException, UnknownHostException {
assertFalse(CorsUtils.isValidOrigin("",
- new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"))));
+ ZeppelinConfiguration.create("zeppelin-site.xml")));
}
@Test
public void notAURIOrigin()
- throws URISyntaxException, UnknownHostException, ConfigurationException {
+ throws URISyntaxException, UnknownHostException {
assertFalse(CorsUtils.isValidOrigin("test123",
- new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-site.xml"))));
+ ZeppelinConfiguration.create("zeppelin-site.xml")));
}
}
diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml
index e6ee797..3df3689 100644
--- a/zeppelin-zengine/pom.xml
+++ b/zeppelin-zengine/pom.xml
@@ -44,7 +44,6 @@
<frontend.maven.plugin.version>1.3</frontend.maven.plugin.version>
<commons.vfs2.version>2.2</commons.vfs2.version>
<eclipse.jgit.version>4.5.4.201711221230-r</eclipse.jgit.version>
- <commons.configuration2.version>2.2</commons.configuration2.version>
<!--test library versions-->
<google.truth.version>0.27</google.truth.version>
<google.testing.nio.version>0.32.0-alpha</google.testing.nio.version>
@@ -211,19 +210,6 @@
</exclusions>
</dependency>
- <!-- Needed for string interpolation in shiro.ini -->
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-configuration2</artifactId>
- <version>${commons.configuration2.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/YarnAppMonitor.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/YarnAppMonitor.java
index d84092b..2705398 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/YarnAppMonitor.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/YarnAppMonitor.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess;
import org.apache.zeppelin.scheduler.SchedulerThreadFactory;
import org.slf4j.Logger;
@@ -93,8 +94,8 @@ public class YarnAppMonitor {
LOGGER.warn("Fail to check yarn app status", e);
}
},
- conf.getInt("zeppelin.interpreter.yarn.monitor.interval_secs", 10),
- conf.getInt("zeppelin.interpreter.yarn.monitor.interval_secs", 10),
+ conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_YARN_MONITOR_INTERVAL_SECS),
+ conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_YARN_MONITOR_INTERVAL_SECS),
TimeUnit.SECONDS);
LOGGER.info("YarnAppMonitor is started");
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
index 19088f2..797e546 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
@@ -100,7 +100,7 @@ public abstract class RemoteInterpreterProcess implements InterpreterClient {
public void init(ZeppelinConfiguration zConf) {
callRemoteFunction(client -> {
- client.init(zConf.getProperties());
+ client.init(zConf.getCompleteConfiguration());
return null;
});
}
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/security/Authentication.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/security/Authentication.java
index f579fe7..5d623b2 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/security/Authentication.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/zeppelinhub/security/Authentication.java
@@ -28,6 +28,7 @@ import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
+import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,7 +53,6 @@ public class Authentication implements Runnable {
private static final String CIPHER_MODE = "AES/CBC/PKCS5PADDING";
private static final int IV_SIZE = 16;
- private static final String ZEPPELINHUB_USER_KEY = "zeppelinhub.user.key";
private String token;
private boolean authEnabled;
private boolean authenticated;
@@ -82,8 +82,7 @@ public class Authentication implements Runnable {
authEnabled = !conf.isAnonymousAllowed();
- userKey = conf.getString("ZEPPELINHUB_USER_KEY",
- ZEPPELINHUB_USER_KEY, "");
+ userKey = conf.getString(ConfVars.ZEPPELINHUB_USER_KEY);
loginEndpoint = getLoginEndpoint(conf);
}
@@ -104,7 +103,7 @@ public class Authentication implements Runnable {
return authenticated;
}
private String getLoginEndpoint(ZeppelinConfiguration conf) {
- int port = conf.getInt("ZEPPELIN_PORT", "zeppelin.server.port" , 8080);
+ int port = conf.getInt(ConfVars.ZEPPELIN_PORT);
if (port <= 0) {
port = 8080;
}
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/user/Credentials.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/user/Credentials.java
index 51b3d62..867d79e 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/user/Credentials.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/user/Credentials.java
@@ -52,7 +52,7 @@ public class Credentials {
*/
public Credentials(ZeppelinConfiguration conf) {
credentialsMap = new HashMap<>();
- if (conf.credentialsPersist().booleanValue()) {
+ if (conf.credentialsPersist()) {
String encryptKey = conf.getCredentialsEncryptKey();
if (StringUtils.isNotBlank(encryptKey)) {
this.encryptor = new Encryptor(encryptKey);
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/conf/ZeppelinConfigurationTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/conf/ZeppelinConfigurationTest.java
index 1557d0e..5ef1040 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/conf/ZeppelinConfigurationTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/conf/ZeppelinConfigurationTest.java
@@ -17,11 +17,10 @@
package org.apache.zeppelin.conf;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
+import org.junit.After;
import org.junit.Assert;
-import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
@@ -32,15 +31,20 @@ import java.util.List;
public class ZeppelinConfigurationTest {
- @Before
- public void clearSystemVariables() {
- System.clearProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName());
+ @BeforeClass
+ public static void clearSystemVariables() {
+ ZeppelinConfiguration.reset();
+ }
+
+ @After
+ public void cleanup() {
+ ZeppelinConfiguration.reset();
}
@Test
- public void getAllowedOrigins2Test() throws MalformedURLException, ConfigurationException {
+ public void getAllowedOrigins2Test() throws MalformedURLException {
- ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/test-zeppelin-site2.xml"));
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create("test-zeppelin-site2.xml");
List<String> origins = conf.getAllowedOrigins();
Assert.assertEquals(2, origins.size());
Assert.assertEquals("http://onehost:8080", origins.get(0));
@@ -48,94 +52,108 @@ public class ZeppelinConfigurationTest {
}
@Test
- public void getAllowedOrigins1Test() throws MalformedURLException, ConfigurationException {
+ public void getAllowedOrigins1Test() throws MalformedURLException {
- ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/test-zeppelin-site1.xml"));
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create("test-zeppelin-site1.xml");
List<String> origins = conf.getAllowedOrigins();
Assert.assertEquals(1, origins.size());
Assert.assertEquals("http://onehost:8080", origins.get(0));
}
@Test
- public void getAllowedOriginsNoneTest() throws MalformedURLException, ConfigurationException {
+ public void getAllowedOriginsNoneTest() throws MalformedURLException {
- ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-test-site.xml"));
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create("zeppelin-test-site.xml");
List<String> origins = conf.getAllowedOrigins();
Assert.assertEquals(1, origins.size());
}
@Test
- public void isWindowsPathTestTrue() throws ConfigurationException {
+ public void isWindowsPathTestTrue() {
- ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-test-site.xml"));
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create("zeppelin-test-site.xml");
Boolean isIt = conf.isWindowsPath("c:\\test\\file.txt");
Assert.assertTrue(isIt);
}
@Test
- public void isWindowsPathTestFalse() throws ConfigurationException {
+ public void isWindowsPathTestFalse() {
- ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-test-site.xml"));
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create("zeppelin-test-site.xml");
Boolean isIt = conf.isWindowsPath("~/test/file.xml");
Assert.assertFalse(isIt);
}
@Test
- public void isPathWithSchemeTestTrue() throws ConfigurationException {
+ public void isPathWithSchemeTestTrue() {
- ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-test-site.xml"));
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create("zeppelin-test-site.xml");
Boolean isIt = conf.isPathWithScheme("hdfs://hadoop.example.com/zeppelin/notebook");
Assert.assertTrue(isIt);
}
@Test
- public void isPathWithSchemeTestFalse() throws ConfigurationException {
+ public void isPathWithSchemeTestFalse() {
- ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-test-site.xml"));
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create("zeppelin-test-site.xml");
Boolean isIt = conf.isPathWithScheme("~/test/file.xml");
Assert.assertFalse(isIt);
}
@Test
- public void isPathWithInvalidSchemeTest() throws ConfigurationException {
+ public void isPathWithInvalidSchemeTest() {
- ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-test-site.xml"));
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create("zeppelin-test-site.xml");
Boolean isIt = conf.isPathWithScheme("c:\\test\\file.txt");
Assert.assertFalse(isIt);
}
@Test
- public void getNotebookDirTest() throws ConfigurationException {
- ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-test-site.xml"));
+ public void getNotebookDirTest() {
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create("zeppelin-test-site.xml");
String notebookLocation = conf.getNotebookDir();
assertTrue(notebookLocation.endsWith("notebook"));
}
@Test
- public void isNotebookPublicTest() throws ConfigurationException {
+ public void isNotebookPublicTest() {
- ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-test-site.xml"));
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create("zeppelin-test-site.xml");
boolean isIt = conf.isNotebookPublic();
assertTrue(isIt);
}
@Test
- public void getPathTest() throws ConfigurationException {
- System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), "/usr/lib/zeppelin");
- ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-test-site.xml"));
+ public void getPathTest() {
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create("zeppelin-test-site.xml");
+ conf.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), "/usr/lib/zeppelin");
Assert.assertEquals("/usr/lib/zeppelin", conf.getZeppelinHome());
Assert.assertEquals("/usr/lib/zeppelin/conf", conf.getConfDir());
}
@Test
- public void getConfigFSPath() throws ConfigurationException {
- System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), "/usr/lib/zeppelin");
- System.setProperty(ConfVars.ZEPPELIN_CONFIG_FS_DIR.getVarName(), "conf");
- ZeppelinConfiguration conf = new ZeppelinConfiguration(this.getClass().getResource("/zeppelin-test-site.xml"));
+ public void getConfigFSPath() {
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create("zeppelin-test-site.xml");
+ conf.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), "/usr/lib/zeppelin");
+ conf.setProperty(ConfVars.ZEPPELIN_CONFIG_FS_DIR.getVarName(), "conf");
assertEquals("/usr/lib/zeppelin/conf", conf.getConfigFSDir(true));
- System.setProperty(ConfVars.ZEPPELIN_CONFIG_STORAGE_CLASS.getVarName(), "org.apache.zeppelin.storage.FileSystemConfigStorage");
+ conf.setProperty(ConfVars.ZEPPELIN_CONFIG_STORAGE_CLASS.getVarName(),
+ "org.apache.zeppelin.storage.FileSystemConfigStorage");
assertEquals("conf", conf.getConfigFSDir(false));
}
+ @Test
+ public void checkParseException() {
+ ZeppelinConfiguration conf = ZeppelinConfiguration.create("zeppelin-test-site.xml");
+ // when
+ conf.setProperty(ConfVars.ZEPPELIN_PORT.getVarName(), "luke skywalker");
+ // then
+ assertEquals(8080, conf.getServerPort());
+
+ // when
+ conf.setProperty(ConfVars.ZEPPELIN_PORT.getVarName(), "12345");
+ // then
+ assertEquals(12345, conf.getServerPort());
+ }
}