You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2017/08/28 01:14:28 UTC
[11/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
[ZEPPELIN-2627] Interpreter refactor
### What is this PR for?
I didn't intended to make such large change at the beginning, but found many things are coupled together that I have to make such large change. Several suggestions for you how to review and read it.
* I move the interpreter package from zeppelin-zengine to zeppelin-interpreter, this is needed for this refactoring.
* The overall change is the same as I described in the design doc. I would suggest you to read the unit test first. These unit test is very readable and easy to understand what the code is doing now. `InterpreterFactoryTest`, `InterpreterGroupTest`, `InterpreterSettingTest`, `InterpreterSettingManagerTest`, `RemoteInterpreterTest`.
* Remove the referent counting logic. Now I will kill the interpreter process as long as all the sessions in the same interpreter group is closed. (I plan to add another kind of policy for the interpreter process lifecycle control, ZEPPELIN-2197)
* The `RemoteFunction` I introduced is for reducing code duplicates when we use RPC.
* The changes in Job.java and RemoteScheduler is for fixing the race issue bug. This bug cause the flaky test we see often in `ZeppelinSparkClusterTest.pySparkTest`
### What type of PR is it?
[Bug Fix | Refactoring]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-2627
### How should this be tested?
Unit test is added
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #2422 from zjffdu/interpreter_refactor and squashes the following commits:
4724c98 [Jeff Zhang] [ZEPPELIN-2627] Interpreter Component Refactoring
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/8d4902e7
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/8d4902e7
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/8d4902e7
Branch: refs/heads/master
Commit: 8d4902e717ba2932ddb0f7110f2a669811f5d1af
Parents: 32517c9
Author: Jeff Zhang <zj...@apache.org>
Authored: Tue May 23 15:52:15 2017 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Mon Aug 28 09:14:11 2017 +0800
----------------------------------------------------------------------
.travis.yml | 1 +
.../zeppelin/helium/ZeppelinDevServer.java | 8 +-
zeppelin-interpreter/pom.xml | 22 +
.../zeppelin/conf/ZeppelinConfiguration.java | 835 +++++++++++++
.../zeppelin/interpreter/Interpreter.java | 1 -
.../interpreter/InterpreterFactory.java | 115 ++
.../zeppelin/interpreter/InterpreterGroup.java | 272 ++---
.../zeppelin/interpreter/InterpreterInfo.java | 82 ++
.../interpreter/InterpreterInfoSaving.java | 101 ++
.../interpreter/InterpreterProperty.java | 1 +
.../zeppelin/interpreter/InterpreterRunner.java | 11 +
.../interpreter/InterpreterSetting.java | 911 ++++++++++++++
.../interpreter/InterpreterSettingManager.java | 886 ++++++++++++++
.../interpreter/install/InstallInterpreter.java | 288 +++++
.../remote/RemoteAngularObjectRegistry.java | 125 ++
.../interpreter/remote/RemoteInterpreter.java | 371 ++++++
.../remote/RemoteInterpreterEventPoller.java | 279 ++---
.../remote/RemoteInterpreterManagedProcess.java | 260 ++++
.../remote/RemoteInterpreterProcess.java | 154 +--
.../remote/RemoteInterpreterRunningProcess.java | 67 ++
.../remote/RemoteInterpreterServer.java | 58 +-
.../zeppelin/resource/ResourcePoolUtils.java | 138 ---
.../java/org/apache/zeppelin/scheduler/Job.java | 11 +-
.../zeppelin/scheduler/RemoteScheduler.java | 134 +--
.../zeppelin/scheduler/SchedulerFactory.java | 49 +-
.../zeppelin/tabledata/TableDataProxy.java | 1 -
.../java/org/apache/zeppelin/util/IdHashes.java | 76 ++
.../java/org/apache/zeppelin/util/Util.java | 76 ++
.../interpreter/AbstractInterpreterTest.java | 74 ++
.../interpreter/DoubleEchoInterpreter.java | 61 +
.../zeppelin/interpreter/DummyInterpreter.java | 43 -
.../zeppelin/interpreter/EchoInterpreter.java | 65 +
.../interpreter/InterpreterFactoryTest.java | 66 +
.../interpreter/InterpreterGroupTest.java | 90 ++
.../InterpreterOutputChangeWatcherTest.java | 11 +-
.../InterpreterSettingManagerTest.java | 270 +++++
.../interpreter/InterpreterSettingTest.java | 411 +++++++
.../zeppelin/interpreter/InterpreterTest.java | 7 +-
.../zeppelin/interpreter/SleepInterpreter.java | 60 +
.../install/InstallInterpreterTest.java | 86 ++
.../interpreter/mock/MockInterpreter1.java | 106 ++
.../interpreter/mock/MockInterpreter11.java | 85 ++
.../interpreter/mock/MockInterpreter2.java | 104 ++
.../remote/AppendOutputRunnerTest.java | 232 ++++
.../remote/RemoteAngularObjectTest.java | 196 +++
.../RemoteInterpreterEventPollerTest.java | 55 +
.../RemoteInterpreterOutputTestStream.java | 178 +++
.../remote/RemoteInterpreterTest.java | 520 ++++++++
.../remote/RemoteInterpreterUtilsTest.java | 33 +
.../remote/mock/GetEnvPropertyInterpreter.java | 82 ++
.../remote/mock/MockInterpreterA.java | 96 ++
.../remote/mock/MockInterpreterAngular.java | 112 ++
.../remote/mock/MockInterpreterB.java | 120 ++
.../mock/MockInterpreterOutputStream.java | 93 ++
.../mock/MockInterpreterResourcePool.java | 125 ++
.../zeppelin/scheduler/RemoteSchedulerTest.java | 343 ++++++
.../src/test/resources/conf/interpreter.json | 115 ++
.../interpreter/test/interpreter-setting.json | 42 +
.../src/test/resources/log4j.properties | 4 +-
.../zeppelin/rest/InterpreterRestApi.java | 4 +-
.../apache/zeppelin/server/ZeppelinServer.java | 35 +-
.../apache/zeppelin/socket/NotebookServer.java | 32 +-
.../interpreter/mock/MockInterpreter1.java | 75 --
.../zeppelin/rest/AbstractTestRestApi.java | 7 +-
.../zeppelin/rest/InterpreterRestApiTest.java | 6 +-
.../zeppelin/socket/NotebookServerTest.java | 13 +-
.../src/test/resources/log4j.properties | 1 -
zeppelin-zengine/pom.xml | 13 +-
.../zeppelin/conf/ZeppelinConfiguration.java | 835 -------------
.../java/org/apache/zeppelin/helium/Helium.java | 51 +-
.../helium/HeliumApplicationFactory.java | 120 +-
.../interpreter/InterpreterFactory.java | 423 -------
.../interpreter/InterpreterGroupFactory.java | 26 -
.../zeppelin/interpreter/InterpreterInfo.java | 82 --
.../interpreter/InterpreterInfoSaving.java | 46 -
.../interpreter/InterpreterSetting.java | 459 -------
.../interpreter/InterpreterSettingManager.java | 1136 ------------------
.../interpreter/install/InstallInterpreter.java | 290 -----
.../remote/RemoteAngularObjectRegistry.java | 133 --
.../interpreter/remote/RemoteInterpreter.java | 597 ---------
.../remote/RemoteInterpreterManagedProcess.java | 247 ----
.../remote/RemoteInterpreterRunningProcess.java | 67 --
.../zeppelin/notebook/ApplicationState.java | 1 -
.../java/org/apache/zeppelin/notebook/Note.java | 12 +-
.../org/apache/zeppelin/notebook/Notebook.java | 23 +-
.../org/apache/zeppelin/notebook/Paragraph.java | 21 +-
.../java/org/apache/zeppelin/util/Util.java | 76 --
.../helium/HeliumApplicationFactoryTest.java | 79 +-
.../org/apache/zeppelin/helium/HeliumTest.java | 8 +-
.../interpreter/InterpreterFactoryTest.java | 497 --------
.../interpreter/InterpreterSettingTest.java | 327 -----
.../install/InstallInterpreterTest.java | 86 --
.../interpreter/mock/MockInterpreter1.java | 105 --
.../interpreter/mock/MockInterpreter11.java | 83 --
.../interpreter/mock/MockInterpreter2.java | 104 --
.../remote/AppendOutputRunnerTest.java | 236 ----
.../remote/RemoteAngularObjectTest.java | 201 ----
.../RemoteInterpreterEventPollerTest.java | 55 -
.../RemoteInterpreterOutputTestStream.java | 191 ---
.../remote/RemoteInterpreterProcessTest.java | 131 --
.../remote/RemoteInterpreterTest.java | 975 ---------------
.../remote/RemoteInterpreterUtilsTest.java | 34 -
.../remote/mock/MockInterpreterA.java | 97 --
.../remote/mock/MockInterpreterAngular.java | 113 --
.../remote/mock/MockInterpreterB.java | 126 --
.../remote/mock/MockInterpreterEnv.java | 80 --
.../mock/MockInterpreterOutputStream.java | 90 --
.../mock/MockInterpreterResourcePool.java | 128 --
.../notebook/NoteInterpreterLoaderTest.java | 243 ----
.../apache/zeppelin/notebook/NotebookTest.java | 181 ++-
.../apache/zeppelin/notebook/ParagraphTest.java | 9 +-
.../notebook/repo/NotebookRepoSyncTest.java | 74 +-
.../notebook/repo/VFSNotebookRepoTest.java | 55 +-
.../resource/DistributedResourcePoolTest.java | 96 +-
.../zeppelin/scheduler/RemoteSchedulerTest.java | 364 ------
.../src/test/resources/conf/interpreter.json | 1 +
.../interpreter/mock/interpreter-setting.json | 12 -
.../interpreter/mock1/interpreter-setting.json | 19 +
.../interpreter/mock2/interpreter-setting.json | 19 +
.../mock_resource_pool/interpreter-setting.json | 19 +
.../src/test/resources/log4j.properties | 3 +-
121 files changed, 8861 insertions(+), 10029 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 97ca60a..64ea559 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -162,6 +162,7 @@ after_success:
after_failure:
- echo "Travis exited with ${TRAVIS_TEST_RESULT}"
- find . -name rat.txt | xargs cat
+ - cat logs/*
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
- cat zeppelin-web/npm-debug.log
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
----------------------------------------------------------------------
diff --git a/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java b/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
index 21ce283..2484469 100644
--- a/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
+++ b/helium-dev/src/main/java/org/apache/zeppelin/helium/ZeppelinDevServer.java
@@ -43,13 +43,13 @@ public class ZeppelinDevServer extends
}
@Override
- protected Interpreter getInterpreter(String sessionKey, String className) throws TException {
+ protected Interpreter getInterpreter(String sessionId, String className) throws TException {
synchronized (this) {
InterpreterGroup interpreterGroup = getInterpreterGroup();
if (interpreterGroup == null || interpreterGroup.isEmpty()) {
createInterpreter(
"dev",
- sessionKey,
+ sessionId,
DevInterpreter.class.getName(),
new HashMap<String, String>(),
"anonymous");
@@ -57,11 +57,11 @@ public class ZeppelinDevServer extends
}
}
- Interpreter intp = super.getInterpreter(sessionKey, className);
+ Interpreter intp = super.getInterpreter(sessionId, className);
interpreter = (DevInterpreter) (
((LazyOpenInterpreter) intp).getInnerInterpreter());
interpreter.setInterpreterEvent(this);
- return super.getInterpreter(sessionKey, className);
+ return super.getInterpreter(sessionId, className);
}
@Override
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/pom.xml
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml
index 384b9d1..d8f9c43 100644
--- a/zeppelin-interpreter/pom.xml
+++ b/zeppelin-interpreter/pom.xml
@@ -52,6 +52,11 @@
<dependencies>
<dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
</dependency>
@@ -226,4 +231,21 @@
<scope>test</scope>
</dependency>
</dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <version>3.0.2</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
new file mode 100644
index 0000000..03cc069
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -0,0 +1,835 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.conf;
+
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.XMLConfiguration;
+import org.apache.commons.configuration.tree.ConfigurationNode;
+import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.util.Util;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Zeppelin configuration.
+ *
+ */
+public class ZeppelinConfiguration extends XMLConfiguration {
+ private static final String ZEPPELIN_SITE_XML = "zeppelin-site.xml";
+ private static final long serialVersionUID = 4749305895693848035L;
+ private static final Logger LOG = LoggerFactory.getLogger(ZeppelinConfiguration.class);
+
+ private static final String HELIUM_PACKAGE_DEFAULT_URL =
+ "https://s3.amazonaws.com/helium-package/helium.json";
+ private static ZeppelinConfiguration conf;
+
+ public ZeppelinConfiguration(URL url) throws ConfigurationException {
+ setDelimiterParsingDisabled(true);
+ load(url);
+ }
+
+ public ZeppelinConfiguration() {
+ ConfVars[] vars = ConfVars.values();
+ for (ConfVars v : vars) {
+ if (v.getType() == ConfVars.VarType.BOOLEAN) {
+ this.setProperty(v.getVarName(), v.getBooleanValue());
+ } else if (v.getType() == ConfVars.VarType.LONG) {
+ this.setProperty(v.getVarName(), v.getLongValue());
+ } else if (v.getType() == ConfVars.VarType.INT) {
+ this.setProperty(v.getVarName(), v.getIntValue());
+ } else if (v.getType() == ConfVars.VarType.FLOAT) {
+ this.setProperty(v.getVarName(), v.getFloatValue());
+ } else if (v.getType() == ConfVars.VarType.STRING) {
+ this.setProperty(v.getVarName(), v.getStringValue());
+ } else {
+ throw new RuntimeException("Unsupported VarType");
+ }
+ }
+
+ }
+
+
+ /**
+ * Load from resource.
+ *url = ZeppelinConfiguration.class.getResource(ZEPPELIN_SITE_XML);
+ * @throws ConfigurationException
+ */
+ public static synchronized ZeppelinConfiguration create() {
+ if (conf != null) {
+ return conf;
+ }
+
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ URL url;
+
+ url = ZeppelinConfiguration.class.getResource(ZEPPELIN_SITE_XML);
+ if (url == null) {
+ ClassLoader cl = ZeppelinConfiguration.class.getClassLoader();
+ if (cl != null) {
+ url = cl.getResource(ZEPPELIN_SITE_XML);
+ }
+ }
+ if (url == null) {
+ url = classLoader.getResource(ZEPPELIN_SITE_XML);
+ }
+
+ if (url == null) {
+ LOG.warn("Failed to load configuration, proceeding with a default");
+ conf = new ZeppelinConfiguration();
+ } else {
+ try {
+ LOG.info("Load configuration from " + url);
+ conf = new ZeppelinConfiguration(url);
+ } catch (ConfigurationException e) {
+ LOG.warn("Failed to load configuration from " + url + " proceeding with a default", e);
+ conf = new ZeppelinConfiguration();
+ }
+ }
+
+ LOG.info("Server Host: " + conf.getServerAddress());
+ if (conf.useSsl() == false) {
+ LOG.info("Server Port: " + conf.getServerPort());
+ } else {
+ LOG.info("Server SSL Port: " + conf.getServerSslPort());
+ }
+ LOG.info("Context Path: " + conf.getServerContextPath());
+ LOG.info("Zeppelin Version: " + Util.getVersion());
+
+ return conf;
+ }
+
+
+ private String getStringValue(String name, String d) {
+ List<ConfigurationNode> properties = getRootNode().getChildren();
+ if (properties == null || properties.isEmpty()) {
+ return d;
+ }
+ for (ConfigurationNode p : properties) {
+ if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
+ && name.equals(p.getChildren("name").get(0).getValue())) {
+ return (String) p.getChildren("value").get(0).getValue();
+ }
+ }
+ return d;
+ }
+
+ private int getIntValue(String name, int d) {
+ List<ConfigurationNode> properties = getRootNode().getChildren();
+ if (properties == null || properties.isEmpty()) {
+ return d;
+ }
+ for (ConfigurationNode p : properties) {
+ if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
+ && name.equals(p.getChildren("name").get(0).getValue())) {
+ return Integer.parseInt((String) p.getChildren("value").get(0).getValue());
+ }
+ }
+ return d;
+ }
+
+ private long getLongValue(String name, long d) {
+ List<ConfigurationNode> properties = getRootNode().getChildren();
+ if (properties == null || properties.isEmpty()) {
+ return d;
+ }
+ for (ConfigurationNode p : properties) {
+ if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
+ && name.equals(p.getChildren("name").get(0).getValue())) {
+ return Long.parseLong((String) p.getChildren("value").get(0).getValue());
+ }
+ }
+ return d;
+ }
+
+ private float getFloatValue(String name, float d) {
+ List<ConfigurationNode> properties = getRootNode().getChildren();
+ if (properties == null || properties.isEmpty()) {
+ return d;
+ }
+ for (ConfigurationNode p : properties) {
+ if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
+ && name.equals(p.getChildren("name").get(0).getValue())) {
+ return Float.parseFloat((String) p.getChildren("value").get(0).getValue());
+ }
+ }
+ return d;
+ }
+
+ private boolean getBooleanValue(String name, boolean d) {
+ List<ConfigurationNode> properties = getRootNode().getChildren();
+ if (properties == null || properties.isEmpty()) {
+ return d;
+ }
+ for (ConfigurationNode p : properties) {
+ if (p.getChildren("name") != null && !p.getChildren("name").isEmpty()
+ && name.equals(p.getChildren("name").get(0).getValue())) {
+ return Boolean.parseBoolean((String) p.getChildren("value").get(0).getValue());
+ }
+ }
+ return d;
+ }
+
+ public String getString(ConfVars c) {
+ return getString(c.name(), c.getVarName(), c.getStringValue());
+ }
+
+ public String getString(String envName, String propertyName, String defaultValue) {
+ if (System.getenv(envName) != null) {
+ return System.getenv(envName);
+ }
+ if (System.getProperty(propertyName) != null) {
+ return System.getProperty(propertyName);
+ }
+
+ return getStringValue(propertyName, defaultValue);
+ }
+
+ public int getInt(ConfVars c) {
+ return getInt(c.name(), c.getVarName(), c.getIntValue());
+ }
+
+ public int getInt(String envName, String propertyName, int defaultValue) {
+ if (System.getenv(envName) != null) {
+ return Integer.parseInt(System.getenv(envName));
+ }
+
+ if (System.getProperty(propertyName) != null) {
+ return Integer.parseInt(System.getProperty(propertyName));
+ }
+ return getIntValue(propertyName, defaultValue);
+ }
+
+ public long getLong(ConfVars c) {
+ return getLong(c.name(), c.getVarName(), c.getLongValue());
+ }
+
+ public long getLong(String envName, String propertyName, long defaultValue) {
+ if (System.getenv(envName) != null) {
+ return Long.parseLong(System.getenv(envName));
+ }
+
+ if (System.getProperty(propertyName) != null) {
+ return Long.parseLong(System.getProperty(propertyName));
+ }
+ return getLongValue(propertyName, defaultValue);
+ }
+
+ public float getFloat(ConfVars c) {
+ return getFloat(c.name(), c.getVarName(), c.getFloatValue());
+ }
+
+ public float getFloat(String envName, String propertyName, float defaultValue) {
+ if (System.getenv(envName) != null) {
+ return Float.parseFloat(System.getenv(envName));
+ }
+ if (System.getProperty(propertyName) != null) {
+ return Float.parseFloat(System.getProperty(propertyName));
+ }
+ return getFloatValue(propertyName, defaultValue);
+ }
+
+ public boolean getBoolean(ConfVars c) {
+ return getBoolean(c.name(), c.getVarName(), c.getBooleanValue());
+ }
+
+ public boolean getBoolean(String envName, String propertyName, boolean defaultValue) {
+ if (System.getenv(envName) != null) {
+ return Boolean.parseBoolean(System.getenv(envName));
+ }
+
+ if (System.getProperty(propertyName) != null) {
+ return Boolean.parseBoolean(System.getProperty(propertyName));
+ }
+ return getBooleanValue(propertyName, defaultValue);
+ }
+
+ public boolean useSsl() {
+ return getBoolean(ConfVars.ZEPPELIN_SSL);
+ }
+
+ public int getServerSslPort() {
+ return getInt(ConfVars.ZEPPELIN_SSL_PORT);
+ }
+
+ public boolean useClientAuth() {
+ return getBoolean(ConfVars.ZEPPELIN_SSL_CLIENT_AUTH);
+ }
+
+ public String getServerAddress() {
+ return getString(ConfVars.ZEPPELIN_ADDR);
+ }
+
+ public int getServerPort() {
+ return getInt(ConfVars.ZEPPELIN_PORT);
+ }
+
+ public String getServerContextPath() {
+ return getString(ConfVars.ZEPPELIN_SERVER_CONTEXT_PATH);
+ }
+
+ public String getKeyStorePath() {
+ String path = getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_PATH);
+ if (path != null && path.startsWith("/") || isWindowsPath(path)) {
+ return path;
+ } else {
+ return getRelativeDir(
+ String.format("%s/%s",
+ getConfDir(),
+ path));
+ }
+ }
+
+ public String getKeyStoreType() {
+ return getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_TYPE);
+ }
+
+ public String getKeyStorePassword() {
+ return getString(ConfVars.ZEPPELIN_SSL_KEYSTORE_PASSWORD);
+ }
+
+ public String getKeyManagerPassword() {
+ String password = getString(ConfVars.ZEPPELIN_SSL_KEY_MANAGER_PASSWORD);
+ if (password == null) {
+ return getKeyStorePassword();
+ } else {
+ return password;
+ }
+ }
+
+ public String getTrustStorePath() {
+ String path = getString(ConfVars.ZEPPELIN_SSL_TRUSTSTORE_PATH);
+ if (path == null) {
+ path = getKeyStorePath();
+ }
+ if (path != null && path.startsWith("/") || isWindowsPath(path)) {
+ return path;
+ } else {
+ return getRelativeDir(
+ String.format("%s/%s",
+ getConfDir(),
+ path));
+ }
+ }
+
+ public String getTrustStoreType() {
+ String type = getString(ConfVars.ZEPPELIN_SSL_TRUSTSTORE_TYPE);
+ if (type == null) {
+ return getKeyStoreType();
+ } else {
+ return type;
+ }
+ }
+
+ public String getTrustStorePassword() {
+ String password = getString(ConfVars.ZEPPELIN_SSL_TRUSTSTORE_PASSWORD);
+ if (password == null) {
+ return getKeyStorePassword();
+ } else {
+ return password;
+ }
+ }
+
+ public String getNotebookDir() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_DIR);
+ }
+
+ public String getUser() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_USER);
+ }
+
+ public String getBucketName() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_BUCKET);
+ }
+
+ public String getEndpoint() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_ENDPOINT);
+ }
+
+ public String getS3KMSKeyID() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID);
+ }
+
+ public String getS3KMSKeyRegion() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION);
+ }
+
+ public String getS3EncryptionMaterialsProviderClass() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP);
+ }
+
+ public boolean isS3ServerSideEncryption() {
+ return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_S3_SSE);
+ }
+
+ public String getMongoUri() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_URI);
+ }
+
+ public String getMongoDatabase() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_DATABASE);
+ }
+
+ public String getMongoCollection() {
+ return getString(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_COLLECTION);
+ }
+
+ public boolean getMongoAutoimport() {
+ return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT);
+ }
+
+ public String getInterpreterListPath() {
+ return getRelativeDir(String.format("%s/interpreter-list", getConfDir()));
+ }
+
+ public String getInterpreterDir() {
+ return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_DIR);
+ }
+
+ public String getInterpreterJson() {
+ return getString(ConfVars.ZEPPELIN_INTERPRETER_JSON);
+ }
+
+ public String getInterpreterSettingPath() {
+ return getRelativeDir(String.format("%s/interpreter.json", getConfDir()));
+ }
+
+ public String getHeliumConfPath() {
+ return getRelativeDir(String.format("%s/helium.json", getConfDir()));
+ }
+
+ public String getHeliumRegistry() {
+ return getRelativeDir(ConfVars.ZEPPELIN_HELIUM_REGISTRY);
+ }
+
+ public String getHeliumNodeInstallerUrl() {
+ return getString(ConfVars.ZEPPELIN_HELIUM_NODE_INSTALLER_URL);
+ }
+
+ public String getHeliumNpmInstallerUrl() {
+ return getString(ConfVars.ZEPPELIN_HELIUM_NPM_INSTALLER_URL);
+ }
+
+ public String getHeliumYarnInstallerUrl() {
+ return getString(ConfVars.ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL);
+ }
+
+ public String getNotebookAuthorizationPath() {
+ return getRelativeDir(String.format("%s/notebook-authorization.json", getConfDir()));
+ }
+
+ public Boolean credentialsPersist() {
+ return getBoolean(ConfVars.ZEPPELIN_CREDENTIALS_PERSIST);
+ }
+
+ public String getCredentialsPath() {
+ return getRelativeDir(String.format("%s/credentials.json", getConfDir()));
+ }
+
+ public String getShiroPath() {
+ String shiroPath = getRelativeDir(String.format("%s/shiro.ini", getConfDir()));
+ return new File(shiroPath).exists() ? shiroPath : StringUtils.EMPTY;
+ }
+
+ public String getInterpreterRemoteRunnerPath() {
+ return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER);
+ }
+
+ public String getInterpreterLocalRepoPath() {
+ return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO);
+ }
+
+ public String getInterpreterMvnRepoPath() {
+ return getString(ConfVars.ZEPPELIN_INTERPRETER_DEP_MVNREPO);
+ }
+
+ public String getRelativeDir(ConfVars c) {
+ return getRelativeDir(getString(c));
+ }
+
+ public String getRelativeDir(String path) {
+ if (path != null && path.startsWith("/") || isWindowsPath(path)) {
+ return path;
+ } else {
+ return getString(ConfVars.ZEPPELIN_HOME) + "/" + path;
+ }
+ }
+
+ public boolean isWindowsPath(String path){
+ return path.matches("^[A-Za-z]:\\\\.*");
+ }
+
+ public boolean isAnonymousAllowed() {
+ return getBoolean(ConfVars.ZEPPELIN_ANONYMOUS_ALLOWED);
+ }
+
+ public boolean isNotebokPublic() {
+ return getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC);
+ }
+
+ public String getConfDir() {
+ return getString(ConfVars.ZEPPELIN_CONF_DIR);
+ }
+
+ public List<String> getAllowedOrigins()
+ {
+ if (getString(ConfVars.ZEPPELIN_ALLOWED_ORIGINS).isEmpty()) {
+ return Arrays.asList(new String[0]);
+ }
+
+ return Arrays.asList(getString(ConfVars.ZEPPELIN_ALLOWED_ORIGINS).toLowerCase().split(","));
+ }
+
+ public String getWebsocketMaxTextMessageSize() {
+ return getString(ConfVars.ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE);
+ }
+
+ public String getJettyName() {
+ return getString(ConfVars.ZEPPELIN_SERVER_JETTY_NAME);
+ }
+
+
+ public String getXFrameOptions() {
+ return getString(ConfVars.ZEPPELIN_SERVER_XFRAME_OPTIONS);
+ }
+
+ public String getXxssProtection() {
+ return getString(ConfVars.ZEPPELIN_SERVER_X_XSS_PROTECTION);
+ }
+
+ public String getStrictTransport() {
+ return getString(ConfVars.ZEPPELIN_SERVER_STRICT_TRANSPORT);
+ }
+
+
+ public Map<String, String> dumpConfigurations(ZeppelinConfiguration conf,
+ ConfigurationKeyPredicate predicate) {
+ Map<String, String> configurations = new HashMap<>();
+
+ for (ConfVars v : ConfVars.values()) {
+ String key = v.getVarName();
+
+ if (!predicate.apply(key)) {
+ continue;
+ }
+
+ ConfVars.VarType type = v.getType();
+ Object value = null;
+ if (type == ConfVars.VarType.BOOLEAN) {
+ value = conf.getBoolean(v);
+ } else if (type == ConfVars.VarType.LONG) {
+ value = conf.getLong(v);
+ } else if (type == ConfVars.VarType.INT) {
+ value = conf.getInt(v);
+ } else if (type == ConfVars.VarType.FLOAT) {
+ value = conf.getFloat(v);
+ } else if (type == ConfVars.VarType.STRING) {
+ value = conf.getString(v);
+ }
+
+ if (value != null) {
+ configurations.put(key, value.toString());
+ }
+ }
+ return configurations;
+ }
+
+ /**
+ * Predication whether key/value pair should be included or not
+ */
+ public interface ConfigurationKeyPredicate {
+ boolean apply(String key);
+ }
+
+ /**
+ * Wrapper class.
+ */
+ public static enum ConfVars {
+ ZEPPELIN_HOME("zeppelin.home", "./"),
+ ZEPPELIN_ADDR("zeppelin.server.addr", "0.0.0.0"),
+ ZEPPELIN_PORT("zeppelin.server.port", 8080),
+ ZEPPELIN_SERVER_CONTEXT_PATH("zeppelin.server.context.path", "/"),
+ ZEPPELIN_SSL("zeppelin.ssl", false),
+ ZEPPELIN_SSL_PORT("zeppelin.server.ssl.port", 8443),
+ ZEPPELIN_SSL_CLIENT_AUTH("zeppelin.ssl.client.auth", false),
+ ZEPPELIN_SSL_KEYSTORE_PATH("zeppelin.ssl.keystore.path", "keystore"),
+ ZEPPELIN_SSL_KEYSTORE_TYPE("zeppelin.ssl.keystore.type", "JKS"),
+ ZEPPELIN_SSL_KEYSTORE_PASSWORD("zeppelin.ssl.keystore.password", ""),
+ ZEPPELIN_SSL_KEY_MANAGER_PASSWORD("zeppelin.ssl.key.manager.password", null),
+ ZEPPELIN_SSL_TRUSTSTORE_PATH("zeppelin.ssl.truststore.path", null),
+ ZEPPELIN_SSL_TRUSTSTORE_TYPE("zeppelin.ssl.truststore.type", null),
+ ZEPPELIN_SSL_TRUSTSTORE_PASSWORD("zeppelin.ssl.truststore.password", null),
+ ZEPPELIN_WAR("zeppelin.war", "zeppelin-web/dist"),
+ ZEPPELIN_WAR_TEMPDIR("zeppelin.war.tempdir", "webapps"),
+ ZEPPELIN_INTERPRETERS("zeppelin.interpreters", "org.apache.zeppelin.spark.SparkInterpreter,"
+ + "org.apache.zeppelin.spark.PySparkInterpreter,"
+ + "org.apache.zeppelin.rinterpreter.RRepl,"
+ + "org.apache.zeppelin.rinterpreter.KnitR,"
+ + "org.apache.zeppelin.spark.SparkRInterpreter,"
+ + "org.apache.zeppelin.spark.SparkSqlInterpreter,"
+ + "org.apache.zeppelin.spark.DepInterpreter,"
+ + "org.apache.zeppelin.markdown.Markdown,"
+ + "org.apache.zeppelin.angular.AngularInterpreter,"
+ + "org.apache.zeppelin.shell.ShellInterpreter,"
+ + "org.apache.zeppelin.livy.LivySparkInterpreter,"
+ + "org.apache.zeppelin.livy.LivySparkSQLInterpreter,"
+ + "org.apache.zeppelin.livy.LivyPySparkInterpreter,"
+ + "org.apache.zeppelin.livy.LivyPySpark3Interpreter,"
+ + "org.apache.zeppelin.livy.LivySparkRInterpreter,"
+ + "org.apache.zeppelin.alluxio.AlluxioInterpreter,"
+ + "org.apache.zeppelin.file.HDFSFileInterpreter,"
+ + "org.apache.zeppelin.pig.PigInterpreter,"
+ + "org.apache.zeppelin.pig.PigQueryInterpreter,"
+ + "org.apache.zeppelin.flink.FlinkInterpreter,"
+ + "org.apache.zeppelin.python.PythonInterpreter,"
+ + "org.apache.zeppelin.python.PythonInterpreterPandasSql,"
+ + "org.apache.zeppelin.python.PythonCondaInterpreter,"
+ + "org.apache.zeppelin.python.PythonDockerInterpreter,"
+ + "org.apache.zeppelin.ignite.IgniteInterpreter,"
+ + "org.apache.zeppelin.ignite.IgniteSqlInterpreter,"
+ + "org.apache.zeppelin.lens.LensInterpreter,"
+ + "org.apache.zeppelin.cassandra.CassandraInterpreter,"
+ + "org.apache.zeppelin.geode.GeodeOqlInterpreter,"
+ + "org.apache.zeppelin.kylin.KylinInterpreter,"
+ + "org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,"
+ + "org.apache.zeppelin.scalding.ScaldingInterpreter,"
+ + "org.apache.zeppelin.jdbc.JDBCInterpreter,"
+ + "org.apache.zeppelin.hbase.HbaseInterpreter,"
+ + "org.apache.zeppelin.bigquery.BigQueryInterpreter,"
+ + "org.apache.zeppelin.beam.BeamInterpreter,"
+ + "org.apache.zeppelin.scio.ScioInterpreter,"
+ + "org.apache.zeppelin.groovy.GroovyInterpreter"
+ ),
+ ZEPPELIN_INTERPRETER_JSON("zeppelin.interpreter.setting", "interpreter-setting.json"),
+ ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"),
+ ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"),
+ ZEPPELIN_INTERPRETER_DEP_MVNREPO("zeppelin.interpreter.dep.mvnRepo",
+ "http://repo1.maven.org/maven2/"),
+ ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000),
+ ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10),
+ ZEPPELIN_INTERPRETER_GROUP_ORDER("zeppelin.interpreter.group.order", "spark,md,angular,sh,"
+ + "livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,"
+ + "scalding,jdbc,hbase,bigquery,beam,pig,scio,groovy"),
+ ZEPPELIN_INTERPRETER_OUTPUT_LIMIT("zeppelin.interpreter.output.limit", 1024 * 100),
+ ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"),
+ ZEPPELIN_NOTEBOOK_DIR("zeppelin.notebook.dir", "notebook"),
+ // use specified notebook (id) as homescreen
+ ZEPPELIN_NOTEBOOK_HOMESCREEN("zeppelin.notebook.homescreen", null),
+ // whether homescreen notebook will be hidden from notebook list or not
+ ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE("zeppelin.notebook.homescreen.hide", false),
+ ZEPPELIN_NOTEBOOK_S3_BUCKET("zeppelin.notebook.s3.bucket", "zeppelin"),
+ ZEPPELIN_NOTEBOOK_S3_ENDPOINT("zeppelin.notebook.s3.endpoint", "s3.amazonaws.com"),
+ ZEPPELIN_NOTEBOOK_S3_USER("zeppelin.notebook.s3.user", "user"),
+ ZEPPELIN_NOTEBOOK_S3_EMP("zeppelin.notebook.s3.encryptionMaterialsProvider", null),
+ ZEPPELIN_NOTEBOOK_S3_KMS_KEY_ID("zeppelin.notebook.s3.kmsKeyID", null),
+ ZEPPELIN_NOTEBOOK_S3_KMS_KEY_REGION("zeppelin.notebook.s3.kmsKeyRegion", null),
+ ZEPPELIN_NOTEBOOK_S3_SSE("zeppelin.notebook.s3.sse", false),
+ ZEPPELIN_NOTEBOOK_AZURE_CONNECTION_STRING("zeppelin.notebook.azure.connectionString", null),
+ ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"),
+ ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"),
+ ZEPPELIN_NOTEBOOK_MONGO_DATABASE("zeppelin.notebook.mongo.database", "zeppelin"),
+ ZEPPELIN_NOTEBOOK_MONGO_COLLECTION("zeppelin.notebook.mongo.collection", "notes"),
+ ZEPPELIN_NOTEBOOK_MONGO_URI("zeppelin.notebook.mongo.uri", "mongodb://localhost"),
+ ZEPPELIN_NOTEBOOK_MONGO_AUTOIMPORT("zeppelin.notebook.mongo.autoimport", false),
+ ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage",
+ "org.apache.zeppelin.notebook.repo.GitNotebookRepo"),
+ ZEPPELIN_NOTEBOOK_ONE_WAY_SYNC("zeppelin.notebook.one.way.sync", false),
+ // whether by default note is public or private
+ ZEPPELIN_NOTEBOOK_PUBLIC("zeppelin.notebook.public", true),
+ ZEPPELIN_INTERPRETER_REMOTE_RUNNER("zeppelin.interpreter.remoterunner",
+ System.getProperty("os.name")
+ .startsWith("Windows") ? "bin/interpreter.cmd" : "bin/interpreter.sh"),
+ // Decide when new note is created, interpreter settings will be binded automatically or not.
+ ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING("zeppelin.notebook.autoInterpreterBinding", true),
+ ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"),
+ ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"),
+ ZEPPELIN_HELIUM_REGISTRY("zeppelin.helium.registry", "helium," + HELIUM_PACKAGE_DEFAULT_URL),
+ ZEPPELIN_HELIUM_NODE_INSTALLER_URL("zeppelin.helium.node.installer.url",
+ "https://nodejs.org/dist/"),
+ ZEPPELIN_HELIUM_NPM_INSTALLER_URL("zeppelin.helium.npm.installer.url",
+ "http://registry.npmjs.org/"),
+ ZEPPELIN_HELIUM_YARNPKG_INSTALLER_URL("zeppelin.helium.yarnpkg.installer.url",
+ "https://github.com/yarnpkg/yarn/releases/download/"),
+ // Allows a way to specify a ',' separated list of allowed origins for rest and websockets
+ // i.e. http://localhost:8080
+ ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"),
+ ZEPPELIN_ANONYMOUS_ALLOWED("zeppelin.anonymous.allowed", true),
+ ZEPPELIN_CREDENTIALS_PERSIST("zeppelin.credentials.persist", true),
+ ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE("zeppelin.websocket.max.text.message.size", "1024000"),
+ ZEPPELIN_SERVER_DEFAULT_DIR_ALLOWED("zeppelin.server.default.dir.allowed", false),
+ ZEPPELIN_SERVER_XFRAME_OPTIONS("zeppelin.server.xframe.options", "SAMEORIGIN"),
+ ZEPPELIN_SERVER_JETTY_NAME("zeppelin.server.jetty.name", null),
+ ZEPPELIN_SERVER_STRICT_TRANSPORT("zeppelin.server.strict.transport", "max-age=631138519"),
+ ZEPPELIN_SERVER_X_XSS_PROTECTION("zeppelin.server.xxss.protection", "1"),
+
+ ZEPPELIN_HDFS_KEYTAB("zeppelin.hdfs.keytab", ""),
+ ZEPPELIN_HDFS_PRINCIPAL("zeppelin.hdfs.principal", "");
+
+ private String varName;
+ @SuppressWarnings("rawtypes")
+ private Class varClass;
+ private String stringValue;
+ private VarType type;
+ private int intValue;
+ private float floatValue;
+ private boolean booleanValue;
+ private long longValue;
+
+
+ ConfVars(String varName, String varValue) {
+ this.varName = varName;
+ this.varClass = String.class;
+ this.stringValue = varValue;
+ this.intValue = -1;
+ this.floatValue = -1;
+ this.longValue = -1;
+ this.booleanValue = false;
+ this.type = VarType.STRING;
+ }
+
+ ConfVars(String varName, int intValue) {
+ this.varName = varName;
+ this.varClass = Integer.class;
+ this.stringValue = null;
+ this.intValue = intValue;
+ this.floatValue = -1;
+ this.longValue = -1;
+ this.booleanValue = false;
+ this.type = VarType.INT;
+ }
+
+ ConfVars(String varName, long longValue) {
+ this.varName = varName;
+ this.varClass = Integer.class;
+ this.stringValue = null;
+ this.intValue = -1;
+ this.floatValue = -1;
+ this.longValue = longValue;
+ this.booleanValue = false;
+ this.type = VarType.LONG;
+ }
+
+ ConfVars(String varName, float floatValue) {
+ this.varName = varName;
+ this.varClass = Float.class;
+ this.stringValue = null;
+ this.intValue = -1;
+ this.longValue = -1;
+ this.floatValue = floatValue;
+ this.booleanValue = false;
+ this.type = VarType.FLOAT;
+ }
+
+ ConfVars(String varName, boolean booleanValue) {
+ this.varName = varName;
+ this.varClass = Boolean.class;
+ this.stringValue = null;
+ this.intValue = -1;
+ this.longValue = -1;
+ this.floatValue = -1;
+ this.booleanValue = booleanValue;
+ this.type = VarType.BOOLEAN;
+ }
+
+ public String getVarName() {
+ return varName;
+ }
+
+ @SuppressWarnings("rawtypes")
+ public Class getVarClass() {
+ return varClass;
+ }
+
+ public int getIntValue() {
+ return intValue;
+ }
+
+ public long getLongValue() {
+ return longValue;
+ }
+
+ public float getFloatValue() {
+ return floatValue;
+ }
+
+ public String getStringValue() {
+ return stringValue;
+ }
+
+ public boolean getBooleanValue() {
+ return booleanValue;
+ }
+
+ public VarType getType() {
+ return type;
+ }
+
+ enum VarType {
+ STRING {
+ @Override
+ void checkType(String value) throws Exception {}
+ },
+ INT {
+ @Override
+ void checkType(String value) throws Exception {
+ Integer.valueOf(value);
+ }
+ },
+ LONG {
+ @Override
+ void checkType(String value) throws Exception {
+ Long.valueOf(value);
+ }
+ },
+ FLOAT {
+ @Override
+ void checkType(String value) throws Exception {
+ Float.valueOf(value);
+ }
+ },
+ BOOLEAN {
+ @Override
+ void checkType(String value) throws Exception {
+ Boolean.valueOf(value);
+ }
+ };
+
+ boolean isType(String value) {
+ try {
+ checkType(value);
+ } catch (Exception e) {
+ LOG.error("Exception in ZeppelinConfiguration while isType", e);
+ return false;
+ }
+ return true;
+ }
+
+ String typeString() {
+ return name().toUpperCase();
+ }
+
+ abstract void checkType(String value) throws Exception;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
index 74506dd..05599a0 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/Interpreter.java
@@ -149,7 +149,6 @@ public abstract class Interpreter {
@ZeppelinApi
public Interpreter(Properties property) {
- logger.debug("Properties: {}", property);
this.property = property;
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
new file mode 100644
index 0000000..a2fdb04
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonatype.aether.RepositoryException;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * //TODO(zjffdu) considering to move to InterpreterSettingManager
+ *
+ * Manage interpreters.
+ */
+public class InterpreterFactory {
+ private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterFactory.class);
+
+ private final InterpreterSettingManager interpreterSettingManager;
+
+ public InterpreterFactory(InterpreterSettingManager interpreterSettingManager) {
+ this.interpreterSettingManager = interpreterSettingManager;
+ }
+
+ private InterpreterSetting getInterpreterSettingByGroup(List<InterpreterSetting> settings,
+ String group) {
+
+ Preconditions.checkNotNull(group, "group should be not null");
+ for (InterpreterSetting setting : settings) {
+ if (group.equals(setting.getName())) {
+ return setting;
+ }
+ }
+ return null;
+ }
+
+ public Interpreter getInterpreter(String user, String noteId, String replName) {
+ List<InterpreterSetting> settings = interpreterSettingManager.getInterpreterSettings(noteId);
+ InterpreterSetting setting;
+ Interpreter interpreter;
+
+ if (settings == null || settings.size() == 0) {
+ LOGGER.error("No interpreter is binded to this note: " + noteId);
+ return null;
+ }
+
+ if (StringUtils.isBlank(replName)) {
+ // Get the default interpreter of the first interpreter binding
+ InterpreterSetting defaultSetting = settings.get(0);
+ return defaultSetting.getDefaultInterpreter(user, noteId);
+ }
+
+ String[] replNameSplit = replName.split("\\.");
+ if (replNameSplit.length == 2) {
+ String group = replNameSplit[0];
+ String name = replNameSplit[1];
+ setting = getInterpreterSettingByGroup(settings, group);
+ if (null != setting) {
+ interpreter = setting.getInterpreter(user, noteId, name);
+ if (null != interpreter) {
+ return interpreter;
+ }
+ }
+ throw new InterpreterException(replName + " interpreter not found");
+
+ } else {
+ // first assume replName is 'name' of interpreter. ('groupName' is ommitted)
+ // search 'name' from first (default) interpreter group
+ // TODO(jl): Handle with noteId to support defaultInterpreter per note.
+ setting = settings.get(0);
+ interpreter = setting.getInterpreter(user, noteId, replName);
+
+ if (null != interpreter) {
+ return interpreter;
+ }
+
+ // next, assume replName is 'group' of interpreter ('name' is ommitted)
+ // search interpreter group and return first interpreter.
+ setting = getInterpreterSettingByGroup(settings, replName);
+
+ if (null != setting) {
+ return setting.getDefaultInterpreter(user, noteId);
+ }
+
+ // Support the legacy way to use it
+ for (InterpreterSetting s : settings) {
+ if (s.getGroup().equals(replName)) {
+ return setting.getDefaultInterpreter(user, noteId);
+ }
+ }
+ }
+ //TODO(zjffdu) throw InterpreterException instead of return null
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
index 5cbab6b..ae7ad3a 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java
@@ -17,14 +17,7 @@
package org.apache.zeppelin.interpreter;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.google.common.annotations.VisibleForTesting;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.apache.zeppelin.resource.ResourcePool;
@@ -33,91 +26,93 @@ import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+
/**
- * InterpreterGroup is list of interpreters in the same interpreter group.
- * For example spark, pyspark, sql interpreters are in the same 'spark' group
- * and InterpreterGroup will have reference to these all interpreters.
+ * InterpreterGroup is collections of interpreter sessions.
+ * One session could include multiple interpreters.
+ * For example spark, pyspark, sql interpreters are in the same 'spark' interpreter session.
*
* Remember, list of interpreters are dedicated to a session. Session could be shared across user
* or notes, so the sessionId could be user or noteId or their combination.
- * So InterpreterGroup internally manages map of [interpreterSessionKey(noteId, user, or
+ * So InterpreterGroup internally manages map of [sessionId(noteId, user, or
* their combination), list of interpreters]
*
- * A InterpreterGroup runs on interpreter process.
- * And unit of interpreter instantiate, restart, bind, unbind.
+ * A InterpreterGroup runs both in zeppelin server process and interpreter process.
*/
-public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter>> {
- String id;
+public class InterpreterGroup {
private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterGroup.class);
- AngularObjectRegistry angularObjectRegistry;
- InterpreterHookRegistry hookRegistry;
- RemoteInterpreterProcess remoteInterpreterProcess; // attached remote interpreter process
- ResourcePool resourcePool;
- boolean angularRegistryPushed = false;
-
- // map [notebook session, Interpreters in the group], to support per note session interpreters
- //Map<String, List<Interpreter>> interpreters = new ConcurrentHashMap<String,
- // List<Interpreter>>();
-
- private static final Map<String, InterpreterGroup> allInterpreterGroups =
- new ConcurrentHashMap<>();
-
- public static InterpreterGroup getByInterpreterGroupId(String id) {
- return allInterpreterGroups.get(id);
- }
+ private String id;
+ // sessionId --> interpreters
+ private Map<String, List<Interpreter>> sessions = new ConcurrentHashMap();
+ private InterpreterSetting interpreterSetting;
+ private AngularObjectRegistry angularObjectRegistry;
+ private InterpreterHookRegistry hookRegistry;
+ private RemoteInterpreterProcess remoteInterpreterProcess; // attached remote interpreter process
+ private ResourcePool resourcePool;
+ private boolean angularRegistryPushed = false;
- public static Collection<InterpreterGroup> getAll() {
- return new LinkedList(allInterpreterGroups.values());
+ /**
+ * Create InterpreterGroup with given id, used in InterpreterProcess
+ * @param id
+ */
+ public InterpreterGroup(String id) {
+ this.id = id;
}
/**
- * Create InterpreterGroup with given id
+ * Create InterpreterGroup with given id and interpreterSetting, used in ZeppelinServer
* @param id
+ * @param interpreterSetting
*/
- public InterpreterGroup(String id) {
+ InterpreterGroup(String id, InterpreterSetting interpreterSetting) {
this.id = id;
- allInterpreterGroups.put(id, this);
+ this.interpreterSetting = interpreterSetting;
}
/**
* Create InterpreterGroup with autogenerated id
*/
+ @VisibleForTesting
public InterpreterGroup() {
- getId();
- allInterpreterGroups.put(id, this);
+ this.id = generateId();
}
private static String generateId() {
- return "InterpreterGroup_" + System.currentTimeMillis() + "_"
- + new Random().nextInt();
+ return "InterpreterGroup_" + System.currentTimeMillis() + "_" + new Random().nextInt();
}
public String getId() {
- synchronized (this) {
- if (id == null) {
- id = generateId();
- }
- return id;
- }
+ return this.id;
}
- /**
- * Get combined property of all interpreters in this group
- * @return
- */
- public Properties getProperty() {
- Properties p = new Properties();
+ //TODO(zjffdu) change it to getSession. For now just keep this method to reduce code change
+ public synchronized List<Interpreter> get(String sessionId) {
+ return sessions.get(sessionId);
+ }
- for (List<Interpreter> intpGroupForASession : this.values()) {
- for (Interpreter intp : intpGroupForASession) {
- p.putAll(intp.getProperty());
- }
- // it's okay to break here while every List<Interpreters> will have the same property set
- break;
+ //TODO(zjffdu) change it to addSession. For now just keep this method to reduce code change
+ public synchronized void put(String sessionId, List<Interpreter> interpreters) {
+ this.sessions.put(sessionId, interpreters);
+ }
+
+ public synchronized void addInterpreterToSession(Interpreter interpreter, String sessionId) {
+ LOGGER.debug("Add Interpreter {} to session {}", interpreter.getClassName(), sessionId);
+ List<Interpreter> interpreters = get(sessionId);
+ if (interpreters == null) {
+ interpreters = new ArrayList<>();
}
- return p;
+ interpreters.add(interpreter);
+ put(sessionId, interpreters);
+ }
+
+ //TODO(zjffdu) rename it to a more proper name.
+ //For now just keep this method to reduce code change
+ public Collection<List<Interpreter>> values() {
+ return sessions.values();
}
public AngularObjectRegistry getAngularObjectRegistry() {
@@ -136,128 +131,69 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
this.hookRegistry = hookRegistry;
}
- public RemoteInterpreterProcess getRemoteInterpreterProcess() {
+ public InterpreterSetting getInterpreterSetting() {
+ return interpreterSetting;
+ }
+
+ public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess() {
+ if (remoteInterpreterProcess == null) {
+ LOGGER.info("Create InterperterProcess for InterpreterGroup: " + getId());
+ remoteInterpreterProcess = interpreterSetting.createInterpreterProcess();
+ }
return remoteInterpreterProcess;
}
- public void setRemoteInterpreterProcess(RemoteInterpreterProcess remoteInterpreterProcess) {
- this.remoteInterpreterProcess = remoteInterpreterProcess;
+ public RemoteInterpreterProcess getRemoteInterpreterProcess() {
+ return remoteInterpreterProcess;
}
+
/**
* Close all interpreter instances in this group
*/
- public void close() {
- LOGGER.info("Close interpreter group " + getId());
- List<Interpreter> intpToClose = new LinkedList<>();
- for (List<Interpreter> intpGroupForSession : this.values()) {
- intpToClose.addAll(intpGroupForSession);
+ public synchronized void close() {
+ LOGGER.info("Close InterpreterGroup: " + id);
+ for (String sessionId : sessions.keySet()) {
+ close(sessionId);
}
- close(intpToClose);
-
- // make sure remote interpreter process terminates
- if (remoteInterpreterProcess != null) {
- while (remoteInterpreterProcess.referenceCount() > 0) {
- remoteInterpreterProcess.dereference();
- }
- remoteInterpreterProcess = null;
- }
- allInterpreterGroups.remove(id);
}
/**
- * Close all interpreter instances in this group for the session
+ * Close all interpreter instances in this session
* @param sessionId
*/
- public void close(String sessionId) {
- LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionId);
- final List<Interpreter> intpForSession = this.get(sessionId);
-
- close(intpForSession);
- }
-
- private void close(final Collection<Interpreter> intpToClose) {
- close(null, null, null, intpToClose);
+ public synchronized void close(String sessionId) {
+ LOGGER.info("Close Session: " + sessionId);
+ close(sessions.remove(sessionId));
+ //TODO(zjffdu) whether close InterpreterGroup if there's no session left in Zeppelin Server
+ if (sessions.isEmpty() && interpreterSetting != null) {
+ LOGGER.info("Remove this InterpreterGroup {} as all the sessions are closed", id);
+ interpreterSetting.removeInterpreterGroup(id);
+ if (remoteInterpreterProcess != null) {
+ LOGGER.info("Kill RemoteInterpreterProcess");
+ remoteInterpreterProcess.stop();
+ remoteInterpreterProcess = null;
+ }
+ }
}
- public void close(final Map<String, InterpreterGroup> interpreterGroupRef,
- final String processKey, final String sessionKey) {
- LOGGER.info("Close interpreter group " + getId() + " for session: " + sessionKey);
- close(interpreterGroupRef, processKey, sessionKey, this.get(sessionKey));
+ public int getSessionNum() {
+ return sessions.size();
}
- private void close(final Map<String, InterpreterGroup> interpreterGroupRef,
- final String processKey, final String sessionKey, final Collection<Interpreter> intpToClose) {
- if (intpToClose == null) {
+ private void close(Collection<Interpreter> interpreters) {
+ if (interpreters == null) {
return;
}
- Thread t = new Thread() {
- public void run() {
- for (Interpreter interpreter : intpToClose) {
- Scheduler scheduler = interpreter.getScheduler();
- interpreter.close();
-
- if (null != scheduler) {
- SchedulerFactory.singleton().removeScheduler(scheduler.getName());
- }
- }
-
- if (remoteInterpreterProcess != null) {
- //TODO(jl): Because interpreter.close() runs as a seprate thread, we cannot guarantee
- // refernceCount is a proper value. And as the same reason, we must not call
- // remoteInterpreterProcess.dereference twice - this method also be called by
- // interpreter.close().
-
- // remoteInterpreterProcess.dereference();
- if (remoteInterpreterProcess.referenceCount() <= 0) {
- remoteInterpreterProcess = null;
- allInterpreterGroups.remove(id);
- }
- }
-
- // TODO(jl): While closing interpreters in a same session, we should remove after all
- // interpreters are removed. OMG. It's too dirty!!
- if (null != interpreterGroupRef && null != processKey && null != sessionKey) {
- InterpreterGroup interpreterGroup = interpreterGroupRef.get(processKey);
- if (1 == interpreterGroup.size() && interpreterGroup.containsKey(sessionKey)) {
- interpreterGroupRef.remove(processKey);
- } else {
- interpreterGroup.remove(sessionKey);
- }
- }
- }
- };
-
- t.start();
- try {
- t.join();
- } catch (InterruptedException e) {
- LOGGER.error("Can't close interpreter: {}", getId(), e);
- }
-
- }
-
- /**
- * Close all interpreter instances in this group
- */
- public void shutdown() {
- LOGGER.info("Close interpreter group " + getId());
-
- // make sure remote interpreter process terminates
- if (remoteInterpreterProcess != null) {
- while (remoteInterpreterProcess.referenceCount() > 0) {
- remoteInterpreterProcess.dereference();
+ for (Interpreter interpreter : interpreters) {
+ Scheduler scheduler = interpreter.getScheduler();
+ interpreter.close();
+ //TODO(zjffdu) move the close of schedule to Interpreter
+ if (null != scheduler) {
+ SchedulerFactory.singleton().removeScheduler(scheduler.getName());
}
- remoteInterpreterProcess = null;
}
- allInterpreterGroups.remove(id);
-
- List<Interpreter> intpToClose = new LinkedList<>();
- for (List<Interpreter> intpGroupForSession : this.values()) {
- intpToClose.addAll(intpGroupForSession);
- }
- close(intpToClose);
}
public void setResourcePool(ResourcePool resourcePool) {
@@ -275,4 +211,22 @@ public class InterpreterGroup extends ConcurrentHashMap<String, List<Interpreter
public void setAngularRegistryPushed(boolean angularRegistryPushed) {
this.angularRegistryPushed = angularRegistryPushed;
}
+
+ public synchronized List<Interpreter> getOrCreateSession(String user, String sessionId) {
+ if (sessions.containsKey(sessionId)) {
+ return sessions.get(sessionId);
+ } else {
+ List<Interpreter> interpreters = interpreterSetting.createInterpreters(user, sessionId);
+ for (Interpreter interpreter : interpreters) {
+ interpreter.setInterpreterGroup(this);
+ }
+ LOGGER.info("Create Session {} in InterpreterGroup {} for user {}", sessionId, id, user);
+ sessions.put(sessionId, interpreters);
+ return interpreters;
+ }
+ }
+
+ public boolean isEmpty() {
+ return sessions.isEmpty();
+ }
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java
new file mode 100644
index 0000000..fd632ce
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.util.Map;
+
+/**
+ * Information of interpreters in this interpreter setting.
+ * this will be serialized for conf/interpreter.json and REST api response.
+ */
+public class InterpreterInfo {
+ private String name;
+ @SerializedName("class") private String className;
+ private boolean defaultInterpreter = false;
+ private Map<String, Object> editor;
+
+ public InterpreterInfo(String className, String name, boolean defaultInterpreter,
+ Map<String, Object> editor) {
+ this.className = className;
+ this.name = name;
+ this.defaultInterpreter = defaultInterpreter;
+ this.editor = editor;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getClassName() {
+ return className;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ boolean isDefaultInterpreter() {
+ return defaultInterpreter;
+ }
+
+ public Map<String, Object> getEditor() {
+ return editor;
+ }
+
+ public void setEditor(Map<String, Object> editor) {
+ this.editor = editor;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof InterpreterInfo)) {
+ return false;
+ }
+ InterpreterInfo other = (InterpreterInfo) obj;
+
+ boolean sameName =
+ null == getName() ? null == other.getName() : getName().equals(other.getName());
+ boolean sameClassName = null == getClassName() ?
+ null == other.getClassName() :
+ getClassName().equals(other.getClassName());
+ boolean sameIsDefaultInterpreter = defaultInterpreter == other.isDefaultInterpreter();
+
+ return sameName && sameClassName && sameIsDefaultInterpreter;
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
new file mode 100644
index 0000000..4ccc262
--- /dev/null
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.internal.StringMap;
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.sonatype.aether.repository.RemoteRepository;
+
+import java.io.BufferedReader;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.PosixFilePermission;
+import java.util.*;
+
+import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
+import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
+
+/**
+ *
+ */
+public class InterpreterInfoSaving {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterInfoSaving.class);
+ private static final Gson gson = new GsonBuilder().setPrettyPrinting().create();
+
+ public Map<String, InterpreterSetting> interpreterSettings = new HashMap<>();
+ public Map<String, List<String>> interpreterBindings = new HashMap<>();
+ public List<RemoteRepository> interpreterRepositories = new ArrayList<>();
+
+ public static InterpreterInfoSaving loadFromFile(Path file) throws IOException {
+ LOGGER.info("Load interpreter setting from file: " + file);
+ InterpreterInfoSaving infoSaving = null;
+ try (BufferedReader json = Files.newBufferedReader(file, StandardCharsets.UTF_8)) {
+ JsonParser jsonParser = new JsonParser();
+ JsonObject jsonObject = jsonParser.parse(json).getAsJsonObject();
+ infoSaving = InterpreterInfoSaving.fromJson(jsonObject.toString());
+
+ if (infoSaving != null && infoSaving.interpreterSettings != null) {
+ for (InterpreterSetting interpreterSetting : infoSaving.interpreterSettings.values()) {
+ // Always use separate interpreter process
+ // While we decided to turn this feature on always (without providing
+ // enable/disable option on GUI).
+ // previously created setting should turn this feature on here.
+ interpreterSetting.getOption().setRemote(true);
+ interpreterSetting.convertPermissionsFromUsersToOwners(
+ jsonObject.getAsJsonObject("interpreterSettings")
+ .getAsJsonObject(interpreterSetting.getId()));
+ }
+ }
+ }
+ return infoSaving == null ? new InterpreterInfoSaving() : infoSaving;
+ }
+
+ public void saveToFile(Path file) throws IOException {
+ if (!Files.exists(file)) {
+ Files.createFile(file);
+ try {
+ Set<PosixFilePermission> permissions = EnumSet.of(OWNER_READ, OWNER_WRITE);
+ Files.setPosixFilePermissions(file, permissions);
+ } catch (UnsupportedOperationException e) {
+ // File system does not support Posix file permissions (likely windows) - continue anyway.
+ LOGGER.warn("unable to setPosixFilePermissions on '{}'.", file);
+ };
+ }
+ LOGGER.info("Save Interpreter Settings to " + file);
+ IOUtils.write(this.toJson(), new FileOutputStream(file.toFile()));
+ }
+
+ public String toJson() {
+ return gson.toJson(this);
+ }
+
+ public static InterpreterInfoSaving fromJson(String json) {
+ return gson.fromJson(json, InterpreterInfoSaving.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java
index 0bb3d42..ac40de3 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterProperty.java
@@ -34,6 +34,7 @@ public class InterpreterProperty {
public InterpreterProperty(String name, Object value) {
this.name = name;
this.value = value;
+ this.type = "textarea";
}
public String getName() {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java
index 020564b..ae7bb88 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterRunner.java
@@ -1,5 +1,6 @@
package org.apache.zeppelin.interpreter;
+import com.google.common.annotations.VisibleForTesting;
import com.google.gson.annotations.SerializedName;
/**
@@ -12,6 +13,16 @@ public class InterpreterRunner {
@SerializedName("win")
private String winPath;
+ public InterpreterRunner() {
+
+ }
+
+ @VisibleForTesting
+ public InterpreterRunner(String linuxPath, String winPath) {
+ this.linuxPath = linuxPath;
+ this.winPath = winPath;
+ }
+
public String getPath() {
return System.getProperty("os.name").startsWith("Windows") ? winPath : linuxPath;
}