You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2018/06/17 03:04:57 UTC
zeppelin git commit: ZEPPELIN-3352. Improve RemoteInterpreterProcess
creation timeout mechanism
Repository: zeppelin
Updated Branches:
refs/heads/master 5c7d65b18 -> 191324a67
ZEPPELIN-3352. Improve RemoteInterpreterProcess creation timeout mechanism
### What is this PR for?
This PR allow interpreter override `zeppelin.interpreter.connect.timeout` in zeppelin-site.xml. So that each interpreter can specify its own timeout threshold without restarting the zeppelin server in case he hit the timeout issue. Besides that I change the default timeout to be 60 seconds. This is what zeppelin did before. Although the default timeout is 30 seconds before, but RemoteInterpreterManagedProcess will still wait for 2 times of timeout threshold. So I directly change it to 60 secodns in this PR.
### What type of PR is it?
[ Improvement ]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-3352
### How should this be tested?
* CI pass
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versons? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #3021 from zjffdu/ZEPPELIN-3352 and squashes the following commits:
9426d8e41 [Jeff Zhang] ZEPPELIN-3352. Improve RemoteInterpreterProcess creation timeout mechanism
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/191324a6
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/191324a6
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/191324a6
Branch: refs/heads/master
Commit: 191324a67aa26601731bbb801fdb790282466bd3
Parents: 5c7d65b
Author: Jeff Zhang <zj...@apache.org>
Authored: Fri Jun 15 11:32:19 2018 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Sun Jun 17 11:04:53 2018 +0800
----------------------------------------------------------------------
.../zeppelin/conf/ZeppelinConfiguration.java | 2 +-
.../interpreter/launcher/InterpreterClient.java | 4 +++-
.../launcher/InterpreterLauncher.java | 13 ++++++++++-
.../launcher/ShellScriptLauncher.java | 3 +--
.../remote/RemoteInterpreterManagedProcess.java | 10 +++++---
.../launcher/ShellScriptLauncherTest.java | 24 ++++++++++++++++++++
.../launcher/SparkInterpreterLauncherTest.java | 22 ++++++++++++++++++
7 files changed, 70 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/191324a6/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index dc46dd0..635adbf 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -728,7 +728,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"),
ZEPPELIN_INTERPRETER_DEP_MVNREPO("zeppelin.interpreter.dep.mvnRepo",
"http://repo1.maven.org/maven2/"),
- ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000),
+ ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 60000),
ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10),
ZEPPELIN_INTERPRETER_GROUP_ORDER("zeppelin.interpreter.group.order", "spark,md,angular,sh,"
+ "livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/191324a6/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java
index 26da270..bfd3e44 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterClient.java
@@ -17,6 +17,8 @@
package org.apache.zeppelin.interpreter.launcher;
+import java.io.IOException;
+
/**
* Interface to InterpreterClient which is created by InterpreterLauncher. This is the component
* that is used to for the communication from zeppelin-server process to zeppelin interpreter
@@ -26,7 +28,7 @@ public interface InterpreterClient {
String getInterpreterSettingName();
- void start(String userName);
+ void start(String userName) throws IOException;
void stop();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/191324a6/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java
index 1cee20e..30cf995 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLauncher.java
@@ -37,5 +37,16 @@ public abstract class InterpreterLauncher {
this.recoveryStorage = recoveryStorage;
}
- public abstract InterpreterClient launch(InterpreterLaunchContext context) throws IOException;
+ protected int getConnectTimeout() {
+ int connectTimeout =
+ zConf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
+ if (properties.containsKey(
+ ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName())) {
+ connectTimeout = Integer.parseInt(properties.getProperty(
+ ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName()));
+ }
+ return connectTimeout;
+ }
+
+ public abstract InterpreterClient launch(InterpreterLaunchContext context) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/191324a6/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java
index b271d9b..18a6dde 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java
@@ -51,8 +51,7 @@ public class ShellScriptLauncher extends InterpreterLauncher {
InterpreterRunner runner = context.getRunner();
String groupName = context.getInterpreterSettingGroup();
String name = context.getInterpreterSettingName();
- int connectTimeout =
- zConf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
+ int connectTimeout = getConnectTimeout();
if (option.isExistingProcess()) {
return new RemoteInterpreterRunningProcess(
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/191324a6/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
index 5160522..db6d263 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
@@ -97,7 +97,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
}
@Override
- public void start(String userName) {
+ public void start(String userName) throws IOException {
// start server process
CommandLine cmdLine = CommandLine.parse(interpreterRunner);
cmdLine.addArgument("-d", false);
@@ -143,11 +143,15 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
try {
synchronized (running) {
if (!running.get()) {
- running.wait(getConnectTimeout() * 2);
+ running.wait(getConnectTimeout());
}
}
if (!running.get()) {
- throw new RuntimeException(new String(cmdOut.toByteArray()));
+ throw new IOException(new String(
+ String.format("Interpreter Process creation is time out in %d seconds",
+ getConnectTimeout()/1000) + "\n" + "You can increase timeout threshold via " +
+ "setting zeppelin.interpreter.connect.timeout of this interpreter.\n" +
+ cmdOut.toString()));
}
} catch (InterruptedException e) {
logger.error("Remote interpreter is not accessible");
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/191324a6/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java
index 6301d0d..ace3f31 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java
@@ -53,10 +53,34 @@ public class ShellScriptLauncherTest {
assertEquals("name", interpreterProcess.getInterpreterSettingName());
assertEquals(".//interpreter/groupName", interpreterProcess.getInterpreterDir());
assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
+ assertEquals(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getIntValue(),
+ interpreterProcess.getConnectTimeout());
assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
assertEquals(1, interpreterProcess.getEnv().size());
assertEquals("VALUE_1", interpreterProcess.getEnv().get("ENV_1"));
assertEquals(true, interpreterProcess.isUserImpersonated());
}
+ @Test
+ public void testConnectTimeOut() throws IOException {
+ ZeppelinConfiguration zConf = new ZeppelinConfiguration();
+ ShellScriptLauncher launcher = new ShellScriptLauncher(zConf, null);
+ Properties properties = new Properties();
+ properties.setProperty(
+ ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "10000");
+ InterpreterOption option = new InterpreterOption();
+ option.setUserImpersonate(true);
+ InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "groupName", "name", 0, "host");
+ InterpreterClient client = launcher.launch(context);
+ assertTrue( client instanceof RemoteInterpreterManagedProcess);
+ RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client;
+ assertEquals("name", interpreterProcess.getInterpreterSettingName());
+ assertEquals(".//interpreter/groupName", interpreterProcess.getInterpreterDir());
+ assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
+ assertEquals(10000, interpreterProcess.getConnectTimeout());
+ assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
+ assertEquals(0, interpreterProcess.getEnv().size());
+ assertEquals(true, interpreterProcess.isUserImpersonated());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/191324a6/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
index 2e4b165..7aab026 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
@@ -38,6 +38,28 @@ public class SparkInterpreterLauncherTest {
}
@Test
+ public void testConnectTimeOut() throws IOException {
+ ZeppelinConfiguration zConf = new ZeppelinConfiguration();
+ SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null);
+ Properties properties = new Properties();
+ properties.setProperty(
+ ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT.getVarName(), "10000");
+ InterpreterOption option = new InterpreterOption();
+ option.setUserImpersonate(true);
+ InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "user1", "intpGroupId", "groupId", "groupName", "name", 0, "host");
+ InterpreterClient client = launcher.launch(context);
+ assertTrue( client instanceof RemoteInterpreterManagedProcess);
+ RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client;
+ assertEquals("name", interpreterProcess.getInterpreterSettingName());
+ assertEquals(".//interpreter/groupName", interpreterProcess.getInterpreterDir());
+ assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
+ assertEquals(10000, interpreterProcess.getConnectTimeout());
+ assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
+ assertTrue(interpreterProcess.getEnv().size() >= 2);
+ assertEquals(true, interpreterProcess.isUserImpersonated());
+ }
+
+ @Test
public void testLocalMode() throws IOException {
ZeppelinConfiguration zConf = new ZeppelinConfiguration();
SparkInterpreterLauncher launcher = new SparkInterpreterLauncher(zConf, null);