You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2015/06/24 22:33:22 UTC
incubator-zeppelin git commit: Configurable interpreter process
connection waiting timeout and larger default value.
Repository: incubator-zeppelin
Updated Branches:
refs/heads/master 971e763cd -> 8045d7082
Configurable interpreter process connection waiting timeout and larger default value.
https://issues.apache.org/jira/browse/ZEPPELIN-124
Increase connection timeout and make it configurable
Author: Lee moon soo <mo...@apache.org>
Closes #116 from Leemoonsoo/ZEPPELIN-124 and squashes the following commits:
7e70289 [Lee moon soo] Increase interpreter connection timeout and make it configurable
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/8045d708
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/8045d708
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/8045d708
Branch: refs/heads/master
Commit: 8045d70829c1f9e5c0a151f620edda19425d727f
Parents: 971e763
Author: Lee moon soo <mo...@apache.org>
Authored: Mon Jun 22 16:38:17 2015 -0700
Committer: Lee moon soo <mo...@apache.org>
Committed: Wed Jun 24 13:33:15 2015 -0700
----------------------------------------------------------------------
conf/zeppelin-site.xml.template | 7 +++++
.../interpreter/remote/RemoteInterpreter.java | 11 +++++--
.../remote/RemoteInterpreterProcess.java | 13 +++++---
.../remote/RemoteAngularObjectTest.java | 3 +-
.../remote/RemoteInterpreterProcessTest.java | 4 +--
.../remote/RemoteInterpreterTest.java | 33 +++++++++++++-------
.../zeppelin/scheduler/RemoteSchedulerTest.java | 3 +-
.../zeppelin/conf/ZeppelinConfiguration.java | 1 +
.../interpreter/InterpreterFactory.java | 4 ++-
9 files changed, 55 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/8045d708/conf/zeppelin-site.xml.template
----------------------------------------------------------------------
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index b467a82..8f8bf06 100644
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -71,6 +71,13 @@
</property>
<property>
+ <name>zeppelin.interpreter.connect.timeout</name>
+ <value>30000</value>
+ <description>Interpreter process connect timeout in msec.</description>
+</property>
+
+
+<property>
<name>zeppelin.ssl</name>
<value>false</value>
<description>Should SSL be used by the servers?</description>
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/8045d708/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
index 1637e9c..22818fc 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
@@ -58,11 +58,13 @@ public class RemoteInterpreter extends Interpreter {
= new HashMap<String, RemoteInterpreterProcess>();
private InterpreterContextRunnerPool interpreterContextRunnerPool;
+ private int connectTimeout;
public RemoteInterpreter(Properties property,
String className,
String interpreterRunner,
- String interpreterPath) {
+ String interpreterPath,
+ int connectTimeout) {
super(property);
this.className = className;
@@ -71,18 +73,21 @@ public class RemoteInterpreter extends Interpreter {
this.interpreterPath = interpreterPath;
env = new HashMap<String, String>();
interpreterContextRunnerPool = new InterpreterContextRunnerPool();
+ this.connectTimeout = connectTimeout;
}
public RemoteInterpreter(Properties property,
String className,
String interpreterRunner,
String interpreterPath,
- Map<String, String> env) {
+ Map<String, String> env,
+ int connectTimeout) {
super(property);
this.className = className;
this.interpreterRunner = interpreterRunner;
this.interpreterPath = interpreterPath;
this.env = env;
+ this.connectTimeout = connectTimeout;
}
@Override
@@ -333,7 +338,7 @@ public class RemoteInterpreter extends Interpreter {
|| (!intpProcess.isRunning() && intpProcess.getPort() == -1)) {
interpreterGroupReference.put(getInterpreterGroupKey(interpreterGroup),
new RemoteInterpreterProcess(interpreterRunner,
- interpreterPath, env, interpreterContextRunnerPool));
+ interpreterPath, env, interpreterContextRunnerPool, connectTimeout));
logger.info("setInterpreterGroup = "
+ getInterpreterGroupKey(interpreterGroup) + " class=" + className
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/8045d708/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
index 61fcb70..5dd2a65 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java
@@ -50,26 +50,29 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
private Map<String, String> env;
private final RemoteInterpreterEventPoller remoteInterpreterEventPoller;
private final InterpreterContextRunnerPool interpreterContextRunnerPool;
+ private int connectTimeout;
public RemoteInterpreterProcess(String intpRunner,
String intpDir,
Map<String, String> env,
- InterpreterContextRunnerPool interpreterContextRunnerPool) {
+ InterpreterContextRunnerPool interpreterContextRunnerPool, int connectTimeout) {
this(intpRunner, intpDir, env, interpreterContextRunnerPool,
- new RemoteInterpreterEventPoller());
+ new RemoteInterpreterEventPoller(), connectTimeout);
}
RemoteInterpreterProcess(String intpRunner,
String intpDir,
Map<String, String> env,
InterpreterContextRunnerPool interpreterContextRunnerPool,
- RemoteInterpreterEventPoller remoteInterpreterEventPoller) {
+ RemoteInterpreterEventPoller remoteInterpreterEventPoller,
+ int connectTimeout) {
this.interpreterRunner = intpRunner;
this.interpreterDir = intpDir;
this.env = env;
this.interpreterContextRunnerPool = interpreterContextRunnerPool;
referenceCount = new AtomicInteger(0);
this.remoteInterpreterEventPoller = remoteInterpreterEventPoller;
+ this.connectTimeout = connectTimeout;
}
@@ -113,7 +116,7 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
long startTime = System.currentTimeMillis();
- while (System.currentTimeMillis() - startTime < 5 * 1000) {
+ while (System.currentTimeMillis() - startTime < connectTimeout) {
if (RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", port)) {
break;
} else {
@@ -123,7 +126,7 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler {
}
}
}
-
+
clientPool = new GenericObjectPool<Client>(new ClientFactory("localhost", port));
remoteInterpreterEventPoller.setInterpreterGroup(interpreterGroup);
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/8045d708/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
index d4909e3..e6da1ec 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java
@@ -68,7 +68,8 @@ public class RemoteAngularObjectTest implements AngularObjectRegistryListener {
MockInterpreterAngular.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
- env
+ env,
+ 10 * 1000
);
intpGroup.add(intp);
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/8045d708/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
index 4ea9a30..0043272 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
@@ -33,7 +33,7 @@ public class RemoteInterpreterProcessTest {
public void testStartStop() {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterProcess rip = new RemoteInterpreterProcess("../bin/interpreter.sh", "nonexists", new HashMap<String, String>(),
- new InterpreterContextRunnerPool());
+ new InterpreterContextRunnerPool(), 10 * 1000);
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
assertEquals(1, rip.reference(intpGroup));
@@ -49,7 +49,7 @@ public class RemoteInterpreterProcessTest {
public void testClientFactory() throws Exception {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterProcess rip = new RemoteInterpreterProcess("../bin/interpreter.sh", "nonexists", new HashMap<String, String>(),
- new InterpreterContextRunnerPool(), mock(RemoteInterpreterEventPoller.class));
+ new InterpreterContextRunnerPool(), mock(RemoteInterpreterEventPoller.class), 10 * 1000);
rip.reference(intpGroup);
assertEquals(0, rip.getNumActiveClient());
assertEquals(0, rip.getNumIdleClient());
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/8045d708/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
index 4d5636d..b49f86d 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java
@@ -74,7 +74,8 @@ public class RemoteInterpreterTest {
MockInterpreterA.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
- env
+ env,
+ 10 * 1000
);
intpGroup.add(intpA);
@@ -85,7 +86,8 @@ public class RemoteInterpreterTest {
MockInterpreterB.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
- env
+ env,
+ 10 * 1000
);
intpGroup.add(intpB);
@@ -135,7 +137,8 @@ public class RemoteInterpreterTest {
MockInterpreterA.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
- env
+ env,
+ 10 * 1000
);
intpGroup.add(intpA);
@@ -146,7 +149,8 @@ public class RemoteInterpreterTest {
MockInterpreterB.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
- env
+ env,
+ 10 * 1000
);
intpGroup.add(intpB);
@@ -197,7 +201,8 @@ public class RemoteInterpreterTest {
MockInterpreterA.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
- env
+ env,
+ 10 * 1000
);
intpGroup.add(intpA);
@@ -208,7 +213,8 @@ public class RemoteInterpreterTest {
MockInterpreterB.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
- env
+ env,
+ 10 * 1000
);
intpGroup.add(intpB);
@@ -311,7 +317,8 @@ public class RemoteInterpreterTest {
MockInterpreterA.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
- env
+ env,
+ 10 * 1000
);
intpGroup.add(intpA);
@@ -390,7 +397,8 @@ public class RemoteInterpreterTest {
MockInterpreterA.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
- env
+ env,
+ 10 * 1000
);
intpGroup.add(intpA);
@@ -468,7 +476,8 @@ public class RemoteInterpreterTest {
MockInterpreterA.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
- env
+ env,
+ 10 * 1000
);
intpA.setInterpreterGroup(intpGroup);
@@ -489,7 +498,8 @@ public class RemoteInterpreterTest {
MockInterpreterA.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
- env
+ env,
+ 10 * 1000
);
intpA.setInterpreterGroup(intpGroup);
@@ -513,7 +523,8 @@ public class RemoteInterpreterTest {
MockInterpreterA.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
- env
+ env,
+ 10 * 1000
);
intpA.setInterpreterGroup(intpGroup);
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/8045d708/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
index bb0fb80..2a1075a 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java
@@ -62,7 +62,8 @@ public class RemoteSchedulerTest {
MockInterpreterA.class.getName(),
new File("../bin/interpreter.sh").getAbsolutePath(),
"fake",
- env
+ env,
+ 10 * 1000
);
intpGroup.add(intpA);
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/8045d708/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index 6bc8a6c..9f05742 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -401,6 +401,7 @@ public class ZeppelinConfiguration extends XMLConfiguration {
+ "org.apache.zeppelin.ignite.IgniteInterpreter,"
+ "org.apache.zeppelin.ignite.IgniteSqlInterpreter"),
ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"),
+ ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000),
ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"),
ZEPPELIN_NOTEBOOK_DIR("zeppelin.notebook.dir", "notebook"),
ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage", VFSNotebookRepo.class.getName()),
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/8045d708/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
index c8fc485..77df7c5 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
@@ -621,8 +621,10 @@ public class InterpreterFactory {
private Interpreter createRemoteRepl(String interpreterPath, String className,
Properties property) {
+ int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
LazyOpenInterpreter intp = new LazyOpenInterpreter(new RemoteInterpreter(
- property, className, conf.getInterpreterRemoteRunnerPath(), interpreterPath));
+ property, className, conf.getInterpreterRemoteRunnerPath(),
+ interpreterPath, connectTimeout));
return intp;
}