You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by pr...@apache.org on 2017/03/15 08:48:05 UTC
zeppelin git commit: [ZEPPELIN-2149] Each interpreter should have a
seperate log file
Repository: zeppelin
Updated Branches:
refs/heads/master 45e80fd60 -> 1ec6e1265
[ZEPPELIN-2149] Each interpreter should have a seperate log file
### What is this PR for?
In the current implementation, both spark and spark2 uses spark interpreter, both phoenix and hive use jdbc interpreter log file.
Each interpreter should have separate log file. The same log file for multiple interpreters confuses the user and complicated debugging.
### What type of PR is it?
[Improvement]
### What is the Jira issue?
* [ZEPPELIN-2149](https://issues.apache.org/jira/browse/ZEPPELIN-2149)
### How should this be tested?
Create multiple interpreters (say shell interpreter) with different names (say sh1, sh2, etc.).
Now on running these different interpreters should log to different files.
### Questions:
* Does the licenses files need update? N/A
* Is there breaking changes for older versions? N/A
* Does this needs documentation? N/A
Author: Prabhjyot Singh <pr...@gmail.com>
Closes #2107 from prabhjyotsingh/ZEPPELIN-2149 and squashes the following commits:
4566037 [Prabhjyot Singh] rename interpreterName to interpreterGroupName
a989f5e [Prabhjyot Singh] Merge remote-tracking branch 'origin/master' into ZEPPELIN-2149
0cf514f [Prabhjyot Singh] fix tests
934f274 [Prabhjyot Singh] ZEPPELIN-2149: Each interpreter should have a seperate log file
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/1ec6e126
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/1ec6e126
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/1ec6e126
Branch: refs/heads/master
Commit: 1ec6e1265338976d2bb928a8450de5fd8cd0f56d
Parents: 45e80fd
Author: Prabhjyot Singh <pr...@gmail.com>
Authored: Fri Mar 10 12:42:03 2017 +0530
Committer: Prabhjyot Singh <pr...@gmail.com>
Committed: Wed Mar 15 14:17:57 2017 +0530
----------------------------------------------------------------------
bin/interpreter.sh | 10 ++++++++--
.../zeppelin/interpreter/remote/RemoteInterpreter.java | 7 ++++---
.../remote/RemoteInterpreterManagedProcess.java | 12 +++++++++---
.../remote/RemoteInterpreterProcessTest.java | 9 +++++----
.../apache/zeppelin/interpreter/InterpreterFactory.java | 3 ++-
.../zeppelin/interpreter/InterpreterFactoryTest.java | 4 ++--
6 files changed, 30 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1ec6e126/bin/interpreter.sh
----------------------------------------------------------------------
diff --git a/bin/interpreter.sh b/bin/interpreter.sh
index 0dbae1f..5565ea9 100755
--- a/bin/interpreter.sh
+++ b/bin/interpreter.sh
@@ -20,10 +20,10 @@ bin=$(dirname "${BASH_SOURCE-$0}")
bin=$(cd "${bin}">/dev/null; pwd)
function usage() {
- echo "usage) $0 -p <port> -d <interpreter dir to load> -l <local interpreter repo dir to load>"
+ echo "usage) $0 -p <port> -d <interpreter dir to load> -l <local interpreter repo dir to load> -g <interpreter group name>"
}
-while getopts "hp:d:l:v:u:" o; do
+while getopts "hp:d:l:v:u:n:" o; do
case ${o} in
h)
usage
@@ -50,6 +50,9 @@ while getopts "hp:d:l:v:u:" o; do
ZEPPELIN_IMPERSONATE_RUN_CMD=$(eval "echo ${ZEPPELIN_IMPERSONATE_CMD} ")
fi
;;
+ g)
+ INTERPRETER_GROUP_NAME=${OPTARG}
+ ;;
esac
done
@@ -86,6 +89,9 @@ ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer
INTERPRETER_ID=$(basename "${INTERPRETER_DIR}")
ZEPPELIN_PID="${ZEPPELIN_PID_DIR}/zeppelin-interpreter-${INTERPRETER_ID}-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.pid"
ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-interpreter-"
+if [[ ! -z "$INTERPRETER_GROUP_NAME" ]]; then
+ ZEPPELIN_LOGFILE+="${INTERPRETER_GROUP_NAME}-"
+fi
if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
ZEPPELIN_LOGFILE+="${ZEPPELIN_IMPERSONATE_USER}-"
fi
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1ec6e126/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
index edd97f4..cafd6d7 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java
@@ -26,7 +26,6 @@ import org.apache.zeppelin.display.GUI;
import org.apache.zeppelin.helium.ApplicationEventListener;
import org.apache.zeppelin.display.Input;
import org.apache.zeppelin.interpreter.*;
-import org.apache.zeppelin.interpreter.InterpreterResult.Type;
import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult;
@@ -64,6 +63,7 @@ public class RemoteInterpreter extends Interpreter {
private String userName;
private Boolean isUserImpersonate;
private int outputLimit = Constants.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT;
+ private String interpreterGroupName;
/**
* Remote interpreter and manage interpreter process
@@ -72,7 +72,7 @@ public class RemoteInterpreter extends Interpreter {
String interpreterRunner, String interpreterPath, String localRepoPath, int connectTimeout,
int maxPoolSize, RemoteInterpreterProcessListener remoteInterpreterProcessListener,
ApplicationEventListener appListener, String userName, Boolean isUserImpersonate,
- int outputLimit) {
+ int outputLimit, String interpreterGroupName) {
super(property);
this.sessionKey = sessionKey;
this.className = className;
@@ -88,6 +88,7 @@ public class RemoteInterpreter extends Interpreter {
this.userName = userName;
this.isUserImpersonate = isUserImpersonate;
this.outputLimit = outputLimit;
+ this.interpreterGroupName = interpreterGroupName;
}
@@ -185,7 +186,7 @@ public class RemoteInterpreter extends Interpreter {
// create new remote process
remoteProcess = new RemoteInterpreterManagedProcess(
interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout,
- remoteInterpreterProcessListener, applicationEventListener);
+ remoteInterpreterProcessListener, applicationEventListener, interpreterGroupName);
}
intpGroup.setRemoteInterpreterProcess(remoteProcess);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1ec6e126/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
index f5d73ed..23bfb84 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
@@ -44,6 +44,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
private int port = -1;
private final String interpreterDir;
private final String localRepoDir;
+ private final String interpreterGroupName;
private Map<String, String> env;
@@ -54,14 +55,15 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
Map<String, String> env,
int connectTimeout,
RemoteInterpreterProcessListener listener,
- ApplicationEventListener appListener) {
+ ApplicationEventListener appListener,
+ String interpreterGroupName) {
super(new RemoteInterpreterEventPoller(listener, appListener),
connectTimeout);
this.interpreterRunner = intpRunner;
this.env = env;
this.interpreterDir = intpDir;
this.localRepoDir = localRepoDir;
-
+ this.interpreterGroupName = interpreterGroupName;
}
RemoteInterpreterManagedProcess(String intpRunner,
@@ -69,13 +71,15 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
String localRepoDir,
Map<String, String> env,
RemoteInterpreterEventPoller remoteInterpreterEventPoller,
- int connectTimeout) {
+ int connectTimeout,
+ String interpreterGroupName) {
super(remoteInterpreterEventPoller,
connectTimeout);
this.interpreterRunner = intpRunner;
this.env = env;
this.interpreterDir = intpDir;
this.localRepoDir = localRepoDir;
+ this.interpreterGroupName = interpreterGroupName;
}
@Override
@@ -108,6 +112,8 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
}
cmdLine.addArgument("-l", false);
cmdLine.addArgument(localRepoDir, false);
+ cmdLine.addArgument("-n", false);
+ cmdLine.addArgument(interpreterGroupName, false);
executor = new DefaultExecutor();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1ec6e126/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
index 39a17ae..b85d7ef 100644
--- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
+++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java
@@ -44,7 +44,7 @@ public class RemoteInterpreterProcessTest {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
- 10 * 1000, null, null);
+ 10 * 1000, null, null,"fakeName");
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
assertEquals(1, rip.reference(intpGroup, "anonymous", false));
@@ -61,7 +61,7 @@ public class RemoteInterpreterProcessTest {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap<String, String>(),
- mock(RemoteInterpreterEventPoller.class), 10 * 1000);
+ mock(RemoteInterpreterEventPoller.class), 10 * 1000, "fakeName");
rip.reference(intpGroup, "anonymous", false);
assertEquals(0, rip.getNumActiveClient());
assertEquals(0, rip.getNumIdleClient());
@@ -104,7 +104,8 @@ public class RemoteInterpreterProcessTest {
"fakeRepo",
new HashMap<String, String>(),
mock(RemoteInterpreterEventPoller.class)
- , 10 * 1000);
+ , 10 * 1000,
+ "fakeName");
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
assertEquals(1, rip.reference(intpGroup, "anonymous", false));
@@ -117,7 +118,7 @@ public class RemoteInterpreterProcessTest {
InterpreterGroup intpGroup = new InterpreterGroup();
RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess(
"echo hello_world", "nonexists", "fakeRepo", new HashMap<String, String>(),
- 10 * 1000, null, null);
+ 10 * 1000, null, null, "fakeName");
assertFalse(rip.isRunning());
assertEquals(0, rip.referenceCount());
try {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1ec6e126/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
index 121c6e8..0df8f67 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
@@ -302,6 +302,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterSettingId;
int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE);
String interpreterRunnerPath;
+ String interpreterGroupName = interpreterSettingManager.get(interpreterSettingId).getName();
if (null != interpreterRunner) {
interpreterRunnerPath = interpreterRunner.getPath();
Path p = Paths.get(interpreterRunnerPath);
@@ -317,7 +318,7 @@ public class InterpreterFactory implements InterpreterGroupFactory {
new RemoteInterpreter(property, interpreterSessionKey, className,
interpreterRunnerPath, interpreterPath, localRepoPath, connectTimeout, maxPoolSize,
remoteInterpreterProcessListener, appEventListener, userName, isUserImpersonate,
- conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT));
+ conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT), interpreterGroupName);
remoteInterpreter.addEnv(env);
return new LazyOpenInterpreter(remoteInterpreter);
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1ec6e126/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
index 70c7a6b..711f957 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java
@@ -473,13 +473,13 @@ public class InterpreterFactoryTest {
InterpreterRunner mockInterpreterRunner = mock(InterpreterRunner.class);
String testInterpreterRunner = "relativePath.sh";
when(mockInterpreterRunner.getPath()).thenReturn(testInterpreterRunner); // This test only for Linux
- Interpreter i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), "settingId", "userName", false, mockInterpreterRunner);
+ Interpreter i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), interpreterSettingManager.get().get(0).getId(), "userName", false, mockInterpreterRunner);
String interpreterRunner = ((RemoteInterpreter) ((LazyOpenInterpreter) i).getInnerInterpreter()).getInterpreterRunner();
assertNotEquals(interpreterRunner, testInterpreterRunner);
testInterpreterRunner = "/AbsolutePath.sh";
when(mockInterpreterRunner.getPath()).thenReturn(testInterpreterRunner);
- i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), "settingId", "userName", false, mockInterpreterRunner);
+ i = factory.createRemoteRepl("path1", "sessionKey", "className", new Properties(), interpreterSettingManager.get().get(0).getId(), "userName", false, mockInterpreterRunner);
interpreterRunner = ((RemoteInterpreter) ((LazyOpenInterpreter) i).getInnerInterpreter()).getInterpreterRunner();
assertEquals(interpreterRunner, testInterpreterRunner);
}