You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by zj...@apache.org on 2017/11/10 07:56:50 UTC
zeppelin git commit: ZEPPELIN-3039. Interpreter logs are mixed
together
Repository: zeppelin
Updated Branches:
refs/heads/master 23c5cac82 -> 3b1a03f38
ZEPPELIN-3039. Interpreter logs are mixed together
### What is this PR for?
This is a bug introduced by ZEPPELIN-2685. Wrong interpreter setting name is passed. This PR fix this issue and also made some code refactoring. After this PR, the log file name is
${ZEPPELIN_LOG_DIR}/zeppelin-interpreter-${INTERPRETER_SETTING_NAME}-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.log
### What type of PR is it?
[Bug Fix]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-3039
### How should this be tested?
Create 2 jdbc interpreters: hive & hive2,and launch them both. There would be 2 log files generated.
* zeppelin-interpreter-hive-jzhang-HW12527.log
* zeppelin-interpreter-hive2-jzhang-HW12527.log
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zj...@apache.org>
Closes #2656 from zjffdu/ZEPPELIN-3039 and squashes the following commits:
2745151 [Jeff Zhang] ZEPPELIN-3039. Interpreter logs are mixed together
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/3b1a03f3
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/3b1a03f3
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/3b1a03f3
Branch: refs/heads/master
Commit: 3b1a03f380437b22884f93dbdb3ee2b9c116fbc2
Parents: 23c5cac
Author: Jeff Zhang <zj...@apache.org>
Authored: Wed Nov 8 10:56:30 2017 +0800
Committer: Jeff Zhang <zj...@apache.org>
Committed: Fri Nov 10 15:56:15 2017 +0800
----------------------------------------------------------------------
bin/interpreter.sh | 10 +++----
.../launcher/InterpreterLaunchContext.java | 28 ++++++++++++--------
.../interpreter/InterpreterSetting.java | 2 +-
.../launcher/ShellScriptLauncher.java | 10 +++----
.../remote/RemoteInterpreterManagedProcess.java | 14 +++++-----
.../launcher/ShellScriptLauncherTest.java | 6 ++---
.../launcher/SparkInterpreterLauncherTest.java | 20 +++++++-------
7 files changed, 45 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3b1a03f3/bin/interpreter.sh
----------------------------------------------------------------------
diff --git a/bin/interpreter.sh b/bin/interpreter.sh
index d27b076..4e983ec 100755
--- a/bin/interpreter.sh
+++ b/bin/interpreter.sh
@@ -54,7 +54,7 @@ while getopts "hc:p:d:l:v:u:g:" o; do
fi
;;
g)
- INTERPRETER_GROUP_NAME=${OPTARG}
+ INTERPRETER_SETTING_NAME=${OPTARG}
;;
esac
done
@@ -91,14 +91,12 @@ ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer
INTERPRETER_ID=$(basename "${INTERPRETER_DIR}")
ZEPPELIN_PID="${ZEPPELIN_PID_DIR}/zeppelin-interpreter-${INTERPRETER_ID}-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.pid"
-ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-interpreter-"
-if [[ ! -z "$INTERPRETER_GROUP_NAME" ]]; then
- ZEPPELIN_LOGFILE+="${INTERPRETER_GROUP_NAME}-"
-fi
+ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/zeppelin-interpreter-${INTERPRETER_SETTING_NAME}-"
+
if [[ ! -z "$ZEPPELIN_IMPERSONATE_USER" ]]; then
ZEPPELIN_LOGFILE+="${ZEPPELIN_IMPERSONATE_USER}-"
fi
-ZEPPELIN_LOGFILE+="${INTERPRETER_ID}-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.log"
+ZEPPELIN_LOGFILE+="${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.log"
JAVA_INTP_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}"
if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3b1a03f3/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLaunchContext.java
----------------------------------------------------------------------
diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLaunchContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLaunchContext.java
index db8f8dd..9e25355 100644
--- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLaunchContext.java
+++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/launcher/InterpreterLaunchContext.java
@@ -17,7 +17,6 @@
package org.apache.zeppelin.interpreter.launcher;
-import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterRunner;
@@ -31,19 +30,22 @@ public class InterpreterLaunchContext {
private Properties properties;
private InterpreterOption option;
private InterpreterRunner runner;
- private String interpreterGroupId;
- private String interpreterGroupName;
+ private String interpreterSettingId;
+ private String interpreterSettingGroup;
+ private String interpreterSettingName;
public InterpreterLaunchContext(Properties properties,
InterpreterOption option,
InterpreterRunner runner,
- String interpreterGroupId,
- String interpreterGroupName) {
+ String interpreterSettingId,
+ String interpreterSettingGroup,
+ String interpreterSettingName) {
this.properties = properties;
this.option = option;
this.runner = runner;
- this.interpreterGroupId = interpreterGroupId;
- this.interpreterGroupName = interpreterGroupName;
+ this.interpreterSettingId = interpreterSettingId;
+ this.interpreterSettingGroup = interpreterSettingGroup;
+ this.interpreterSettingName = interpreterSettingName;
}
public Properties getProperties() {
@@ -58,11 +60,15 @@ public class InterpreterLaunchContext {
return runner;
}
- public String getInterpreterGroupId() {
- return interpreterGroupId;
+ public String getInterpreterSettingId() {
+ return interpreterSettingId;
}
- public String getInterpreterGroupName() {
- return interpreterGroupName;
+ public String getInterpreterSettingGroup() {
+ return interpreterSettingGroup;
+ }
+
+ public String getInterpreterSettingName() {
+ return interpreterSettingName;
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3b1a03f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
index 944672c..26fcd8e 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
@@ -671,7 +671,7 @@ public class InterpreterSetting {
createLauncher();
}
InterpreterLaunchContext launchContext = new
- InterpreterLaunchContext(getJavaProperties(), option, interpreterRunner, id, group);
+ InterpreterLaunchContext(getJavaProperties(), option, interpreterRunner, id, group, name);
RemoteInterpreterProcess process = (RemoteInterpreterProcess) launcher.launch(launchContext);
process.setRemoteInterpreterEventPoller(
new RemoteInterpreterEventPoller(remoteInterpreterProcessListener, appEventListener));
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3b1a03f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java
index f419967..0966ec5 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncher.java
@@ -29,7 +29,6 @@ import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Map;
-import java.util.Properties;
/**
* Interpreter Launcher which use shell script to launch the interpreter process.
@@ -45,11 +44,12 @@ public class ShellScriptLauncher extends InterpreterLauncher {
@Override
public InterpreterClient launch(InterpreterLaunchContext context) {
- LOGGER.info("Launching Interpreter: " + context.getInterpreterGroupName());
+ LOGGER.info("Launching Interpreter: " + context.getInterpreterSettingGroup());
this.properties = context.getProperties();
InterpreterOption option = context.getOption();
InterpreterRunner runner = context.getRunner();
- String groupName = context.getInterpreterGroupName();
+ String groupName = context.getInterpreterSettingGroup();
+ String name = context.getInterpreterSettingName();
int connectTimeout =
zConf.getInt(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT);
@@ -61,12 +61,12 @@ public class ShellScriptLauncher extends InterpreterLauncher {
} else {
// create new remote process
String localRepoPath = zConf.getInterpreterLocalRepoPath() + "/"
- + context.getInterpreterGroupId();
+ + context.getInterpreterSettingId();
return new RemoteInterpreterManagedProcess(
runner != null ? runner.getPath() : zConf.getInterpreterRemoteRunnerPath(),
zConf.getCallbackPortRange(),
zConf.getInterpreterDir() + "/" + groupName, localRepoPath,
- buildEnvFromProperties(), connectTimeout, groupName);
+ buildEnvFromProperties(), connectTimeout, name);
}
}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3b1a03f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
index 6e26e58..9f8f346 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java
@@ -31,8 +31,6 @@ import org.apache.thrift.server.TServer;
import org.apache.thrift.server.TThreadPoolServer;
import org.apache.thrift.transport.TServerSocket;
import org.apache.thrift.transport.TTransportException;
-import org.apache.zeppelin.helium.ApplicationEventListener;
-import org.apache.zeppelin.interpreter.InterpreterException;
import org.apache.zeppelin.interpreter.thrift.CallbackInfo;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterCallbackService;
import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService;
@@ -63,7 +61,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
private int port = -1;
private final String interpreterDir;
private final String localRepoDir;
- private final String interpreterGroupName;
+ private final String interpreterSettingName;
private Map<String, String> env;
@@ -74,14 +72,14 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
String localRepoDir,
Map<String, String> env,
int connectTimeout,
- String interpreterGroupName) {
+ String interpreterSettingName) {
super(connectTimeout);
this.interpreterRunner = intpRunner;
this.portRange = portRange;
this.env = env;
this.interpreterDir = intpDir;
this.localRepoDir = localRepoDir;
- this.interpreterGroupName = interpreterGroupName;
+ this.interpreterSettingName = interpreterSettingName;
}
@Override
@@ -167,7 +165,7 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
cmdLine.addArgument("-l", false);
cmdLine.addArgument(localRepoDir, false);
cmdLine.addArgument("-g", false);
- cmdLine.addArgument(interpreterGroupName, false);
+ cmdLine.addArgument(interpreterSettingName, false);
executor = new DefaultExecutor();
@@ -263,8 +261,8 @@ public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess
}
@VisibleForTesting
- public String getInterpreterGroupName() {
- return interpreterGroupName;
+ public String getInterpreterSettingName() {
+ return interpreterSettingName;
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3b1a03f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java
index 9708ee7..0c7f4ba 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/ShellScriptLauncherTest.java
@@ -18,10 +18,8 @@
package org.apache.zeppelin.interpreter.launcher;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
-import org.apache.zeppelin.interpreter.InterpreterContext;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess;
-import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess;
import org.junit.Test;
import java.util.Properties;
@@ -39,11 +37,11 @@ public class ShellScriptLauncherTest {
properties.setProperty("ENV_1", "VALUE_1");
properties.setProperty("property_1", "value_1");
InterpreterOption option = new InterpreterOption();
- InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "groupName");
+ InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "groupName", "name");
InterpreterClient client = launcher.launch(context);
assertTrue( client instanceof RemoteInterpreterManagedProcess);
RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client;
- assertEquals("groupName", interpreterProcess.getInterpreterGroupName());
+ assertEquals("name", interpreterProcess.getInterpreterSettingName());
assertEquals(".//interpreter/groupName", interpreterProcess.getInterpreterDir());
assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3b1a03f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
----------------------------------------------------------------------
diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
index 45e009f..b788ebd 100644
--- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
+++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
@@ -41,11 +41,11 @@ public class SparkInterpreterLauncherTest {
properties.setProperty("spark.jars", "jar_1");
InterpreterOption option = new InterpreterOption();
- InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark");
+ InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark", "spark");
InterpreterClient client = launcher.launch(context);
assertTrue( client instanceof RemoteInterpreterManagedProcess);
RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client;
- assertEquals("spark", interpreterProcess.getInterpreterGroupName());
+ assertEquals("spark", interpreterProcess.getInterpreterSettingName());
assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir());
assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
@@ -66,11 +66,11 @@ public class SparkInterpreterLauncherTest {
properties.setProperty("spark.jars", "jar_1");
InterpreterOption option = new InterpreterOption();
- InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark");
+ InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark", "spark");
InterpreterClient client = launcher.launch(context);
assertTrue( client instanceof RemoteInterpreterManagedProcess);
RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client;
- assertEquals("spark", interpreterProcess.getInterpreterGroupName());
+ assertEquals("spark", interpreterProcess.getInterpreterSettingName());
assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir());
assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
@@ -92,11 +92,11 @@ public class SparkInterpreterLauncherTest {
properties.setProperty("spark.jars", "jar_1");
InterpreterOption option = new InterpreterOption();
- InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark");
+ InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark", "spark");
InterpreterClient client = launcher.launch(context);
assertTrue( client instanceof RemoteInterpreterManagedProcess);
RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client;
- assertEquals("spark", interpreterProcess.getInterpreterGroupName());
+ assertEquals("spark", interpreterProcess.getInterpreterSettingName());
assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir());
assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
@@ -117,11 +117,11 @@ public class SparkInterpreterLauncherTest {
properties.setProperty("spark.jars", "jar_1");
InterpreterOption option = new InterpreterOption();
- InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark");
+ InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark", "spark");
InterpreterClient client = launcher.launch(context);
assertTrue( client instanceof RemoteInterpreterManagedProcess);
RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client;
- assertEquals("spark", interpreterProcess.getInterpreterGroupName());
+ assertEquals("spark", interpreterProcess.getInterpreterSettingName());
assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir());
assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());
@@ -144,11 +144,11 @@ public class SparkInterpreterLauncherTest {
properties.setProperty("spark.jars", "jar_1");
InterpreterOption option = new InterpreterOption();
- InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark");
+ InterpreterLaunchContext context = new InterpreterLaunchContext(properties, option, null, "groupId", "spark", "spark");
InterpreterClient client = launcher.launch(context);
assertTrue( client instanceof RemoteInterpreterManagedProcess);
RemoteInterpreterManagedProcess interpreterProcess = (RemoteInterpreterManagedProcess) client;
- assertEquals("spark", interpreterProcess.getInterpreterGroupName());
+ assertEquals("spark", interpreterProcess.getInterpreterSettingName());
assertEquals(".//interpreter/spark", interpreterProcess.getInterpreterDir());
assertEquals(".//local-repo/groupId", interpreterProcess.getLocalRepoDir());
assertEquals(zConf.getInterpreterRemoteRunnerPath(), interpreterProcess.getInterpreterRunner());