You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by pr...@apache.org on 2016/08/01 06:12:09 UTC
zeppelin git commit: ZEPPELIN-1175. AM log is not available for
yarn-client mode
Repository: zeppelin
Updated Branches:
refs/heads/master a2a229d16 -> 1e478b229
ZEPPELIN-1175. AM log is not available for yarn-client mode
### What is this PR for?
For now, we share the same class path for zeppelin server and remote interpreter process. The cause the issue that AM log is not available for yarn-client mode because the yarn app also use the `ZEPPELIN_HOME/conf/log4j.properties` which is only for zeppelin server. So this PR just distinguish the CLASSPATH of zeppelin server and remote interpreter process. I use `ZEPPELIN_INTP_CLASSPATH` to represent the classpath of remote interpreter process and won't include `ZEPPELIN_HOME/conf/log4j.properties` in `ZEPPELIN_INTP_CLASSPATH`.
### What type of PR is it?
[Improvement]
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-1175
### How should this be tested?
Tested manually.
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? Yes, if user put custom config file (hive-site.xml) under ZEPPELIN_HOME/conf, it won't take effect after this PR
* Does this needs documentation? Yes
Author: Jeff Zhang <zj...@apache.org>
Closes #1228 from zjffdu/ZEPPELIN-1175 and squashes the following commits:
0973477 [Jeff Zhang] ZEPPELIN-1175. AM log is not available for yarn-client mode
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/1e478b22
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/1e478b22
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/1e478b22
Branch: refs/heads/master
Commit: 1e478b2293ba29f77b03eed81bfb2f88028d8fa9
Parents: a2a229d
Author: Jeff Zhang <zj...@apache.org>
Authored: Tue Jul 26 10:41:50 2016 +0800
Committer: Prabhjyot Singh <pr...@gmail.com>
Committed: Mon Aug 1 11:42:03 2016 +0530
----------------------------------------------------------------------
bin/common.sh | 13 +++++++++++++
bin/interpreter.sh | 36 ++++++++++++++++++------------------
2 files changed, 31 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e478b22/bin/common.sh
----------------------------------------------------------------------
diff --git a/bin/common.sh b/bin/common.sh
index a70be7c..592aa1c 100644
--- a/bin/common.sh
+++ b/bin/common.sh
@@ -74,6 +74,13 @@ function addEachJarInDirRecursive(){
fi
}
+function addEachJarInDirRecursiveForIntp(){
+ if [[ -d "${1}" ]]; then
+ for jar in $(find -L "${1}" -type f -name '*jar'); do
+ ZEPPELIN_INTP_CLASSPATH="$jar:$ZEPPELIN_INTP_CLASSPATH"
+ done
+ fi
+}
function addJarInDir(){
if [[ -d "${1}" ]]; then
@@ -81,6 +88,12 @@ function addJarInDir(){
fi
}
+function addJarInDirForIntp() {
+ if [[ -d "${1}" ]]; then
+ ZEPPELIN_INTP_CLASSPATH="${1}/*:${ZEPPELIN_INTP_CLASSPATH}"
+ fi
+}
+
ZEPPELIN_COMMANDLINE_MAIN=org.apache.zeppelin.utils.CommandLineUtils
function getZeppelinVersion(){
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e478b22/bin/interpreter.sh
----------------------------------------------------------------------
diff --git a/bin/interpreter.sh b/bin/interpreter.sh
index 69c94f6..38d0f69 100755
--- a/bin/interpreter.sh
+++ b/bin/interpreter.sh
@@ -53,18 +53,18 @@ fi
. "${bin}/common.sh"
-ZEPPELIN_CLASSPATH+=":${ZEPPELIN_CONF_DIR}"
+ZEPPELIN_INTP_CLASSPATH=""
# construct classpath
if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then
- ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes"
+ ZEPPELIN_INTP_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes"
else
ZEPPELIN_INTERPRETER_JAR="$(ls ${ZEPPELIN_HOME}/lib/zeppelin-interpreter*.jar)"
- ZEPPELIN_CLASSPATH+=":${ZEPPELIN_INTERPRETER_JAR}"
+ ZEPPELIN_INTP_CLASSPATH+=":${ZEPPELIN_INTERPRETER_JAR}"
fi
-addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
-addJarInDir "${INTERPRETER_DIR}"
+addJarInDirForIntp "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
+addJarInDirForIntp "${INTERPRETER_DIR}"
HOSTNAME=$(hostname)
ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer
@@ -85,7 +85,7 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
export SPARK_SUBMIT="${SPARK_HOME}/bin/spark-submit"
SPARK_APP_JAR="$(ls ${ZEPPELIN_HOME}/interpreter/spark/zeppelin-spark*.jar)"
# This will evantually passes SPARK_APP_JAR to classpath of SparkIMain
- ZEPPELIN_CLASSPATH+=${SPARK_APP_JAR}
+ ZEPPELIN_INTP_CLASSPATH+=":${SPARK_APP_JAR}"
pattern="$SPARK_HOME/python/lib/py4j-*-src.zip"
py4j=($pattern)
@@ -96,14 +96,14 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
# add Hadoop jars into classpath
if [[ -n "${HADOOP_HOME}" ]]; then
# Apache
- addEachJarInDirRecursive "${HADOOP_HOME}/share"
+ addEachJarInDirRecursiveForIntp "${HADOOP_HOME}/share"
# CDH
- addJarInDir "${HADOOP_HOME}"
- addJarInDir "${HADOOP_HOME}/lib"
+ addJarInDirForIntp "${HADOOP_HOME}"
+ addJarInDirForIntp "${HADOOP_HOME}/lib"
fi
- addJarInDir "${INTERPRETER_DIR}/dep"
+ addJarInDirForIntp "${INTERPRETER_DIR}/dep"
pattern="${ZEPPELIN_HOME}/interpreter/spark/pyspark/py4j-*-src.zip"
py4j=($pattern)
@@ -127,29 +127,29 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
fi
if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
- ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}"
+ ZEPPELIN_INTP_CLASSPATH+=":${HADOOP_CONF_DIR}"
fi
- export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
+ export SPARK_CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}"
fi
elif [[ "${INTERPRETER_ID}" == "hbase" ]]; then
if [[ -n "${HBASE_CONF_DIR}" ]]; then
- ZEPPELIN_CLASSPATH+=":${HBASE_CONF_DIR}"
+ ZEPPELIN_INTP_CLASSPATH+=":${HBASE_CONF_DIR}"
elif [[ -n "${HBASE_HOME}" ]]; then
- ZEPPELIN_CLASSPATH+=":${HBASE_HOME}/conf"
+ ZEPPELIN_INTP_CLASSPATH+=":${HBASE_HOME}/conf"
else
echo "HBASE_HOME and HBASE_CONF_DIR are not set, configuration might not be loaded"
fi
fi
-addJarInDir "${LOCAL_INTERPRETER_REPO}"
+addJarInDirForIntp "${LOCAL_INTERPRETER_REPO}"
-CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
+CLASSPATH+=":${ZEPPELIN_INTP_CLASSPATH}"
if [[ -n "${SPARK_SUBMIT}" ]]; then
- ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path "${ZEPPELIN_CLASSPATH_OVERRIDES}:${CLASSPATH}" --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} &
+ ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path "${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH}" --driver-java-options "${JAVA_INTP_OPTS}" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT} &
else
- ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_CLASSPATH_OVERRIDES}:${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} &
+ ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} ${ZEPPELIN_INTP_MEM} -cp ${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} &
fi
pid=$!