You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2015/05/13 07:16:54 UTC
incubator-zeppelin git commit: Simplify classpath.
Repository: incubator-zeppelin
Updated Branches:
refs/heads/master aaee71977 -> dcb03a9fe
Simplify classpath.
When Zeppelin constructs JVM CLASSPATH, it adds path of every single jar. That creates really long CLASSPATH and it may cause some problem like https://issues.apache.org/jira/browse/ZEPPELIN-68.
This PR solves it by constructing CLASSPATH with wildcard.
Another problem is, Zeppelin constructs CLASSPATH with duplicated entry or not related entry.
This PR solves it by constructing CLASSPATH not in common.sh but in each script zeppelin-daemon.sh, interpreter.sh, zeppelin.sh while they need different entry in CLASSPATH.
Author: Lee moon soo <mo...@apache.org>
Closes #65 from Leemoonsoo/ZEPPELIN-68 and squashes the following commits:
ae3ea5a [Lee moon soo] Print rat.txt when build failed
4eee837 [Lee moon soo] ZEPPELIN-68 simplify classpath
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/dcb03a9f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/dcb03a9f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/dcb03a9f
Branch: refs/heads/master
Commit: dcb03a9fe66d753348154c41f5b40927125723aa
Parents: aaee719
Author: Lee moon soo <mo...@apache.org>
Authored: Wed May 13 01:45:30 2015 +0200
Committer: Lee moon soo <mo...@apache.org>
Committed: Wed May 13 14:16:19 2015 +0900
----------------------------------------------------------------------
.travis.yml | 1 +
bin/common.sh | 23 ++++++-----------------
bin/interpreter.sh | 9 +++++++--
bin/zeppelin-daemon.sh | 22 ++++++++++++++++++++++
bin/zeppelin.sh | 22 ++++++++++++++++++++++
5 files changed, 58 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/dcb03a9f/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 7eb5935..b09e228 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,6 +29,7 @@ script:
- mvn verify -Pusing-packaged-distr -B
after_failure:
+ - cat target/rat.txt
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.log
- cat zeppelin-distribution/target/zeppelin-*-SNAPSHOT/zeppelin-*-SNAPSHOT/logs/zeppelin*.out
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/dcb03a9f/bin/common.sh
----------------------------------------------------------------------
diff --git a/bin/common.sh b/bin/common.sh
index a825b76..8087e9d 100644
--- a/bin/common.sh
+++ b/bin/common.sh
@@ -70,28 +70,19 @@ fi
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_CONF_DIR}"
-function addJarInDir(){
+function addEachJarInDir(){
if [[ -d "${1}" ]]; then
for jar in $(find -L "${1}" -maxdepth 1 -name '*jar'); do
ZEPPELIN_CLASSPATH="$jar:$ZEPPELIN_CLASSPATH"
done
fi
}
-
-addJarInDir "${ZEPPELIN_HOME}"
-addJarInDir "${ZEPPELIN_HOME}/lib"
-addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
-addJarInDir "${ZEPPELIN_HOME}/zeppelin-zengine/target/lib"
-addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib"
-addJarInDir "${ZEPPELIN_HOME}/zeppelin-web/target/lib"
-
-if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then
- ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes"
-fi
-if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then
- ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes"
-fi
+function addJarInDir(){
+ if [[ -d "${1}" ]]; then
+ export ZEPPELIN_CLASSPATH="${1}/*:${ZEPPELIN_CLASSPATH}"
+ fi
+}
if [[ ! -z "${SPARK_HOME}" ]] && [[ -d "${SPARK_HOME}" ]]; then
addJarInDir "${SPARK_HOME}"
@@ -106,8 +97,6 @@ if [[ ! -z "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
fi
export ZEPPELIN_CLASSPATH
-export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
-export CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
# Text encoding for
# read/write job into files,
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/dcb03a9f/bin/interpreter.sh
----------------------------------------------------------------------
diff --git a/bin/interpreter.sh b/bin/interpreter.sh
index 025df36..c214c30 100755
--- a/bin/interpreter.sh
+++ b/bin/interpreter.sh
@@ -49,10 +49,16 @@ fi
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_CONF_DIR}"
+# construct classpath
+if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then
+ ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes"
+fi
+
addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
addJarInDir "${INTERPRETER_DIR}"
-export CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
+export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
+CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
HOSTNAME=$(hostname)
ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer
@@ -68,7 +74,6 @@ if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then
fi
-
${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} -cp ${CLASSPATH} ${ZEPPELIN_SERVER} ${PORT} &
pid=$!
if [[ -z "${pid}" ]]; then
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/dcb03a9f/bin/zeppelin-daemon.sh
----------------------------------------------------------------------
diff --git a/bin/zeppelin-daemon.sh b/bin/zeppelin-daemon.sh
index 56e6c88..2440c12 100755
--- a/bin/zeppelin-daemon.sh
+++ b/bin/zeppelin-daemon.sh
@@ -54,6 +54,28 @@ ZEPPELIN_PID="${ZEPPELIN_PID_DIR}/zeppelin-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.
ZEPPELIN_MAIN=org.apache.zeppelin.server.ZeppelinServer
JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}"
+# construct classpath
+if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then
+ ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes"
+fi
+
+if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then
+ ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes"
+fi
+
+if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then
+ ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes"
+fi
+
+addJarInDir "${ZEPPELIN_HOME}"
+addJarInDir "${ZEPPELIN_HOME}/lib"
+addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
+addJarInDir "${ZEPPELIN_HOME}/zeppelin-zengine/target/lib"
+addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib"
+addJarInDir "${ZEPPELIN_HOME}/zeppelin-web/target/lib"
+
+CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
+
if [[ "${ZEPPELIN_NICENESS}" = "" ]]; then
export ZEPPELIN_NICENESS=0
fi
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/dcb03a9f/bin/zeppelin.sh
----------------------------------------------------------------------
diff --git a/bin/zeppelin.sh b/bin/zeppelin.sh
index d81556c..eb2f691 100755
--- a/bin/zeppelin.sh
+++ b/bin/zeppelin.sh
@@ -51,6 +51,28 @@ LOG="${ZEPPELIN_LOG_DIR}/zeppelin-cli-${ZEPPELIN_IDENT_STRING}-${HOSTNAME}.out"
ZEPPELIN_SERVER=org.apache.zeppelin.server.ZeppelinServer
JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}"
+# construct classpath
+if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then
+ ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes"
+fi
+
+if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then
+ ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes"
+fi
+
+if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then
+ ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes"
+fi
+
+addJarInDir "${ZEPPELIN_HOME}"
+addJarInDir "${ZEPPELIN_HOME}/lib"
+addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib"
+addJarInDir "${ZEPPELIN_HOME}/zeppelin-zengine/target/lib"
+addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib"
+addJarInDir "${ZEPPELIN_HOME}/zeppelin-web/target/lib"
+
+CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
+
if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then
echo "Log dir doesn't exist, create ${ZEPPELIN_LOG_DIR}"
$(mkdir -p "${ZEPPELIN_LOG_DIR}")