You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2015/11/02 11:39:32 UTC
incubator-zeppelin git commit: ZEPPELIN-377 CI hanging on
./testing/startSparkCluster.sh 1.4.0 2.3
Repository: incubator-zeppelin
Updated Branches:
refs/heads/master ac5ff10b6 -> 0a82a93ce
ZEPPELIN-377 CI hanging on ./testing/startSparkCluster.sh 1.4.0 2.3
Address https://issues.apache.org/jira/browse/ZEPPELIN-377.
This patch change spark package download location from apache archive to mirror, to download in 10min.
Also add missing test for 1.5.1 and change test version from 1.4.0 to 1.4.1
Author: Lee moon soo <mo...@apache.org>
Closes #380 from Leemoonsoo/fix_spark_test and squashes the following commits:
142583a [Lee moon soo] Add test for 1.5.1
b8323e6 [Lee moon soo] Use mirror for 1.3.x and later version of spark
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/0a82a93c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/0a82a93c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/0a82a93c
Branch: refs/heads/master
Commit: 0a82a93ce495cf537b28104a024a8c0361fd80ac
Parents: ac5ff10
Author: Lee moon soo <mo...@apache.org>
Authored: Mon Nov 2 13:46:02 2015 +0900
Committer: Lee moon soo <mo...@apache.org>
Committed: Mon Nov 2 19:39:31 2015 +0900
----------------------------------------------------------------------
.travis.yml | 17 ++++++++++++++---
pom.xml | 1 +
testing/startSparkCluster.sh | 18 +++++++++++++++---
.../apache/zeppelin/rest/AbstractTestRestApi.java | 9 +++++++++
4 files changed, 39 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/0a82a93c/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index a1b467d..0c74edf 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -28,27 +28,38 @@ before_script:
-
script:
-# spark 1.4
+ # spark 1.5
- mvn package -Pbuild-distr -Phadoop-2.3 -Ppyspark -B
- - ./testing/startSparkCluster.sh 1.4.0 2.3
+ - ./testing/startSparkCluster.sh 1.5.1 2.3
+ - echo "export SPARK_HOME=`pwd`/spark-1.5.1-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn verify -Pusing-packaged-distr -Phadoop-2.3 -Ppyspark -B
- - ./testing/stopSparkCluster.sh 1.4.0 2.3
+ - ./testing/stopSparkCluster.sh 1.5.1 2.3
+ # spark 1.4
+ - rm -rf `pwd`/interpreter/spark
+ - mvn package -DskipTests -Pspark-1.4 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
+ - ./testing/startSparkCluster.sh 1.4.1 2.3
+ - echo "export SPARK_HOME=`pwd`/spark-1.4.1-bin-hadoop2.3" > conf/zeppelin-env.sh
+ - mvn package -Pspark-1.4 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
+ - ./testing/stopSparkCluster.sh 1.4.1 2.3
# spark 1.3
- rm -rf `pwd`/interpreter/spark
- mvn package -DskipTests -Pspark-1.3 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.3.1 2.3
+ - echo "export SPARK_HOME=`pwd`/spark-1.3.1-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn package -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.3.1 2.3
# spark 1.2
- rm -rf `pwd`/interpreter/spark
- mvn package -Pspark-1.2 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.2.1 2.3
+ - echo "export SPARK_HOME=`pwd`/spark-1.2.1-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn package -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.2.1 2.3
# spark 1.1
- rm -rf `pwd`/interpreter/spark
- mvn package -Pspark-1.1 -Phadoop-2.3 -Ppyspark -B -pl 'zeppelin-interpreter,spark-dependencies,spark'
- ./testing/startSparkCluster.sh 1.1.1 2.3
+ - echo "export SPARK_HOME=`pwd`/spark-1.1.1-bin-hadoop2.3" > conf/zeppelin-env.sh
- mvn package -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,zeppelin-zengine,zeppelin-server' -Dtest=org.apache.zeppelin.rest.*Test -DfailIfNoTests=false
- ./testing/stopSparkCluster.sh 1.1.1 2.3
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/0a82a93c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 45335fe..394f853 100755
--- a/pom.xml
+++ b/pom.xml
@@ -456,6 +456,7 @@
<exclude>STYLE.md</exclude>
<exclude>Roadmap.md</exclude>
<exclude>conf/interpreter.json</exclude>
+ <exclude>conf/zeppelin-env.sh</exclude>
<exclude>spark-*-bin*/**</exclude>
</excludes>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/0a82a93c/testing/startSparkCluster.sh
----------------------------------------------------------------------
diff --git a/testing/startSparkCluster.sh b/testing/startSparkCluster.sh
index 1f70fe6..7333ab0 100755
--- a/testing/startSparkCluster.sh
+++ b/testing/startSparkCluster.sh
@@ -31,7 +31,19 @@ ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
export SPARK_HOME=${ZEPPELIN_HOME}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}
echo "SPARK_HOME is ${SPARK_HOME} "
if [ ! -d "${SPARK_HOME}" ]; then
- wget -q http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
+ echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
+ if [ $? -eq 0 ]; then
+ # spark 1.1.x and spark 1.2.x can be downloaded from archive
+ wget http://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
+ else
+ # spark 1.3.x and later can be downloaded from mirror
+ # get download address from mirror
+ MIRROR_INFO=$(curl -s "http://www.apache.org/dyn/closer.cgi/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz?asjson=1")
+
+ PREFFERED=$(echo "${MIRROR_INFO}" | grep preferred | sed 's/[^"]*.preferred.: .\([^"]*\).*/\1/g')
+ PATHINFO=$(echo "${MIRROR_INFO}" | grep path_info | sed 's/[^"]*.path_info.: .\([^"]*\).*/\1/g')
+ wget "${PREFFERED}${PATHINFO}"
+ fi
tar zxf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
fi
@@ -46,8 +58,8 @@ export SPARK_MASTER_WEBUI_PORT=7072
export SPARK_WORKER_WEBUI_PORT=8082
${SPARK_HOME}/sbin/start-master.sh
-echo ${SPARK_VERSION} | grep "^1.4" > /dev/null
-if [ $? -ne 0 ]; then # spark 1.3 or prior
+echo ${SPARK_VERSION} | grep "^1.[123].[0-9]" > /dev/null
+if [ $? -eq 0 ]; then # spark 1.3 or prior
${SPARK_HOME}/sbin/start-slave.sh 1 `hostname`:${SPARK_MASTER_PORT}
else
${SPARK_HOME}/sbin/start-slave.sh spark://`hostname`:7071
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/0a82a93c/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
index ecf0c72..1895e16 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.lang.ref.WeakReference;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.List;
import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -29,6 +30,7 @@ import java.util.concurrent.Executors;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.methods.*;
+import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter;
import org.apache.zeppelin.interpreter.InterpreterGroup;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.InterpreterSetting;
@@ -197,6 +199,13 @@ public abstract class AbstractTestRestApi {
protected static void shutDown() throws Exception {
if (!wasRunning) {
+ // restart interpreter to stop all interpreter processes
+ List<String> settingList = ZeppelinServer.notebook.getInterpreterFactory()
+ .getDefaultInterpreterSettingList();
+ for (String setting : settingList) {
+ ZeppelinServer.notebook.getInterpreterFactory().restart(setting);
+ }
+
LOG.info("Terminating test Zeppelin...");
ZeppelinServer.jettyServer.stop();
executor.shutdown();