You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2015/06/24 22:26:45 UTC
incubator-zeppelin git commit: ZEPPELIN-104 Spark-1.4 as default
Zeppelin's build profile
Repository: incubator-zeppelin
Updated Branches:
refs/heads/master 955d6aa3b -> 3480a6865
ZEPPELIN-104 Spark-1.4 as default Zeppelin's build profile
This PR makes Spark-1.4 as default Zeppelin's build profile. https://issues.apache.org/jira/browse/ZEPPELIN-104
And Enabling CI test Zeppelin with spark version 1.4
Author: Lee moon soo <mo...@apache.org>
Closes #99 from Leemoonsoo/ZEPPELIN-104 and squashes the following commits:
675301d [Lee moon soo] start spark worker correctly for version 1.4
19f29c0 [Lee moon soo] Avoid travis log message size limit 4MB
220a7fb [Lee moon soo] Let CI test with spark 1.4
d7d6ba5 [Lee moon soo] Make default spark version 1.4
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/3480a686
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/3480a686
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/3480a686
Branch: refs/heads/master
Commit: 3480a6865c93b667a80f8faa800f2a20f8f54473
Parents: 955d6aa
Author: Lee moon soo <mo...@apache.org>
Authored: Mon Jun 22 17:00:50 2015 -0700
Committer: Lee moon soo <mo...@apache.org>
Committed: Wed Jun 24 13:26:37 2015 -0700
----------------------------------------------------------------------
.travis.yml | 18 ++++++++++++------
spark/pom.xml | 2 +-
testing/startSparkCluster.sh | 8 +++++++-
3 files changed, 20 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/3480a686/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 8bd717d..4031e78 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -28,22 +28,28 @@ before_script:
-
script:
-# spark 1.3
+# spark 1.4
- mvn package -Pbuild-distr -Phadoop-2.3 -B
+ - ./testing/startSparkCluster.sh 1.4.0 2.3
+ - SPARK_HOME=./spark-1.4.1-bin-hadoop2.3 mvn verify -Pusing-packaged-distr -Phadoop-2.3 -B
+ - ./testing/stopSparkCluster.sh 1.4.0 2.3
+ # spark 1.3
+ - mvn clean package -DskipTests -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
+ - mvn package -Pbuild-distr -Pspark-1.3 -Phadoop-2.3 -B
- ./testing/startSparkCluster.sh 1.3.1 2.3
- - SPARK_HOME=./spark-1.3.1-bin-hadoop2.3 mvn verify -Pusing-packaged-distr -Phadoop-2.3 -B
+ - SPARK_HOME=./spark-1.3.1-bin-hadoop2.3 mvn verify -Pspark-1.3 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
- ./testing/stopSparkCluster.sh 1.3.1 2.3
# spark 1.2
- - mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl '!zeppelin-web,!zeppelin-distribution'
+ - mvn clean package -DskipTests -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
- mvn package -Pbuild-distr -Pspark-1.2 -Phadoop-2.3 -B
- ./testing/startSparkCluster.sh 1.2.1 2.3
- - SPARK_HOME=./spark-1.2.1-bin-hadoop2.3 mvn verify -Pusing-packaged-distr -Pspark-1.2 -Phadoop-2.3 -B
+ - SPARK_HOME=./spark-1.2.1-bin-hadoop2.3 mvn verify -Pspark-1.2 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
- ./testing/stopSparkCluster.sh 1.2.1 2.3
# spark 1.1
- - mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl '!zeppelin-web,!zeppelin-distribution'
+ - mvn clean package -DskipTests -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
- mvn package -Pbuild-distr -Pspark-1.1 -Phadoop-2.3 -B
- ./testing/startSparkCluster.sh 1.1.1 2.3
- - SPARK_HOME=./spark-1.1.1-bin-hadoop2.3 mvn verify -Pusing-packaged-distr -Pspark-1.1 -Phadoop-2.3 -B
+ - SPARK_HOME=./spark-1.1.1-bin-hadoop2.3 mvn verify -Pspark-1.1 -Phadoop-2.3 -B -pl 'zeppelin-interpreter,spark'
- ./testing/stopSparkCluster.sh 1.1.1 2.3
after_failure:
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/3480a686/spark/pom.xml
----------------------------------------------------------------------
diff --git a/spark/pom.xml b/spark/pom.xml
index f819c09..8e20185 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -35,7 +35,7 @@
<properties>
- <spark.version>1.3.1</spark.version>
+ <spark.version>1.4.0</spark.version>
<scala.version>2.10.4</scala.version>
<scala.binary.version>2.10</scala.binary.version>
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/3480a686/testing/startSparkCluster.sh
----------------------------------------------------------------------
diff --git a/testing/startSparkCluster.sh b/testing/startSparkCluster.sh
index e2d8079..8ee0a32 100755
--- a/testing/startSparkCluster.sh
+++ b/testing/startSparkCluster.sh
@@ -45,4 +45,10 @@ export SPARK_MASTER_PORT=7071
export SPARK_MASTER_WEBUI_PORT=7072
export SPARK_WORKER_WEBUI_PORT=8082
${SPARK_HOME}/sbin/start-master.sh
-${SPARK_HOME}/sbin/start-slave.sh 1 `hostname`:${SPARK_MASTER_PORT}
+
+echo ${SPARK_VERSION} | grep "^1.4" > /dev/null
+if [ $? -ne 0 ]; then # spark 1.3 or prior
+ ${SPARK_HOME}/sbin/start-slave.sh 1 `hostname`:${SPARK_MASTER_PORT}
+else
+ ${SPARK_HOME}/sbin/start-slave.sh spark://`hostname`:7071
+fi