You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by jo...@apache.org on 2016/10/11 08:13:29 UTC
zeppelin git commit: [ZEPPELIN-1520] Change test environment to use
the local mode of Apache Spark
Repository: zeppelin
Updated Branches:
refs/heads/master 52202afd3 -> d2f63f6f9
[ZEPPELIN-1520] Change test environment to use the local mode of Apache Spark
### What is this PR for?
Simplify travis test to reduce usage of resources
### What type of PR is it?
[Improvement]
### Todos
* [x] - Remove start-up/stop SparkCluster
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-1520
### How should this be tested?
Travis will pass without any error
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? no
* Is there breaking changes for older versions? no
* Does this needs documentation? no
Author: Jongyoul Lee <jo...@gmail.com>
Closes #1487 from jongyoul/ZEPPELIN-1520 and squashes the following commits:
3bccf66 [Jongyoul Lee] Removed some unused scripts anymore
15a3711 [Jongyoul Lee] Cleaned up commented lines
1237658 [Jongyoul Lee] Removed checking mechanism
f37dacf [Jongyoul Lee] Changed master to local[2]
2aac444 [Jongyoul Lee] Remove scripts of start/stop SparkCluster
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/d2f63f6f
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/d2f63f6f
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/d2f63f6f
Branch: refs/heads/master
Commit: d2f63f6f96f1921ebc901915abc606f431461cb8
Parents: 52202af
Author: Jongyoul Lee <jo...@gmail.com>
Authored: Mon Oct 10 00:02:25 2016 +0900
Committer: Jongyoul Lee <jo...@apache.org>
Committed: Tue Oct 11 17:13:20 2016 +0900
----------------------------------------------------------------------
.travis.yml | 4 --
testing/startSparkCluster.sh | 67 --------------------
testing/stopSparkCluster.sh | 42 ------------
.../zeppelin/rest/AbstractTestRestApi.java | 12 +---
4 files changed, 3 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d2f63f6f/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index e2a115c..d314b2f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -89,7 +89,6 @@ install:
before_script:
- travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
- - ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER
- echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
- tail conf/zeppelin-env.sh
@@ -109,6 +108,3 @@ after_failure:
- cat zeppelin-web/npm-debug.log
- cat spark-*/logs/*
-after_script:
- - ./testing/stopSparkCluster.sh $SPARK_VER $HADOOP_VER
-
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d2f63f6f/testing/startSparkCluster.sh
----------------------------------------------------------------------
diff --git a/testing/startSparkCluster.sh b/testing/startSparkCluster.sh
deleted file mode 100755
index dc7613d..0000000
--- a/testing/startSparkCluster.sh
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-if [[ "$#" -ne 2 ]]; then
- echo "usage) $0 [spark version] [hadoop version]"
- echo " eg) $0 1.3.1 2.6"
- exit 1
-fi
-
-SPARK_VERSION="${1}"
-HADOOP_VERSION="${2}"
-
-echo "${SPARK_VERSION}" | grep "^1.[123].[0-9]" > /dev/null
-if [[ "$?" -eq 0 ]]; then
- echo "${SPARK_VERSION}" | grep "^1.[12].[0-9]" > /dev/null
- if [[ "$?" -eq 0 ]]; then
- SPARK_VER_RANGE="<=1.2"
- else
- SPARK_VER_RANGE="<=1.3"
- fi
-else
- SPARK_VER_RANGE=">1.3"
-fi
-
-set -xe
-
-FWDIR="$(dirname "${BASH_SOURCE-$0}")"
-ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
-
-SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
-export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
-echo "SPARK_HOME is ${SPARK_HOME}"
-
-# create PID dir. test case detect pid file so they can select active spark home dir for test
-export SPARK_PID_DIR="${SPARK_HOME}/run"
-mkdir -p "${SPARK_PID_DIR}"
-
-# start
-export SPARK_MASTER_PORT=7071
-export SPARK_MASTER_WEBUI_PORT=7072
-export SPARK_WORKER_WEBUI_PORT=8082
-${SPARK_HOME}/sbin/start-master.sh
-
-if [ "${SPARK_VER_RANGE}" == "<=1.3" ]||[ "${SPARK_VER_RANGE}" == "<=1.2" ]; then
- # spark 1.3 or prior
- ${SPARK_HOME}/sbin/start-slave.sh 1 `hostname`:${SPARK_MASTER_PORT}
-else
- ${SPARK_HOME}/sbin/start-slave.sh spark://`hostname`:7071
-fi
-
-set +xe
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d2f63f6f/testing/stopSparkCluster.sh
----------------------------------------------------------------------
diff --git a/testing/stopSparkCluster.sh b/testing/stopSparkCluster.sh
deleted file mode 100755
index e049ec4..0000000
--- a/testing/stopSparkCluster.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-if [[ "$#" -ne 2 ]]; then
- echo "usage) $0 [spark version] [hadoop version]"
- echo " eg) $0 1.3.1 2.6"
- exit 1
-fi
-
-SPARK_VERSION="${1}"
-HADOOP_VERSION="${2}"
-
-set -xe
-
-FWDIR="$(dirname "${BASH_SOURCE-$0}")"
-ZEPPELIN_HOME="$(cd "${FWDIR}/.."; pwd)"
-
-SPARK_ARCHIVE="spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}"
-export SPARK_HOME="${ZEPPELIN_HOME}/${SPARK_ARCHIVE}"
-echo "SPARK_HOME is ${SPARK_HOME}"
-
-# set create PID dir
-export SPARK_PID_DIR="${SPARK_HOME}/run"
-
-${SPARK_HOME}/sbin/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker 1
-${SPARK_HOME}/sbin/stop-master.sh
-set +xe
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d2f63f6f/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
----------------------------------------------------------------------
diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
index eb080fe..de2607f 100644
--- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
+++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java
@@ -152,7 +152,7 @@ public abstract class AbstractTestRestApi {
}
// set spark master and other properties
- sparkIntpSetting.getProperties().setProperty("master", "spark://" + getHostname() + ":7071");
+ sparkIntpSetting.getProperties().setProperty("master", "local[2]");
sparkIntpSetting.getProperties().setProperty("spark.cores.max", "2");
sparkIntpSetting.getProperties().setProperty("zeppelin.spark.useHiveContext", "false");
// set spark home for pyspark
@@ -175,7 +175,7 @@ public abstract class AbstractTestRestApi {
sparkIntpSetting.getProperties().setProperty("master", System.getenv("SPARK_MASTER"));
} else {
sparkIntpSetting.getProperties()
- .setProperty("master", "spark://" + getHostname() + ":7071");
+ .setProperty("master", "local[2]");
}
sparkIntpSetting.getProperties().setProperty("spark.cores.max", "2");
// set spark home for pyspark
@@ -238,13 +238,7 @@ public abstract class AbstractTestRestApi {
}
private static boolean isActiveSparkHome(File dir) {
- if (dir.getName().matches("spark-[0-9\\.]+[A-Za-z-]*-bin-hadoop[0-9\\.]+")) {
- File pidDir = new File(dir, "run");
- if (pidDir.isDirectory() && pidDir.listFiles().length > 0) {
- return true;
- }
- }
- return false;
+ return dir.getName().matches("spark-[0-9\\.]+[A-Za-z-]*-bin-hadoop[0-9\\.]+");
}
protected static void shutDown() throws Exception {