You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ev...@apache.org on 2019/03/23 10:40:44 UTC
[bigtop] branch master updated: BIGTOP-3192. Remove Spark 1.6 from
the stack
This is an automated email from the ASF dual-hosted git repository.
evansye pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 877e260 BIGTOP-3192. Remove Spark 1.6 from the stack
877e260 is described below
commit 877e260d0cd22458b3a41369b1d1baf4f6dd411e
Author: Konstantin Boudnik <co...@apache.org>
AuthorDate: Sun Mar 17 19:49:04 2019 -0400
BIGTOP-3192. Remove Spark 1.6 from the stack
Removing Spark 1.6 component from the stack
---
.../spark/layer-spark/tests/03-scale-standalone.py | 2 -
.../src/common/crunch/do-component-build | 2 +-
.../src/common/mahout/do-component-build | 2 +-
.../src/common/phoenix/do-component-build | 2 +-
.../src/common/spark1/do-component-build | 46 ----
bigtop-packages/src/common/spark1/install_spark.sh | 211 -----------------
bigtop-packages/src/common/spark1/patch0.diff | 11 -
bigtop-packages/src/common/spark1/spark-env.sh | 43 ----
.../src/common/spark1/spark-history-server.svc | 70 ------
bigtop-packages/src/common/spark1/spark-master.svc | 81 -------
.../src/common/spark1/spark-thriftserver.svc | 75 ------
bigtop-packages/src/common/spark1/spark-worker.svc | 75 ------
bigtop-packages/src/deb/spark1/changelog | 1 -
bigtop-packages/src/deb/spark1/compat | 1 -
bigtop-packages/src/deb/spark1/control | 77 -------
bigtop-packages/src/deb/spark1/copyright | 15 --
bigtop-packages/src/deb/spark1/rules | 42 ----
bigtop-packages/src/deb/spark1/source/format | 1 -
bigtop-packages/src/deb/spark1/spark-core.install | 27 ---
bigtop-packages/src/deb/spark1/spark-core.postinst | 35 ---
bigtop-packages/src/deb/spark1/spark-core.preinst | 63 -----
bigtop-packages/src/deb/spark1/spark-core.prerm | 38 ---
.../src/deb/spark1/spark-datanucleus.install | 2 -
.../src/deb/spark1/spark-extras.install | 1 -
.../src/deb/spark1/spark-python.install | 4 -
.../src/deb/spark1/spark-yarn-shuffle.install | 2 -
bigtop-packages/src/rpm/spark1/BUILD/.gitignore | 0
bigtop-packages/src/rpm/spark1/RPMS/.gitignore | 0
bigtop-packages/src/rpm/spark1/SOURCES/.gitignore | 0
bigtop-packages/src/rpm/spark1/SPECS/spark1.spec | 255 ---------------------
bigtop-packages/src/rpm/spark1/SRPMS/.gitignore | 0
bigtop.bom | 13 +-
32 files changed, 4 insertions(+), 1193 deletions(-)
diff --git a/bigtop-packages/src/charm/spark/layer-spark/tests/03-scale-standalone.py b/bigtop-packages/src/charm/spark/layer-spark/tests/03-scale-standalone.py
index 7bef4b5..ecbc37d 100755
--- a/bigtop-packages/src/charm/spark/layer-spark/tests/03-scale-standalone.py
+++ b/bigtop-packages/src/charm/spark/layer-spark/tests/03-scale-standalone.py
@@ -47,10 +47,8 @@ class TestScaleStandalone(unittest.TestCase):
time.sleep(120)
spark0_unit = self.d.sentry['spark-test-scale'][0]
- spark1_unit = self.d.sentry['spark-test-scale'][1]
spark2_unit = self.d.sentry['spark-test-scale'][2]
(stdout0, errcode0) = spark0_unit.run('grep spark.master /etc/spark/conf/spark-defaults.conf')
- (stdout1, errcode1) = spark1_unit.run('grep spark.master /etc/spark/conf/spark-defaults.conf')
(stdout2, errcode2) = spark2_unit.run('grep spark.master /etc/spark/conf/spark-defaults.conf')
# ensure units agree on the master
assert stdout0 == stdout2
diff --git a/bigtop-packages/src/common/crunch/do-component-build b/bigtop-packages/src/common/crunch/do-component-build
index b8642ae..32f6108 100644
--- a/bigtop-packages/src/common/crunch/do-component-build
+++ b/bigtop-packages/src/common/crunch/do-component-build
@@ -27,7 +27,7 @@ fi
mvn -DskipTests -Dcrunch.platform=2 \
-Dhadoop.version=$HADOOP_VERSION \
-Dhbase.version=$HBASE_VERSION \
- -Dspark.version=$SPARK1_VERSION \
+ -Dspark.version=$SPARK_VERSION \
-Papache-release -Ddependency.locations.enabled=false \
${MAVEN_ADDITIONAL} \
clean site:site package "$@"
diff --git a/bigtop-packages/src/common/mahout/do-component-build b/bigtop-packages/src/common/mahout/do-component-build
index 6211892..0593f71 100644
--- a/bigtop-packages/src/common/mahout/do-component-build
+++ b/bigtop-packages/src/common/mahout/do-component-build
@@ -22,7 +22,7 @@ mvn clean install -DskipTests -Dmahout.skip.distribution=false \
-Phadoop2 \
-Dhadoop.version=$HADOOP_VERSION \
-Dhbase.version=$HBASE_VERSION \
- -Dspark.version=${SPARK1_VERSION} \
+ -Dspark.version=${SPARK_VERSION} \
"$@"
mkdir build
diff --git a/bigtop-packages/src/common/phoenix/do-component-build b/bigtop-packages/src/common/phoenix/do-component-build
index d5df723..6525566 100644
--- a/bigtop-packages/src/common/phoenix/do-component-build
+++ b/bigtop-packages/src/common/phoenix/do-component-build
@@ -22,7 +22,7 @@ mvn -DskipTests \
-Dhadoop-two.version=$HADOOP_VERSION \
-Dhbase.version=$HBASE_VERSION \
-Dflume.version=$FLUME_VERSION \
- -Dspark.version=${SPARK1_VERSION} \
+ -Dspark.version=${SPARK_VERSION} \
-Dhive.version=${HIVE_VERSION} \
clean install "$@"
rm -rf build
diff --git a/bigtop-packages/src/common/spark1/do-component-build b/bigtop-packages/src/common/spark1/do-component-build
deleted file mode 100644
index 7ea85d4..0000000
--- a/bigtop-packages/src/common/spark1/do-component-build
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -ex
-
-. `dirname $0`/bigtop.bom
-
-# BIGTOP-2607
-sed -i -e /useZincServer/d -e /recompileMode/d pom.xml
-
-BUILD_OPTS="-Divy.home=${HOME}/.ivy2 -Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${HOME} \
- -Drepo.maven.org=$IVY_MIRROR_PROP \
- -Dreactor.repo=file://${HOME}/.m2/repository \
- -Dhadoop.version=$HADOOP_VERSION \
- -Dyarn.version=$HADOOP_VERSION \
- -Dprotobuf.version=2.5.0 \
- -DrecompileMode=all \
- -Pbigtop-dist \
- -Pyarn -Phadoop-2.6 \
- -Phive -Phive-thriftserver \
- $SPARK_BUILD_OPTS"
-
-## this might be an issue at times
-# http://maven.40175.n5.nabble.com/Not-finding-artifact-in-local-repo-td3727753.html
-export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m"
-
-mvn $BUILD_OPTS -DskipTests install
-
-# Tests must be run after Spark has already been packaged.
-# See http://spark.apache.org/docs/latest/building-spark.html#spark-tests-in-maven
-if [ "$SPARK_RUN_TESTS" = "true" ]; then
- mvn $BUILD_OPTS test
-fi
diff --git a/bigtop-packages/src/common/spark1/install_spark.sh b/bigtop-packages/src/common/spark1/install_spark.sh
deleted file mode 100644
index c612849..0000000
--- a/bigtop-packages/src/common/spark1/install_spark.sh
+++ /dev/null
@@ -1,211 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-usage() {
- echo "
-usage: $0 <options>
- Required not-so-options:
- --build-dir=DIR path to dist.dir
- --source-dir=DIR path to package shared files dir
- --prefix=PREFIX path to install into
-
- Optional options:
- --doc-dir=DIR path to install docs into [/usr/share/doc/spark]
- --lib-dir=DIR path to install Spark home [/usr/lib/spark]
- --installed-lib-dir=DIR path where lib-dir will end up on target system
- --bin-dir=DIR path to install bins [/usr/bin]
- --examples-dir=DIR path to install examples [doc-dir/examples]
- --pyspark-python executable to use for Python interpreter [python]
- ... [ see source for more similar options ]
- "
- exit 1
-}
-
-OPTS=$(getopt \
- -n $0 \
- -o '' \
- -l 'prefix:' \
- -l 'doc-dir:' \
- -l 'lib-dir:' \
- -l 'installed-lib-dir:' \
- -l 'bin-dir:' \
- -l 'source-dir:' \
- -l 'examples-dir:' \
- -l 'pyspark-python:' \
- -l 'build-dir:' -- "$@")
-
-if [ $? != 0 ] ; then
- usage
-fi
-
-eval set -- "$OPTS"
-while true ; do
- case "$1" in
- --prefix)
- PREFIX=$2 ; shift 2
- ;;
- --build-dir)
- BUILD_DIR=$2 ; shift 2
- ;;
- --source-dir)
- SOURCE_DIR=$2 ; shift 2
- ;;
- --doc-dir)
- DOC_DIR=$2 ; shift 2
- ;;
- --lib-dir)
- LIB_DIR=$2 ; shift 2
- ;;
- --installed-lib-dir)
- INSTALLED_LIB_DIR=$2 ; shift 2
- ;;
- --bin-dir)
- BIN_DIR=$2 ; shift 2
- ;;
- --examples-dir)
- EXAMPLES_DIR=$2 ; shift 2
- ;;
- --pyspark-python)
- PYSPARK_PYTHON=$2 ; shift 2
- ;;
- --)
- shift ; break
- ;;
- *)
- echo "Unknown option: $1"
- usage
- exit 1
- ;;
- esac
-done
-
-for var in PREFIX BUILD_DIR SOURCE_DIR; do
- if [ -z "$(eval "echo \$$var")" ]; then
- echo Missing param: $var
- usage
- fi
-done
-
-if [ -f "$SOURCE_DIR/bigtop.bom" ]; then
- . $SOURCE_DIR/bigtop.bom
-fi
-
-MAN_DIR=${MAN_DIR:-/usr/share/man/man1}
-DOC_DIR=${DOC_DIR:-/usr/share/doc/spark}
-LIB_DIR=${LIB_DIR:-/usr/lib/spark}
-INSTALLED_LIB_DIR=${INSTALLED_LIB_DIR:-/usr/lib/spark}
-EXAMPLES_DIR=${EXAMPLES_DIR:-$DOC_DIR/examples}
-BIN_DIR=${BIN_DIR:-/usr/bin}
-CONF_DIR=${CONF_DIR:-/etc/spark/conf.dist}
-PYSPARK_PYTHON=${PYSPARK_PYTHON:-python}
-
-install -d -m 0755 $PREFIX/$LIB_DIR
-install -d -m 0755 $PREFIX/$LIB_DIR/lib
-install -d -m 0755 $PREFIX/$LIB_DIR/bin
-install -d -m 0755 $PREFIX/$LIB_DIR/sbin
-install -d -m 0755 $PREFIX/$LIB_DIR/extras
-install -d -m 0755 $PREFIX/$LIB_DIR/extras/lib
-install -d -m 0755 $PREFIX/$LIB_DIR/yarn
-install -d -m 0755 $PREFIX/$LIB_DIR/yarn/lib
-install -d -m 0755 $PREFIX/$DOC_DIR
-install -d -m 0755 $PREFIX/$EXAMPLES_DIR
-
-install -d -m 0755 $PREFIX/var/lib/spark/
-install -d -m 0755 $PREFIX/var/log/spark/
-install -d -m 0755 $PREFIX/var/run/spark/
-install -d -m 0755 $PREFIX/var/run/spark/work/
-
-tar --wildcards -C $PREFIX/$LIB_DIR/lib -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz spark-assembly\*
-tar --wildcards --strip-components=1 -C $PREFIX/$LIB_DIR/lib -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz \*datanucleus\*
-tar --wildcards -C $PREFIX/$LIB_DIR/ -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz bin\*
-tar --wildcards -C $PREFIX/$LIB_DIR/ -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz sbin\*
-
-rm -rf $PREFIX/$LIB_DIR/bin/*.cmd
-
-# External/extra jars
-ls ${BUILD_DIR}/{external,extras}/*/target/*${SPARK1_VERSION}.jar | grep -v 'original-\|assembly' | xargs -IJARS cp JARS $PREFIX/$LIB_DIR/extras/lib
-
-# Examples jar
-cp ${BUILD_DIR}/examples/target/spark-examples*${SPARK1_VERSION}.jar $PREFIX/$LIB_DIR/lib/spark-examples-${SPARK1_VERSION}-hadoop${HADOOP_VERSION}.jar
-
-# Spark YARN Shuffle jar
-cp ${BUILD_DIR}/network/yarn/target/*/spark-${SPARK1_VERSION}-yarn-shuffle.jar $PREFIX/$LIB_DIR/lib/
-
-# Examples src
-cp -ra ${BUILD_DIR}/examples/src $PREFIX/$EXAMPLES_DIR/
-ln -s $EXAMPLES_DIR $PREFIX/$LIB_DIR/examples
-
-# Data
-cp -ra ${BUILD_DIR}/data $PREFIX/$LIB_DIR/
-
-chmod 755 $PREFIX/$LIB_DIR/bin/*
-chmod 755 $PREFIX/$LIB_DIR/sbin/*
-
-# Copy in the configuration files
-install -d -m 0755 $PREFIX/$CONF_DIR
-cp -a ${BUILD_DIR}/conf/* $PREFIX/$CONF_DIR
-cp $SOURCE_DIR/spark-env.sh $PREFIX/$CONF_DIR
-ln -s /etc/spark/conf $PREFIX/$LIB_DIR/conf
-
-# Copy in the wrappers
-install -d -m 0755 $PREFIX/$BIN_DIR
-for wrap in bin/spark-class bin/spark-shell bin/spark-sql bin/spark-submit; do
- cat > $PREFIX/$BIN_DIR/`basename $wrap` <<EOF
-#!/bin/bash
-
-# Autodetect JAVA_HOME if not defined
-. /usr/lib/bigtop-utils/bigtop-detect-javahome
-
-exec $INSTALLED_LIB_DIR/$wrap "\$@"
-EOF
- chmod 755 $PREFIX/$BIN_DIR/`basename $wrap`
-done
-
-ln -s /var/run/spark/work $PREFIX/$LIB_DIR/work
-
-cp -r ${BUILD_DIR}/python ${PREFIX}/${INSTALLED_LIB_DIR}/
-rm -f ${PREFIX}/${INSTALLED_LIB_DIR}/python/.gitignore
-cat > $PREFIX/$BIN_DIR/pyspark <<EOF
-#!/bin/bash
-
-# Autodetect JAVA_HOME if not defined
-. /usr/lib/bigtop-utils/bigtop-detect-javahome
-
-export PYSPARK_PYTHON=${PYSPARK_PYTHON}
-
-exec $INSTALLED_LIB_DIR/bin/pyspark "\$@"
-EOF
-chmod 755 $PREFIX/$BIN_DIR/pyspark
-
-touch $PREFIX/$LIB_DIR/RELEASE
-cp ${BUILD_DIR}/{LICENSE,NOTICE} ${PREFIX}/${LIB_DIR}/
-
-# Version-less symlinks
-(cd $PREFIX/$LIB_DIR/lib; ln -s spark-assembly*.jar spark-assembly.jar; ln -s spark-examples*.jar spark-examples.jar)
-pushd $PREFIX/$LIB_DIR/yarn/lib
-ln -s ../../lib/spark-*-yarn-shuffle.jar spark-yarn-shuffle.jar
-ln -s ../../lib/datanucleus-api-jdo*.jar datanucleus-api-jdo.jar
-ln -s ../../lib/datanucleus-core*.jar datanucleus-core.jar
-ln -s ../../lib/datanucleus-rdbms*.jar datanucleus-rdbms.jar
-popd
-pushd $PREFIX/$LIB_DIR/extras/lib
-for j in $(ls *.jar); do
- ln -s $j $(echo $j | sed -n 's/\(.*\)\(_[0-9.]\+-[0-9.]\+\)\(.jar\)/\1\3/p')
-done
-popd
diff --git a/bigtop-packages/src/common/spark1/patch0.diff b/bigtop-packages/src/common/spark1/patch0.diff
deleted file mode 100644
index 5e1aefb..0000000
--- a/bigtop-packages/src/common/spark1/patch0.diff
+++ /dev/null
@@ -1,11 +0,0 @@
---- spark-1.6.2/pom.xml 2016-06-21 20:56:39.000000000 -0500
-+++ spark-1.6.2-patched/pom.xml 2016-11-01 21:46:37.629234248 -0500
-@@ -2511,7 +2511,7 @@
- <profile>
- <id>hadoop-2.6</id>
- <properties>
-- <hadoop.version>2.6.0</hadoop.version>
-+ <hadoop.version>${hadoop.version}</hadoop.version>
- <jets3t.version>0.9.3</jets3t.version>
- <zookeeper.version>3.4.6</zookeeper.version>
- <curator.version>2.6.0</curator.version>
diff --git a/bigtop-packages/src/common/spark1/spark-env.sh b/bigtop-packages/src/common/spark1/spark-env.sh
deleted file mode 100644
index 885aed1..0000000
--- a/bigtop-packages/src/common/spark1/spark-env.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-export SPARK_HOME=${SPARK_HOME:-/usr/lib/spark}
-export SPARK_LOG_DIR=${SPARK_LOG_DIR:-/var/log/spark}
-
-export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop}
-export HADOOP_HDFS_HOME=${HADOOP_HDFS_HOME:-${HADOOP_HOME}/../hadoop-hdfs}
-export HADOOP_MAPRED_HOME=${HADOOP_MAPRED_HOME:-${HADOOP_HOME}/../hadoop-mapreduce}
-export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-${HADOOP_HOME}/../hadoop-yarn}
-export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
-
-# Let's run everything with JVM runtime, instead of Scala
-export SPARK_LAUNCH_WITH_SCALA=0
-export SPARK_LIBRARY_PATH=${SPARK_LIBRARY_PATH:-${SPARK_HOME}/lib}
-export SCALA_LIBRARY_PATH=${SCALA_LIBRARY_PATH:-${SPARK_HOME}/lib}
-
-# Let's make sure that all needed hadoop libs are added properly
-export CLASSPATH="$CLASSPATH:$HADOOP_HOME/*:$HADOOP_HDFS_HOME/*:$HADOOP_YARN_HOME/*:$HADOOP_MAPRED_HOME/*"
-export SPARK_LIBRARY_PATH=$SPARK_LIBRARY_PATH:${HADOOP_HOME}/lib/native
-
-export STANDALONE_SPARK_MASTER_HOST=`hostname -f`
-export SPARK_MASTER_PORT=7077
-export SPARK_MASTER_WEBUI_PORT=18080
-
-export SPARK_WORKER_DIR=${SPARK_WORKER_DIR:-/var/run/spark/work}
-export SPARK_WORKER_PORT=7078
-export SPARK_WORKER_WEBUI_PORT=18081
-
-export SPARK_HISTORY_OPTS="$SPARK_HISTORY_OPTS -Dspark.history.fs.logDirectory=hdfs:///var/log/spark/apps -Dspark.history.ui.port=18082"
diff --git a/bigtop-packages/src/common/spark1/spark-history-server.svc b/bigtop-packages/src/common/spark1/spark-history-server.svc
deleted file mode 100644
index 0410d16..0000000
--- a/bigtop-packages/src/common/spark1/spark-history-server.svc
+++ /dev/null
@@ -1,70 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-TYPE="history-server"
-DAEMON="spark-${TYPE}"
-DESC="Spark ${TYPE}"
-EXEC_PATH="/usr/lib/spark/bin/spark-class"
-SVC_USER="spark"
-WORKING_DIR="/var/lib/spark"
-DAEMON_FLAGS=""
-CONF_DIR="/etc/spark/conf"
-PIDFILE="/var/run/spark/${DAEMON}.pid"
-
-generate_start() {
-
-cat <<'__EOT__'
-start() {
- [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
- log_success_msg "Starting $DESC (${DAEMON}): "
-
- checkstatusofproc
- status=$?
- if [ "$status" -eq "$STATUS_RUNNING" ]; then
- log_success_msg "${DESC} is running"
- exit 0
- fi
-
- LOG_FILE=/var/log/spark/${DAEMON}.out
-
- su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \
- ${EXEC_PATH} org.apache.spark.deploy.history.HistoryServer $DAEMON_FLAGS \
- > $LOG_FILE 2>&1 & "'echo $!' > "$PIDFILE"
-
- sleep 3
-
- checkstatusofproc
- RETVAL=$?
- [ $RETVAL -eq $STATUS_RUNNING ] && touch $LOCKFILE
- return $RETVAL
-}
-__EOT__
-
-}
-
-generate_stop() {
-
-cat <<'__EOT__'
-stop() {
- log_success_msg "Stopping $DESC (${DAEMON}): "
- killproc -p $PIDFILE java
- RETVAL=$?
-
- [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
- return $RETVAL
-}
-__EOT__
-
-}
diff --git a/bigtop-packages/src/common/spark1/spark-master.svc b/bigtop-packages/src/common/spark1/spark-master.svc
deleted file mode 100644
index 1823b99..0000000
--- a/bigtop-packages/src/common/spark1/spark-master.svc
+++ /dev/null
@@ -1,81 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-TYPE="master"
-DAEMON="spark-${TYPE}"
-DESC="Spark ${TYPE}"
-EXEC_PATH="/usr/lib/spark/bin/spark-class"
-SVC_USER="spark"
-WORKING_DIR="/var/lib/spark"
-DAEMON_FLAGS=""
-CONF_DIR="/etc/spark/conf"
-PIDFILE="/var/run/spark/${DAEMON}.pid"
-
-generate_start() {
-
-cat <<'__EOT__'
-start() {
- [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
- log_success_msg "Starting $DESC (${DAEMON}): "
-
- checkstatusofproc
- status=$?
- if [ "$status" -eq "$STATUS_RUNNING" ]; then
- log_success_msg "${DESC} is running"
- exit 0
- fi
-
- LOG_FILE=/var/log/spark/${DAEMON}.out
-
- if [ -f $CONF_DIR/spark-env.sh ]; then
- . $CONF_DIR/spark-env.sh
- fi
-
- if [ "$SPARK_MASTER_IP" = "" ]; then
- SPARK_MASTER_IP=`hostname`
- fi
-
-
- su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \
- ${EXEC_PATH} org.apache.spark.deploy.master.Master \
- --ip $SPARK_MASTER_IP $DAEMON_FLAGS \
- > $LOG_FILE 2>&1 & "'echo $!' > "$PIDFILE"
-
- sleep 3
-
- checkstatusofproc
- RETVAL=$?
- [ $RETVAL -eq $STATUS_RUNNING ] && touch $LOCKFILE
- return $RETVAL
-}
-__EOT__
-
-}
-
-generate_stop() {
-
-cat <<'__EOT__'
-stop() {
- log_success_msg "Stopping $DESC (${DAEMON}): "
- killproc -p $PIDFILE java
- RETVAL=$?
-
- [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
- return $RETVAL
-}
-__EOT__
-
-}
-
diff --git a/bigtop-packages/src/common/spark1/spark-thriftserver.svc b/bigtop-packages/src/common/spark1/spark-thriftserver.svc
deleted file mode 100644
index 39a9748..0000000
--- a/bigtop-packages/src/common/spark1/spark-thriftserver.svc
+++ /dev/null
@@ -1,75 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-TYPE="thriftserver"
-DAEMON="spark-${TYPE}"
-DESC="Spark ${TYPE}"
-EXEC_PATH="/usr/lib/spark/bin/spark-submit"
-SVC_USER="spark"
-WORKING_DIR="/var/lib/spark"
-DAEMON_FLAGS=""
-CONF_DIR="/etc/spark/conf"
-PIDFILE="/var/run/spark/${DAEMON}.pid"
-
-generate_start() {
-
-cat <<'__EOT__'
-start() {
- [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
- log_success_msg "Starting $DESC (${DAEMON}): "
-
- checkstatusofproc
- status=$?
- if [ "$status" -eq "$STATUS_RUNNING" ]; then
- log_success_msg "${DESC} is running"
- exit 0
- fi
-
- function usage {
- echo
- }
- export SUBMIT_USAGE_FUNCTION=usage
-
- LOG_FILE=/var/log/spark/${DAEMON}.out
-
- su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \
- ${EXEC_PATH} --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 1 $DAEMON_FLAGS \
- > $LOG_FILE 2>&1 & "'echo $!' > "$PIDFILE"
-
- sleep 7
-
- checkstatusofproc
- RETVAL=$?
- [ $RETVAL -eq $STATUS_RUNNING ] && touch $LOCKFILE
- return $RETVAL
-}
-__EOT__
-
-}
-
-generate_stop() {
-
-cat <<'__EOT__'
-stop() {
- log_success_msg "Stopping $DESC (${DAEMON}): "
- killproc -p $PIDFILE java
- RETVAL=$?
-
- [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
- return $RETVAL
-}
-__EOT__
-
-}
diff --git a/bigtop-packages/src/common/spark1/spark-worker.svc b/bigtop-packages/src/common/spark1/spark-worker.svc
deleted file mode 100644
index 7c12374..0000000
--- a/bigtop-packages/src/common/spark1/spark-worker.svc
+++ /dev/null
@@ -1,75 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-TYPE="worker"
-DAEMON="spark-${TYPE}"
-DESC="Spark ${TYPE}"
-EXEC_PATH="/usr/lib/spark/bin/spark-class"
-SVC_USER="spark"
-WORKING_DIR="/var/lib/spark"
-DAEMON_FLAGS=""
-CONF_DIR="/etc/spark/conf"
-PIDFILE="/var/run/spark/${DAEMON}.pid"
-
-generate_start() {
-
-cat <<'__EOT__'
-start() {
- [ -x $EXE_FILE ] || exit $ERROR_PROGRAM_NOT_INSTALLED
- log_success_msg "Starting $DESC (${DAEMON}): "
-
- checkstatusofproc
- status=$?
- if [ "$status" -eq "$STATUS_RUNNING" ]; then
- log_success_msg "${DESC} is running"
- exit 0
- fi
-
- LOG_FILE=/var/log/spark/${DAEMON}.out
-
- if [ -f $CONF_DIR/spark-env.sh ]; then
- . $CONF_DIR/spark-env.sh
- fi
-
- su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \
- ${EXEC_PATH} org.apache.spark.deploy.worker.Worker spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT $DAEMON_FLAGS \
- > $LOG_FILE 2>&1 & "'echo $!' > "$PIDFILE"
-
- sleep 3
-
- checkstatusofproc
- RETVAL=$?
- [ $RETVAL -eq $STATUS_RUNNING ] && touch $LOCKFILE
- return $RETVAL
-}
-__EOT__
-
-}
-
-generate_stop() {
-
-cat <<'__EOT__'
-stop() {
- log_success_msg "Stopping $DESC (${DAEMON}): "
- killproc -p $PIDFILE java
- RETVAL=$?
-
- [ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
- return $RETVAL
-}
-__EOT__
-
-}
-
diff --git a/bigtop-packages/src/deb/spark1/changelog b/bigtop-packages/src/deb/spark1/changelog
deleted file mode 100644
index 547ed02..0000000
--- a/bigtop-packages/src/deb/spark1/changelog
+++ /dev/null
@@ -1 +0,0 @@
---- This is auto-generated
diff --git a/bigtop-packages/src/deb/spark1/compat b/bigtop-packages/src/deb/spark1/compat
deleted file mode 100644
index ec63514..0000000
--- a/bigtop-packages/src/deb/spark1/compat
+++ /dev/null
@@ -1 +0,0 @@
-9
diff --git a/bigtop-packages/src/deb/spark1/control b/bigtop-packages/src/deb/spark1/control
deleted file mode 100644
index 7ebc970..0000000
--- a/bigtop-packages/src/deb/spark1/control
+++ /dev/null
@@ -1,77 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-Source: spark-core
-Section: misc
-Priority: extra
-Maintainer: Bigtop <de...@bigtop.apache.org>
-Build-Depends: debhelper (>= 6)
-Standards-Version: 3.8.0
-Homepage: http://spark.apache.org/
-
-Package: spark-core
-Architecture: all
-Depends: adduser, bigtop-utils (>= 0.7), hadoop-client
-Description: Lightning-Fast Cluster Computing
- Spark is a MapReduce-like cluster computing framework designed to support
- low-latency iterative jobs and interactive use from an interpreter. It is
- written in Scala, a high-level language for the JVM, and exposes a clean
- language-integrated syntax that makes it easy to write parallel jobs.
- Spark runs on top of the Apache Mesos cluster manager.
-
-Package: spark-master
-Architecture: all
-Depends: spark-core (= ${source:Version})
-Description: Server for Spark master
- Server for Spark master
-
-Package: spark-worker
-Architecture: all
-Depends: spark-core (= ${source:Version})
-Description: Server for Spark worker
- Server for Spark worker
-
-Package: spark-python
-Architecture: all
-Depends: spark-core (= ${source:Version}), python
-Description: Python client for Spark
- Includes PySpark, an interactive Python shell for Spark, and related libraries
-
-Package: spark-history-server
-Architecture: all
-Depends: spark-core (= ${source:Version})
-Description: History server for Apache Spark
- History server for Apache Spark
-
-Package: spark-thriftserver
-Architecture: all
-Depends: spark-core (= ${source:Version})
-Description: Thrift server for Spark SQL
- Thrift server for Spark SQL
-
-Package: spark-datanucleus
-Architecture: all
-Description: DataNucleus libraries for Apache Spark
- DataNucleus libraries used by Spark SQL with Hive Support
-
-Package: spark-extras
-Architecture: all
-Description: External/extra libraries for Apache Spark
- External/extra libraries built for Apache Spark but not included in the main assembly JAR (e.g., external streaming libraries)
-
-Package: spark-yarn-shuffle
-Architecture: all
-Description: Spark YARN Shuffle Service
- Spark YARN Shuffle Service
diff --git a/bigtop-packages/src/deb/spark1/copyright b/bigtop-packages/src/deb/spark1/copyright
deleted file mode 100644
index 3255f3d..0000000
--- a/bigtop-packages/src/deb/spark1/copyright
+++ /dev/null
@@ -1,15 +0,0 @@
-Format: http://dep.debian.net/deps/dep5
-Source: http://spark.apache.org/
-Upstream-Name: Spark Project
-
-Files: *
-Copyright: 2010-2011, The Spark Project
-License: Apache-2.0
-
-Files debian/*
-Copyright: 2011, The Apache Software Foundation
-License: Apache-2.0
-
-License: Apache-2.0
- On Debian systems, the complete text of the Apache 2.0 license
- can be found in "/usr/share/common-licenses/Apache-2.0".
diff --git a/bigtop-packages/src/deb/spark1/rules b/bigtop-packages/src/deb/spark1/rules
deleted file mode 100644
index 5881518..0000000
--- a/bigtop-packages/src/deb/spark1/rules
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/make -f
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# -*- makefile -*-
-
-# Uncomment this to turn on verbose mode.
-export DH_VERBOSE=1
-
-# This has to be exported to make some magic below work.
-export DH_OPTIONS
-
-%:
- dh $@
-
-override_dh_auto_build:
- bash debian/do-component-build
-
-svcs=spark-master spark-worker spark-history-server spark-thriftserver
-
-$(svcs): debian/init.d.tmpl
- bash $< debian/$@.svc deb debian/$@.init
- touch $@
-
-override_dh_auto_install: $(svcs)
- bash -x debian/install_spark.sh \
- --build-dir=`pwd` \
- --doc-dir=/usr/share/doc/spark \
- --source-dir=debian \
- --prefix=debian/tmp
diff --git a/bigtop-packages/src/deb/spark1/source/format b/bigtop-packages/src/deb/spark1/source/format
deleted file mode 100644
index 163aaf8..0000000
--- a/bigtop-packages/src/deb/spark1/source/format
+++ /dev/null
@@ -1 +0,0 @@
-3.0 (quilt)
diff --git a/bigtop-packages/src/deb/spark1/spark-core.install b/bigtop-packages/src/deb/spark1/spark-core.install
deleted file mode 100644
index 70f588c..0000000
--- a/bigtop-packages/src/deb/spark1/spark-core.install
+++ /dev/null
@@ -1,27 +0,0 @@
-/etc/spark
-/usr/bin/spark-class
-/usr/bin/spark-submit
-/usr/bin/spark-shell
-/usr/bin/spark-sql
-/usr/lib/spark/LICENSE
-/usr/lib/spark/RELEASE
-/usr/lib/spark/NOTICE
-/usr/lib/spark/bin/beeline
-/usr/lib/spark/bin/load-spark-env.sh
-/usr/lib/spark/bin/run-example
-/usr/lib/spark/bin/spark-class
-/usr/lib/spark/bin/spark-shell
-/usr/lib/spark/bin/spark-sql
-/usr/lib/spark/bin/spark-submit
-/usr/lib/spark/conf
-/usr/lib/spark/data
-/usr/lib/spark/examples
-/usr/lib/spark/lib/spark-assembly*.jar
-/usr/lib/spark/lib/spark-examples*.jar
-/usr/lib/spark/sbin
-/usr/lib/spark/work
-/usr/share/doc/spark*
-/var/lib/spark/
-/var/log/spark/
-/var/run/spark/
-/var/run/spark/work/
diff --git a/bigtop-packages/src/deb/spark1/spark-core.postinst b/bigtop-packages/src/deb/spark1/spark-core.postinst
deleted file mode 100644
index f6cca0a..0000000
--- a/bigtop-packages/src/deb/spark1/spark-core.postinst
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-case "$1" in
- configure)
- # Install config alternatives
- update-alternatives --install /etc/spark/conf spark-conf /etc/spark/conf.dist 30
- chown -R spark:spark /var/log/spark /var/lib/spark /var/run/spark
- ;;
-
- abort-upgrade|abort-remove|abort-deconfigure)
- ;;
-
- *)
- echo "postinst called with unknown argument \`$1'" >&2
- exit 1
- ;;
-esac
-
-#DEBHELPER#
diff --git a/bigtop-packages/src/deb/spark1/spark-core.preinst b/bigtop-packages/src/deb/spark1/spark-core.preinst
deleted file mode 100644
index d0b716f..0000000
--- a/bigtop-packages/src/deb/spark1/spark-core.preinst
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# preinst script for spark
-#
-# see: dh_installdeb(1)
-
-set -e
-
-# summary of how this script can be called:
-# * <new-preinst> `install'
-# * <new-preinst> `install' <old-version>
-# * <new-preinst> `upgrade' <old-version>
-# * <old-preinst> `abort-upgrade' <new-version>
-# for details, see http://www.debian.org/doc/debian-policy/ or
-# the debian-policy package
-
-
-case "$1" in
- install|upgrade)
- getent group spark >/dev/null || groupadd -r spark
- if ! getent passwd spark >/dev/null; then
- # Adding system user: spark .
- adduser \
- --system \
- --ingroup spark \
- --home /var/lib/spark \
- --gecos "Spark" \
- --shell /bin/false \
- spark >/dev/null
- fi
- ;;
-
- abort-upgrade)
- ;;
-
- *)
- echo "preinst called with unknown argument \`$1'" >&2
- exit 1
- ;;
-esac
-
-# dh_installdeb will replace this with shell code automatically
-# generated by other debhelper scripts.
-
-#DEBHELPER#
-
-exit 0
-
diff --git a/bigtop-packages/src/deb/spark1/spark-core.prerm b/bigtop-packages/src/deb/spark1/spark-core.prerm
deleted file mode 100644
index 8c40b7a..0000000
--- a/bigtop-packages/src/deb/spark1/spark-core.prerm
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -e
-
-case "$1" in
- remove|upgrade|deconfigure)
- update-alternatives --remove spark-conf /etc/spark/conf.dist || :
- ;;
-
- failed-upgrade)
- ;;
-
- *)
- echo "prerm called with unknown argument \`$1'" >&2
- exit 1
- ;;
-esac
-
-# dh_installdeb will replace this with shell code automatically
-# generated by other debhelper scripts.
-
-#DEBHELPER#
-
-exit 0
diff --git a/bigtop-packages/src/deb/spark1/spark-datanucleus.install b/bigtop-packages/src/deb/spark1/spark-datanucleus.install
deleted file mode 100644
index 69f619b..0000000
--- a/bigtop-packages/src/deb/spark1/spark-datanucleus.install
+++ /dev/null
@@ -1,2 +0,0 @@
-/usr/lib/spark/lib/datanucleus-*.jar
-/usr/lib/spark/yarn/lib/datanucleus-*.jar
diff --git a/bigtop-packages/src/deb/spark1/spark-extras.install b/bigtop-packages/src/deb/spark1/spark-extras.install
deleted file mode 100644
index c65fbc0..0000000
--- a/bigtop-packages/src/deb/spark1/spark-extras.install
+++ /dev/null
@@ -1 +0,0 @@
-/usr/lib/spark/extras/lib
diff --git a/bigtop-packages/src/deb/spark1/spark-python.install b/bigtop-packages/src/deb/spark1/spark-python.install
deleted file mode 100644
index 8211931..0000000
--- a/bigtop-packages/src/deb/spark1/spark-python.install
+++ /dev/null
@@ -1,4 +0,0 @@
-/usr/bin/pyspark
-/usr/lib/spark/bin/pyspark
-/usr/lib/spark/python
-
diff --git a/bigtop-packages/src/deb/spark1/spark-yarn-shuffle.install b/bigtop-packages/src/deb/spark1/spark-yarn-shuffle.install
deleted file mode 100644
index 79c799a..0000000
--- a/bigtop-packages/src/deb/spark1/spark-yarn-shuffle.install
+++ /dev/null
@@ -1,2 +0,0 @@
-/usr/lib/spark/lib/spark-*-yarn-shuffle.jar
-/usr/lib/spark/yarn/lib/spark-yarn-shuffle.jar
diff --git a/bigtop-packages/src/rpm/spark1/BUILD/.gitignore b/bigtop-packages/src/rpm/spark1/BUILD/.gitignore
deleted file mode 100644
index e69de29..0000000
diff --git a/bigtop-packages/src/rpm/spark1/RPMS/.gitignore b/bigtop-packages/src/rpm/spark1/RPMS/.gitignore
deleted file mode 100644
index e69de29..0000000
diff --git a/bigtop-packages/src/rpm/spark1/SOURCES/.gitignore b/bigtop-packages/src/rpm/spark1/SOURCES/.gitignore
deleted file mode 100644
index e69de29..0000000
diff --git a/bigtop-packages/src/rpm/spark1/SPECS/spark1.spec b/bigtop-packages/src/rpm/spark1/SPECS/spark1.spec
deleted file mode 100644
index 1d80adf..0000000
--- a/bigtop-packages/src/rpm/spark1/SPECS/spark1.spec
+++ /dev/null
@@ -1,255 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-%define spark_name spark
-%define lib_spark /usr/lib/%{spark_name}
-%define var_lib_spark /var/lib/%{spark_name}
-%define var_run_spark /var/run/%{spark_name}
-%define var_log_spark /var/log/%{spark_name}
-%define bin_spark /usr/lib/%{spark_name}/bin
-%define etc_spark /etc/%{spark_name}
-%define config_spark %{etc_spark}/conf
-%define bin /usr/bin
-%define man_dir /usr/share/man
-%define spark_services master worker history-server thriftserver
-
-%if %{?suse_version:1}0
-%define doc_spark %{_docdir}/spark
-%define alternatives_cmd update-alternatives
-%else
-%define doc_spark %{_docdir}/spark-%{spark1_version}
-%define alternatives_cmd alternatives
-%endif
-
-# disable repacking jars
-%define __os_install_post %{nil}
-
-Name: spark-core
-Version: %{spark1_version}
-Release: %{spark1_release}
-Summary: Lightning-Fast Cluster Computing
-URL: http://spark.apache.org/
-Group: Development/Libraries
-BuildArch: noarch
-Buildroot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX)
-License: ASL 2.0
-Source0: %{spark_name}-%{spark1_base_version}.tar.gz
-Source1: do-component-build
-Source2: install_%{spark_name}.sh
-Source3: spark-master.svc
-Source4: spark-worker.svc
-Source6: init.d.tmpl
-Source7: spark-history-server.svc
-Source8: spark-thriftserver.svc
-Source9: bigtop.bom
-Requires: bigtop-utils >= 0.7, hadoop-client
-Requires(preun): /sbin/service
-
-%global initd_dir %{_sysconfdir}/init.d
-
-%if %{?suse_version:1}0
-# Required for init scripts
-Requires: insserv
-%global initd_dir %{_sysconfdir}/rc.d
-
-%else
-# Required for init scripts
-Requires: /lib/lsb/init-functions
-
-%global initd_dir %{_sysconfdir}/rc.d/init.d
-
-%endif
-
-%description
-Spark is a MapReduce-like cluster computing framework designed to support
-low-latency iterative jobs and interactive use from an interpreter. It is
-written in Scala, a high-level language for the JVM, and exposes a clean
-language-integrated syntax that makes it easy to write parallel jobs.
-Spark runs on top of the Apache Mesos cluster manager.
-
-%package -n spark-master
-Summary: Server for Spark master
-Group: Development/Libraries
-Requires: spark-core = %{version}-%{release}
-
-%description -n spark-master
-Server for Spark master
-
-%package -n spark-worker
-Summary: Server for Spark worker
-Group: Development/Libraries
-Requires: spark-core = %{version}-%{release}
-
-%description -n spark-worker
-Server for Spark worker
-
-%package -n spark-python
-Summary: Python client for Spark
-Group: Development/Libraries
-Requires: spark-core = %{version}-%{release}, python
-
-%description -n spark-python
-Includes PySpark, an interactive Python shell for Spark, and related libraries
-
-%package -n spark-history-server
-Summary: History server for Apache Spark
-Group: Development/Libraries
-Requires: spark-core = %{version}-%{release}
-
-%description -n spark-history-server
-History server for Apache Spark
-
-%package -n spark-thriftserver
-Summary: Thrift server for Spark SQL
-Group: Development/Libraries
-Requires: spark-core = %{version}-%{release}
-
-%description -n spark-thriftserver
-Thrift server for Spark SQL
-
-%package -n spark-datanucleus
-Summary: DataNucleus libraries for Apache Spark
-Group: Development/Libraries
-
-%description -n spark-datanucleus
-DataNucleus libraries used by Spark SQL with Hive Support
-
-%package -n spark-extras
-Summary: External/extra libraries for Apache Spark
-Group: Development/Libraries
-
-%description -n spark-extras
-External/extra libraries built for Apache Spark but not included in the main
-assembly JAR (e.g., external streaming libraries)
-
-%package -n spark-yarn-shuffle
-Summary: Spark YARN Shuffle Service
-Group: Development/Libraries
-
-%description -n spark-yarn-shuffle
-Spark YARN Shuffle Service
-
-%prep
-%setup -n %{spark_name}-%{spark1_base_version}
-
-%build
-bash $RPM_SOURCE_DIR/do-component-build
-
-%install
-%__rm -rf $RPM_BUILD_ROOT
-%__install -d -m 0755 $RPM_BUILD_ROOT/%{initd_dir}/
-
-bash $RPM_SOURCE_DIR/install_spark.sh \
- --build-dir=`pwd` \
- --source-dir=$RPM_SOURCE_DIR \
- --prefix=$RPM_BUILD_ROOT \
- --doc-dir=%{doc_spark} \
- --pyspark-python=python
-
-for service in %{spark_services}
-do
- # Install init script
- init_file=$RPM_BUILD_ROOT/%{initd_dir}/%{spark_name}-${service}
- bash $RPM_SOURCE_DIR/init.d.tmpl $RPM_SOURCE_DIR/spark-${service}.svc rpm $init_file
-done
-
-%pre
-getent group spark >/dev/null || groupadd -r spark
-getent passwd spark >/dev/null || useradd -c "Spark" -s /sbin/nologin -g spark -r -d %{var_lib_spark} spark 2> /dev/null || :
-
-%post
-%{alternatives_cmd} --install %{config_spark} %{spark_name}-conf %{config_spark}.dist 30
-
-%preun
-if [ "$1" = 0 ]; then
- %{alternatives_cmd} --remove %{spark_name}-conf %{config_spark}.dist || :
-fi
-
-for service in %{spark_services}; do
- /sbin/service %{spark_name}-${service} status > /dev/null 2>&1
- if [ $? -eq 0 ]; then
- /sbin/service %{spark_name}-${service} stop > /dev/null 2>&1
- fi
-done
-
-#######################
-#### FILES SECTION ####
-#######################
-%files
-%defattr(-,root,root,755)
-%config(noreplace) %{config_spark}.dist
-%doc %{doc_spark}
-%{lib_spark}/conf
-%{lib_spark}/LICENSE
-%{lib_spark}/RELEASE
-%{lib_spark}/NOTICE
-%{lib_spark}/bin
-%{lib_spark}/lib
-%exclude %{lib_spark}/lib/datanucleus-*.jar
-%exclude %{lib_spark}/lib/spark-*-yarn-shuffle.jar
-%{lib_spark}/sbin
-%{lib_spark}/data
-%{lib_spark}/examples
-%{lib_spark}/work
-%exclude %{bin_spark}/pyspark
-%exclude %{lib_spark}/python
-%{etc_spark}
-%attr(0755,spark,spark) %{var_lib_spark}
-%attr(0755,spark,spark) %{var_run_spark}
-%attr(0755,spark,spark) %{var_log_spark}
-%{bin}/spark-class
-%{bin}/spark-shell
-%{bin}/spark-sql
-%{bin}/spark-submit
-
-%files -n spark-python
-%defattr(-,root,root,755)
-%attr(0755,root,root) %{bin}/pyspark
-%attr(0755,root,root) %{lib_spark}/bin/pyspark
-%{lib_spark}/python
-
-%files -n spark-datanucleus
-%defattr(-,root,root,755)
-%{lib_spark}/lib/datanucleus-*.jar
-%{lib_spark}/yarn/lib/datanucleus-*.jar
-
-%files -n spark-extras
-%defattr(-,root,root,755)
-%{lib_spark}/extras
-
-%files -n spark-yarn-shuffle
-%defattr(-,root,root,755)
-%{lib_spark}/lib/spark-*-yarn-shuffle.jar
-%{lib_spark}/yarn/lib/spark-yarn-shuffle.jar
-
-%define service_macro() \
-%files -n %1 \
-%attr(0755,root,root)/%{initd_dir}/%1 \
-%post -n %1 \
-chkconfig --add %1 \
-\
-%preun -n %1 \
-if [ $1 = 0 ] ; then \
- service %1 stop > /dev/null 2>&1 \
- chkconfig --del %1 \
-fi \
-%postun -n %1 \
-if [ $1 -ge 1 ]; then \
- service %1 condrestart >/dev/null 2>&1 \
-fi
-%service_macro spark-master
-%service_macro spark-worker
-%service_macro spark-history-server
-%service_macro spark-thriftserver
diff --git a/bigtop-packages/src/rpm/spark1/SRPMS/.gitignore b/bigtop-packages/src/rpm/spark1/SRPMS/.gitignore
deleted file mode 100644
index e69de29..0000000
diff --git a/bigtop.bom b/bigtop.bom
index 2835289..8ee9ff7 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -111,7 +111,7 @@ bigtop {
dependencies = [
zookeeper:['hadoop', 'hbase'],
hadoop:['ignite-hadoop', 'hbase', 'crunch', 'hive', 'tez', 'sqoop', 'sqoop2',
- 'oozie', 'mahout', 'flume', 'giraph', 'solr', 'spark','spark1',
+ 'oozie', 'mahout', 'flume', 'giraph', 'solr', 'spark',
'phoenix', 'alluxio', 'kafka', 'ycsb', 'hama', 'zeppelin',
'tajo', 'apex'
],
@@ -272,17 +272,6 @@ bigtop {
site = "${apache.APACHE_MIRROR}/${download_path}"
archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
}
- 'spark1' {
- name = 'spark1'
- pkg = 'spark-core'
- relNotes = 'Apache Spark'
- version { base = '1.6.2'; pkg = base; release = 1 }
- tarball { destination = "spark-${version.base}.tar.gz"
- source = "spark-${version.base}.tgz" }
- url { download_path = "/spark/spark-${version.base}"
- site = "${apache.APACHE_MIRROR}/${download_path}"
- archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
- }
'spark' {
name = 'spark'
pkg = 'spark-core'