You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by of...@apache.org on 2016/11/14 20:10:05 UTC
bigtop git commit: BIGTOP-2569. Upgrade to Spark 2.0.1
Repository: bigtop
Updated Branches:
refs/heads/master 037da0d76 -> 71df9df1f
BIGTOP-2569. Upgrade to Spark 2.0.1
This includes various packaging changes, mostly due to the fact that Spark 2.0
no longer builds a Spark assembly JAR.
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/71df9df1
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/71df9df1
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/71df9df1
Branch: refs/heads/master
Commit: 71df9df1f8b1a60befb6da827cd743a52b62b424
Parents: 037da0d
Author: Jonathan Kelly <jo...@amazon.com>
Authored: Mon Nov 14 09:49:32 2016 -0800
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Mon Nov 14 21:08:46 2016 +0100
----------------------------------------------------------------------
.../puppet/modules/spark/manifests/init.pp | 2 +-
.../src/common/spark/do-component-build | 19 ++--
.../src/common/spark/install_spark.sh | 93 ++++++++------------
bigtop-packages/src/common/spark/patch0.diff | 11 ---
bigtop-packages/src/deb/spark/control | 6 +-
.../src/deb/spark/spark-core.install | 8 +-
.../src/deb/spark/spark-datanucleus.install | 2 +-
.../src/deb/spark/spark-external.install | 1 +
.../src/deb/spark/spark-extras.install | 1 -
.../src/deb/spark/spark-yarn-shuffle.install | 2 +-
bigtop-packages/src/rpm/spark/SPECS/spark.spec | 41 ++++-----
bigtop.bom | 6 +-
12 files changed, 83 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71df9df1/bigtop-deploy/puppet/modules/spark/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/spark/manifests/init.pp b/bigtop-deploy/puppet/modules/spark/manifests/init.pp
index 5437782..7bc4a5b 100644
--- a/bigtop-deploy/puppet/modules/spark/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/spark/manifests/init.pp
@@ -49,7 +49,7 @@ class spark {
require => Package['spark-core'],
}
- package { 'spark-extras':
+ package { 'spark-external':
ensure => latest,
require => Package['spark-core'],
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71df9df1/bigtop-packages/src/common/spark/do-component-build
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/spark/do-component-build b/bigtop-packages/src/common/spark/do-component-build
index 8d74f4a..7f2c540 100644
--- a/bigtop-packages/src/common/spark/do-component-build
+++ b/bigtop-packages/src/common/spark/do-component-build
@@ -25,19 +25,20 @@ BUILD_OPTS="-Divy.home=${HOME}/.ivy2 -Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${
-Dyarn.version=$HADOOP_VERSION \
-Dprotobuf.version=2.5.0 \
-DrecompileMode=all \
- -Pbigtop-dist \
-Pyarn -Phadoop-2.6 \
-Phive -Phive-thriftserver \
$SPARK_BUILD_OPTS"
-## this might be an issue at times
-# http://maven.40175.n5.nabble.com/Not-finding-artifact-in-local-repo-td3727753.html
-export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=512m -XX:PermSize=1024m -XX:MaxPermSize=1024m"
+./dev/make-distribution.sh $BUILD_OPTS -DskipTests
-mvn $BUILD_OPTS -DskipTests install
+SPARK_SKIP_TESTS=$([ "$SPARK_RUN_TESTS" = "true" ] && echo false || echo true)
-# Tests must be run after Spark has already been packaged.
+# make-distribution.sh will only run "mvn clean package", so in order to get the
+# Spark packages installed in the local Maven repository (or to run the tests),
+# we need to run "mvn install" again. However, it will be relatively fast because
+# we are not running the "clean" phase.
+#
+# This is also the point that we can run the tests if desired, since tests must
+# be run after Spark has already been packaged.
# See http://spark.apache.org/docs/latest/building-spark.html#spark-tests-in-maven
-if [ "$SPARK_RUN_TESTS" = "true" ]; then
- mvn $BUILD_OPTS test
-fi
+mvn $BUILD_OPTS install -DskipTests=$SPARK_SKIP_TESTS
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71df9df1/bigtop-packages/src/common/spark/install_spark.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/spark/install_spark.sh b/bigtop-packages/src/common/spark/install_spark.sh
index 530b355..d4f8405 100644
--- a/bigtop-packages/src/common/spark/install_spark.sh
+++ b/bigtop-packages/src/common/spark/install_spark.sh
@@ -30,8 +30,6 @@ usage: $0 <options>
--lib-dir=DIR path to install Spark home [/usr/lib/spark]
--installed-lib-dir=DIR path where lib-dir will end up on target system
--bin-dir=DIR path to install bins [/usr/bin]
- --examples-dir=DIR path to install examples [doc-dir/examples]
- --pyspark-python executable to use for Python interpreter [python]
... [ see source for more similar options ]
"
exit 1
@@ -46,8 +44,6 @@ OPTS=$(getopt \
-l 'installed-lib-dir:' \
-l 'bin-dir:' \
-l 'source-dir:' \
- -l 'examples-dir:' \
- -l 'pyspark-python:' \
-l 'build-dir:' -- "$@")
if [ $? != 0 ] ; then
@@ -78,12 +74,6 @@ while true ; do
--bin-dir)
BIN_DIR=$2 ; shift 2
;;
- --examples-dir)
- EXAMPLES_DIR=$2 ; shift 2
- ;;
- --pyspark-python)
- PYSPARK_PYTHON=$2 ; shift 2
- ;;
--)
shift ; break
;;
@@ -106,67 +96,53 @@ if [ -f "$SOURCE_DIR/bigtop.bom" ]; then
. $SOURCE_DIR/bigtop.bom
fi
+DIST_DIR=${BUILD_DIR}/dist
MAN_DIR=${MAN_DIR:-/usr/share/man/man1}
DOC_DIR=${DOC_DIR:-/usr/share/doc/spark}
LIB_DIR=${LIB_DIR:-/usr/lib/spark}
INSTALLED_LIB_DIR=${INSTALLED_LIB_DIR:-/usr/lib/spark}
-EXAMPLES_DIR=${EXAMPLES_DIR:-$DOC_DIR/examples}
BIN_DIR=${BIN_DIR:-/usr/bin}
CONF_DIR=${CONF_DIR:-/etc/spark/conf.dist}
PYSPARK_PYTHON=${PYSPARK_PYTHON:-python}
install -d -m 0755 $PREFIX/$LIB_DIR
-install -d -m 0755 $PREFIX/$LIB_DIR/lib
-install -d -m 0755 $PREFIX/$LIB_DIR/bin
-install -d -m 0755 $PREFIX/$LIB_DIR/sbin
-install -d -m 0755 $PREFIX/$LIB_DIR/extras
-install -d -m 0755 $PREFIX/$LIB_DIR/extras/lib
-install -d -m 0755 $PREFIX/$LIB_DIR/yarn
+install -d -m 0755 $PREFIX/$LIB_DIR/external/lib
install -d -m 0755 $PREFIX/$LIB_DIR/yarn/lib
+install -d -m 0755 $PREFIX/$CONF_DIR
install -d -m 0755 $PREFIX/$DOC_DIR
-install -d -m 0755 $PREFIX/$EXAMPLES_DIR
install -d -m 0755 $PREFIX/var/lib/spark/
install -d -m 0755 $PREFIX/var/log/spark/
install -d -m 0755 $PREFIX/var/run/spark/
install -d -m 0755 $PREFIX/var/run/spark/work/
-tar --wildcards -C $PREFIX/$LIB_DIR/lib -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz spark-assembly\*
-tar --wildcards --strip-components=1 -C $PREFIX/$LIB_DIR/lib -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz \*datanucleus\*
-tar --wildcards -C $PREFIX/$LIB_DIR/ -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz bin\*
-tar --wildcards -C $PREFIX/$LIB_DIR/ -zxf ${BUILD_DIR}/assembly/target/spark-assembly*-dist.tar.gz sbin\*
-
-rm -rf $PREFIX/$LIB_DIR/bin/*.cmd
-
-# External/extra jars
-ls ${BUILD_DIR}/{external,extras}/*/target/*${SPARK_VERSION}.jar | grep -v 'original-\|assembly' | xargs -IJARS cp JARS $PREFIX/$LIB_DIR/extras/lib
+rm $DIST_DIR/bin/*.cmd
+cp -r $DIST_DIR/* $PREFIX/$LIB_DIR
-# Examples jar
-cp ${BUILD_DIR}/examples/target/spark-examples*${SPARK_VERSION}.jar $PREFIX/$LIB_DIR/lib/spark-examples-${SPARK_VERSION}-hadoop${HADOOP_VERSION}.jar
+ln -s $LIB_DIR/examples $PREFIX/$DOC_DIR/
-# Spark YARN Shuffle jar
-cp ${BUILD_DIR}/network/yarn/target/*/spark-${SPARK_VERSION}-yarn-shuffle.jar $PREFIX/$LIB_DIR/lib/
-
-# Examples src
-cp -ra ${BUILD_DIR}/examples/src $PREFIX/$EXAMPLES_DIR/
-ln -s $EXAMPLES_DIR $PREFIX/$LIB_DIR/examples
-
-# Data
-cp -ra ${BUILD_DIR}/data $PREFIX/$LIB_DIR/
-
-chmod 755 $PREFIX/$LIB_DIR/bin/*
-chmod 755 $PREFIX/$LIB_DIR/sbin/*
+# External libraries not included in dist
+find_external_modules() {
+ # If corresponding assembly sub-module exists, copy the assembly jar instead of the non-assembly jar
+ find ${BUILD_DIR}/external -mindepth 1 -maxdepth 1 -not -name '*-assembly' \
+ -exec sh -c '([ -d "$1/target" ] && echo "$1") || ([ -d "$0/target" ] && echo "$0")' {} {}-assembly \;
+}
+copy_external_jars() {
+ xargs -L1 -r -IMODULE find MODULE/target -name "*${SPARK_VERSION}.jar" -and -not -name "original-*" \
+ -exec cp {} $PREFIX/$LIB_DIR/external/lib \;
+}
+find_external_modules | copy_external_jars
-# Copy in the configuration files
-install -d -m 0755 $PREFIX/$CONF_DIR
-cp -a ${BUILD_DIR}/conf/* $PREFIX/$CONF_DIR
+# Move the configuration files to the correct location
+mv $PREFIX/$LIB_DIR/conf/* $PREFIX/$CONF_DIR
cp $SOURCE_DIR/spark-env.sh $PREFIX/$CONF_DIR
+rmdir $PREFIX/$LIB_DIR/conf
ln -s /etc/spark/conf $PREFIX/$LIB_DIR/conf
# Copy in the wrappers
install -d -m 0755 $PREFIX/$BIN_DIR
for wrap in bin/spark-class bin/spark-shell bin/spark-sql bin/spark-submit; do
- cat > $PREFIX/$BIN_DIR/`basename $wrap` <<EOF
+ cat > $PREFIX/$BIN_DIR/$(basename $wrap) <<EOF
#!/bin/bash
# Autodetect JAVA_HOME if not defined
@@ -174,12 +150,11 @@ for wrap in bin/spark-class bin/spark-shell bin/spark-sql bin/spark-submit; do
exec $INSTALLED_LIB_DIR/$wrap "\$@"
EOF
- chmod 755 $PREFIX/$BIN_DIR/`basename $wrap`
+ chmod 755 $PREFIX/$BIN_DIR/$(basename $wrap)
done
ln -s /var/run/spark/work $PREFIX/$LIB_DIR/work
-cp -r ${BUILD_DIR}/python ${PREFIX}/${INSTALLED_LIB_DIR}/
rm -f ${PREFIX}/${INSTALLED_LIB_DIR}/python/.gitignore
cat > $PREFIX/$BIN_DIR/pyspark <<EOF
#!/bin/bash
@@ -193,19 +168,29 @@ exec $INSTALLED_LIB_DIR/bin/pyspark "\$@"
EOF
chmod 755 $PREFIX/$BIN_DIR/pyspark
+cat > $PREFIX/$BIN_DIR/spark-example <<EOF
+#!/bin/bash
+
+# Autodetect JAVA_HOME if not defined
+. /usr/lib/bigtop-utils/bigtop-detect-javahome
+
+exec $INSTALLED_LIB_DIR/bin/run-example "\$@"
+EOF
+chmod 755 $PREFIX/$BIN_DIR/spark-example
+
touch $PREFIX/$LIB_DIR/RELEASE
cp ${BUILD_DIR}/{LICENSE,NOTICE} ${PREFIX}/${LIB_DIR}/
# Version-less symlinks
-(cd $PREFIX/$LIB_DIR/lib; ln -s spark-assembly*.jar spark-assembly.jar; ln -s spark-examples*.jar spark-examples.jar)
+(cd $PREFIX/$LIB_DIR/examples/jars; ln -s spark-examples*.jar spark-examples.jar)
pushd $PREFIX/$LIB_DIR/yarn/lib
-ln -s ../../lib/spark-*-yarn-shuffle.jar spark-yarn-shuffle.jar
-ln -s ../../lib/datanucleus-api-jdo*.jar datanucleus-api-jdo.jar
-ln -s ../../lib/datanucleus-core*.jar datanucleus-core.jar
-ln -s ../../lib/datanucleus-rdbms*.jar datanucleus-rdbms.jar
+ln -s ../spark-*-yarn-shuffle.jar spark-yarn-shuffle.jar
+ln -s ../../jars/datanucleus-api-jdo*.jar datanucleus-api-jdo.jar
+ln -s ../../jars/datanucleus-core*.jar datanucleus-core.jar
+ln -s ../../jars/datanucleus-rdbms*.jar datanucleus-rdbms.jar
popd
-pushd $PREFIX/$LIB_DIR/extras/lib
+pushd $PREFIX/$LIB_DIR/external/lib
for j in $(ls *.jar); do
- ln -s $j $(echo $j | sed -n 's/\(.*\)\(_[0-9.]\+-[0-9.]\+\)\(.jar\)/\1\3/p')
+ ln -s $j $(echo $j | sed -n 's/\(.*\)\(_.\+\)\(.jar\)/\1\3/p')
done
popd
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71df9df1/bigtop-packages/src/common/spark/patch0.diff
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/spark/patch0.diff b/bigtop-packages/src/common/spark/patch0.diff
deleted file mode 100644
index 5e1aefb..0000000
--- a/bigtop-packages/src/common/spark/patch0.diff
+++ /dev/null
@@ -1,11 +0,0 @@
---- spark-1.6.2/pom.xml 2016-06-21 20:56:39.000000000 -0500
-+++ spark-1.6.2-patched/pom.xml 2016-11-01 21:46:37.629234248 -0500
-@@ -2511,7 +2511,7 @@
- <profile>
- <id>hadoop-2.6</id>
- <properties>
-- <hadoop.version>2.6.0</hadoop.version>
-+ <hadoop.version>${hadoop.version}</hadoop.version>
- <jets3t.version>0.9.3</jets3t.version>
- <zookeeper.version>3.4.6</zookeeper.version>
- <curator.version>2.6.0</curator.version>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71df9df1/bigtop-packages/src/deb/spark/control
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/control b/bigtop-packages/src/deb/spark/control
index 7ebc970..4b7f2ff 100644
--- a/bigtop-packages/src/deb/spark/control
+++ b/bigtop-packages/src/deb/spark/control
@@ -66,10 +66,10 @@ Architecture: all
Description: DataNucleus libraries for Apache Spark
DataNucleus libraries used by Spark SQL with Hive Support
-Package: spark-extras
+Package: spark-external
Architecture: all
-Description: External/extra libraries for Apache Spark
- External/extra libraries built for Apache Spark but not included in the main assembly JAR (e.g., external streaming libraries)
+Description: External libraries for Apache Spark
+ External libraries built for Apache Spark but not included in the main distribution (e.g., external streaming libraries)
Package: spark-yarn-shuffle
Architecture: all
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71df9df1/bigtop-packages/src/deb/spark/spark-core.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/spark-core.install b/bigtop-packages/src/deb/spark/spark-core.install
index 70f588c..fb9bee3 100644
--- a/bigtop-packages/src/deb/spark/spark-core.install
+++ b/bigtop-packages/src/deb/spark/spark-core.install
@@ -1,9 +1,11 @@
/etc/spark
/usr/bin/spark-class
-/usr/bin/spark-submit
+/usr/bin/spark-example
/usr/bin/spark-shell
/usr/bin/spark-sql
+/usr/bin/spark-submit
/usr/lib/spark/LICENSE
+/usr/lib/spark/README.md
/usr/lib/spark/RELEASE
/usr/lib/spark/NOTICE
/usr/lib/spark/bin/beeline
@@ -16,8 +18,8 @@
/usr/lib/spark/conf
/usr/lib/spark/data
/usr/lib/spark/examples
-/usr/lib/spark/lib/spark-assembly*.jar
-/usr/lib/spark/lib/spark-examples*.jar
+/usr/lib/spark/jars
+/usr/lib/spark/licenses
/usr/lib/spark/sbin
/usr/lib/spark/work
/usr/share/doc/spark*
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71df9df1/bigtop-packages/src/deb/spark/spark-datanucleus.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/spark-datanucleus.install b/bigtop-packages/src/deb/spark/spark-datanucleus.install
index 69f619b..6471fad 100644
--- a/bigtop-packages/src/deb/spark/spark-datanucleus.install
+++ b/bigtop-packages/src/deb/spark/spark-datanucleus.install
@@ -1,2 +1,2 @@
-/usr/lib/spark/lib/datanucleus-*.jar
+/usr/lib/spark/jars/datanucleus-*.jar
/usr/lib/spark/yarn/lib/datanucleus-*.jar
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71df9df1/bigtop-packages/src/deb/spark/spark-external.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/spark-external.install b/bigtop-packages/src/deb/spark/spark-external.install
new file mode 100644
index 0000000..ec2af71
--- /dev/null
+++ b/bigtop-packages/src/deb/spark/spark-external.install
@@ -0,0 +1 @@
+/usr/lib/spark/external/lib
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71df9df1/bigtop-packages/src/deb/spark/spark-extras.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/spark-extras.install b/bigtop-packages/src/deb/spark/spark-extras.install
deleted file mode 100644
index c65fbc0..0000000
--- a/bigtop-packages/src/deb/spark/spark-extras.install
+++ /dev/null
@@ -1 +0,0 @@
-/usr/lib/spark/extras/lib
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71df9df1/bigtop-packages/src/deb/spark/spark-yarn-shuffle.install
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/spark-yarn-shuffle.install b/bigtop-packages/src/deb/spark/spark-yarn-shuffle.install
index 79c799a..d7bbb99 100644
--- a/bigtop-packages/src/deb/spark/spark-yarn-shuffle.install
+++ b/bigtop-packages/src/deb/spark/spark-yarn-shuffle.install
@@ -1,2 +1,2 @@
-/usr/lib/spark/lib/spark-*-yarn-shuffle.jar
+/usr/lib/spark/yarn/spark-*-yarn-shuffle.jar
/usr/lib/spark/yarn/lib/spark-yarn-shuffle.jar
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71df9df1/bigtop-packages/src/rpm/spark/SPECS/spark.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
index daecc72..77dc252 100644
--- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec
+++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
@@ -126,13 +126,13 @@ Group: Development/Libraries
%description -n spark-datanucleus
DataNucleus libraries used by Spark SQL with Hive Support
-%package -n spark-extras
-Summary: External/extra libraries for Apache Spark
+%package -n spark-external
+Summary: External libraries for Apache Spark
Group: Development/Libraries
-%description -n spark-extras
-External/extra libraries built for Apache Spark but not included in the main
-assembly JAR (e.g., external streaming libraries)
+%description -n spark-external
+External libraries built for Apache Spark but not included in the main
+distribution (e.g., external streaming libraries)
%package -n spark-yarn-shuffle
Summary: Spark YARN Shuffle Service
@@ -191,28 +191,25 @@ done
%defattr(-,root,root,755)
%config(noreplace) %{config_spark}.dist
%doc %{doc_spark}
-%{lib_spark}/conf
%{lib_spark}/LICENSE
-%{lib_spark}/RELEASE
%{lib_spark}/NOTICE
-%{lib_spark}/bin
-%{lib_spark}/lib
-%exclude %{lib_spark}/lib/datanucleus-*.jar
-%exclude %{lib_spark}/lib/spark-*-yarn-shuffle.jar
-%{lib_spark}/sbin
+%{lib_spark}/README.md
+%{lib_spark}/RELEASE
+%{bin_spark}
+%exclude %{bin_spark}/pyspark
+%{lib_spark}/conf
%{lib_spark}/data
%{lib_spark}/examples
+%{lib_spark}/jars
+%exclude %{lib_spark}/jars/datanucleus-*.jar
+%{lib_spark}/licenses
+%{lib_spark}/sbin
%{lib_spark}/work
-%exclude %{bin_spark}/pyspark
-%exclude %{lib_spark}/python
%{etc_spark}
%attr(0755,spark,spark) %{var_lib_spark}
%attr(0755,spark,spark) %{var_run_spark}
%attr(0755,spark,spark) %{var_log_spark}
-%{bin}/spark-class
-%{bin}/spark-shell
-%{bin}/spark-sql
-%{bin}/spark-submit
+%{bin}/spark-*
%files -n spark-python
%defattr(-,root,root,755)
@@ -222,16 +219,16 @@ done
%files -n spark-datanucleus
%defattr(-,root,root,755)
-%{lib_spark}/lib/datanucleus-*.jar
+%{lib_spark}/jars/datanucleus-*.jar
%{lib_spark}/yarn/lib/datanucleus-*.jar
-%files -n spark-extras
+%files -n spark-external
%defattr(-,root,root,755)
-%{lib_spark}/extras
+%{lib_spark}/external
%files -n spark-yarn-shuffle
%defattr(-,root,root,755)
-%{lib_spark}/lib/spark-*-yarn-shuffle.jar
+%{lib_spark}/yarn/spark-*-yarn-shuffle.jar
%{lib_spark}/yarn/lib/spark-yarn-shuffle.jar
%define service_macro() \
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71df9df1/bigtop.bom
----------------------------------------------------------------------
diff --git a/bigtop.bom b/bigtop.bom
index 3d90283..d9bbab4 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -73,7 +73,7 @@ bigtop {
version = "1.2.0-SNAPSHOT"
stack {
'jdk' { version = '1.7'; version_base = version }
- 'scala' { version = '2.10.4'; version_base = version }
+ 'scala' { version = '2.11.8'; version_base = version }
}
apache {
APACHE_MIRROR = "https://www-us.apache.org"
@@ -299,10 +299,10 @@ bigtop {
name = 'spark'
pkg = 'spark-core'
relNotes = 'Apache Spark'
- version { base = '1.6.2'; pkg = base; release = 1 }
+ version { base = '2.0.1'; pkg = base; release = 1 }
tarball { destination = "$name-${version.base}.tar.gz"
source = "$name-${version.base}.tgz" }
- url { download_path = "/spark/spark-${version.base}"
+ url { download_path = "/$name/$name-${version.base}"
site = "${apache.APACHE_MIRROR}/${download_path}"
archive = "${apache.APACHE_ARCHIVE}/${download_path}" }
}