You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ev...@apache.org on 2017/07/01 13:37:22 UTC
[38/50] [abbrv] bigtop git commit: BIGTOP-2807: Upgrade Spark to 2.1.1
BIGTOP-2807: Upgrade Spark to 2.1.1
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/1bb35133
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/1bb35133
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/1bb35133
Branch: refs/heads/branch-1.2
Commit: 1bb35133ea7c04a684a9d1776f0afa791b3d9ab1
Parents: e2a1937
Author: Youngwoo Kim <yw...@apache.org>
Authored: Mon Jun 12 13:21:29 2017 +0900
Committer: Evans Ye <ev...@apache.org>
Committed: Sat Jul 1 12:52:59 2017 +0000
----------------------------------------------------------------------
.../src/common/spark/do-component-build | 2 +-
bigtop-packages/src/deb/spark/control | 2 +-
bigtop-packages/src/deb/spark/rules | 21 ++++++++++++++++++++
bigtop-packages/src/rpm/spark/SPECS/spark.spec | 21 +++++++++++++++++++-
bigtop.bom | 2 +-
5 files changed, 44 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/1bb35133/bigtop-packages/src/common/spark/do-component-build
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/spark/do-component-build b/bigtop-packages/src/common/spark/do-component-build
index 21dffbe..3b79b0d 100644
--- a/bigtop-packages/src/common/spark/do-component-build
+++ b/bigtop-packages/src/common/spark/do-component-build
@@ -28,7 +28,7 @@ BUILD_OPTS="-Divy.home=${HOME}/.ivy2 -Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${
-Dyarn.version=$HADOOP_VERSION \
-Dprotobuf.version=2.5.0 \
-DrecompileMode=all \
- -Pyarn -Phadoop-2.6 \
+ -Pyarn -Phadoop-2.7 \
-Phive -Phive-thriftserver \
$SPARK_BUILD_OPTS"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/1bb35133/bigtop-packages/src/deb/spark/control
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/control b/bigtop-packages/src/deb/spark/control
index 4b7f2ff..01a393d 100644
--- a/bigtop-packages/src/deb/spark/control
+++ b/bigtop-packages/src/deb/spark/control
@@ -23,7 +23,7 @@ Homepage: http://spark.apache.org/
Package: spark-core
Architecture: all
-Depends: adduser, bigtop-utils (>= 0.7), hadoop-client
+Depends: adduser, bigtop-utils (>= 0.7), hadoop-client, hadoop-yarn
Description: Lightning-Fast Cluster Computing
Spark is a MapReduce-like cluster computing framework designed to support
low-latency iterative jobs and interactive use from an interpreter. It is
http://git-wip-us.apache.org/repos/asf/bigtop/blob/1bb35133/bigtop-packages/src/deb/spark/rules
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/rules b/bigtop-packages/src/deb/spark/rules
index 0bc7fc7..d5fdb6d 100644
--- a/bigtop-packages/src/deb/spark/rules
+++ b/bigtop-packages/src/deb/spark/rules
@@ -22,6 +22,10 @@ export DH_VERBOSE=1
# This has to be exported to make some magic below work.
export DH_OPTIONS
+lib_spark=/usr/lib/spark
+lib_hadoop_client=/usr/lib/hadoop/client
+lib_hadoop_yarn=/usr/lib/hadoop-yarn/
+
%:
dh $@
@@ -45,3 +49,20 @@ override_dh_install:
dh_install
# BIGTOP-2588 drop datanucleus jars from spark-core
rm -Rf debian/spark-core/usr/lib/spark/jars/datanucleus*
+
+ rm -f debian/tmp/${lib_spark}/jars/hadoop-*.jar
+ ln -s ${lib_hadoop_client}/hadoop-annotations.jar debian/tmp/${lib_spark}/jars/
+ ln -s ${lib_hadoop_client}/hadoop-auth.jar debian/tmp/${lib_spark}/jars/
+ ln -s ${lib_hadoop_client}/hadoop-client.jar debian/tmp/${lib_spark}/jars/
+ ln -s ${lib_hadoop_client}/hadoop-common.jar debian/tmp/${lib_spark}/jars/
+ ln -s ${lib_hadoop_client}/hadoop-hdfs.jar debian/tmp/${lib_spark}/jars/
+ ln -s ${lib_hadoop_client}/hadoop-mapreduce-client-app.jar debian/tmp/${lib_spark}/jars/
+ ln -s ${lib_hadoop_client}/hadoop-mapreduce-client-common.jar debian/tmp/${lib_spark}/jars/
+ ln -s ${lib_hadoop_client}/hadoop-mapreduce-client-core.jar debian/tmp/${lib_spark}/jars/
+ ln -s ${lib_hadoop_client}/hadoop-mapreduce-client-jobclient.jar debian/tmp/${lib_spark}/jars/
+ ln -s ${lib_hadoop_client}/hadoop-mapreduce-client-shuffle.jar debian/tmp/${lib_spark}/jars/
+ ln -s ${lib_hadoop_yarn}/hadoop-yarn-api.jar debian/tmp/${lib_spark}/jars/
+ ln -s ${lib_hadoop_yarn}/hadoop-yarn-client.jar debian/tmp/${lib_spark}/jars/
+ ln -s ${lib_hadoop_yarn}/hadoop-yarn-common.jar debian/tmp/${lib_spark}/jars/
+ ln -s ${lib_hadoop_yarn}/hadoop-yarn-server-common.jar debian/tmp/${lib_spark}/jars/
+ ln -s ${lib_hadoop_yarn}/hadoop-yarn-server-web-proxy.jar debian/tmp/${lib_spark}/jars/
http://git-wip-us.apache.org/repos/asf/bigtop/blob/1bb35133/bigtop-packages/src/rpm/spark/SPECS/spark.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
index d2b07a6..2496109 100644
--- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec
+++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
@@ -24,6 +24,8 @@
%define bin /usr/bin
%define man_dir /usr/share/man
%define spark_services master worker history-server thriftserver
+%define lib_hadoop_client /usr/lib/hadoop/client
+%define lib_hadoop_yarn /usr/lib/hadoop-yarn/
%if %{?suse_version:1}0
%define doc_spark %{_docdir}/spark
@@ -54,7 +56,7 @@ Source6: init.d.tmpl
Source7: spark-history-server.svc
Source8: spark-thriftserver.svc
Source9: bigtop.bom
-Requires: bigtop-utils >= 0.7, hadoop-client
+Requires: bigtop-utils >= 0.7, hadoop-client, hadoop-yarn
Requires(preun): /sbin/service
%global initd_dir %{_sysconfdir}/init.d
@@ -157,6 +159,23 @@ bash $RPM_SOURCE_DIR/install_spark.sh \
--prefix=$RPM_BUILD_ROOT \
--doc-dir=%{doc_spark}
+%__rm -f $RPM_BUILD_ROOT/%{lib_spark}/jars/hadoop-*.jar
+%__ln_s %{lib_hadoop_client}/hadoop-annotations.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+%__ln_s %{lib_hadoop_client}/hadoop-auth.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+%__ln_s %{lib_hadoop_client}/hadoop-client.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+%__ln_s %{lib_hadoop_client}/hadoop-common.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+%__ln_s %{lib_hadoop_client}/hadoop-hdfs.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+%__ln_s %{lib_hadoop_client}/hadoop-mapreduce-client-app.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+%__ln_s %{lib_hadoop_client}/hadoop-mapreduce-client-common.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+%__ln_s %{lib_hadoop_client}/hadoop-mapreduce-client-core.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+%__ln_s %{lib_hadoop_client}/hadoop-mapreduce-client-jobclient.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+%__ln_s %{lib_hadoop_client}/hadoop-mapreduce-client-shuffle.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+%__ln_s %{lib_hadoop_yarn}/hadoop-yarn-api.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+%__ln_s %{lib_hadoop_yarn}/hadoop-yarn-client.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+%__ln_s %{lib_hadoop_yarn}/hadoop-yarn-common.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+%__ln_s %{lib_hadoop_yarn}/hadoop-yarn-server-common.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+%__ln_s %{lib_hadoop_yarn}/hadoop-yarn-server-web-proxy.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
+
for service in %{spark_services}
do
# Install init script
http://git-wip-us.apache.org/repos/asf/bigtop/blob/1bb35133/bigtop.bom
----------------------------------------------------------------------
diff --git a/bigtop.bom b/bigtop.bom
index bd6771c..a428654 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -304,7 +304,7 @@ bigtop {
name = 'spark'
pkg = 'spark-core'
relNotes = 'Apache Spark'
- version { base = '2.1.0'; pkg = base; release = 1 }
+ version { base = '2.1.1'; pkg = base; release = 1 }
tarball { destination = "$name-${version.base}.tar.gz"
source = "$name-${version.base}.tgz" }
url { download_path = "/$name/$name-${version.base}"