You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by yw...@apache.org on 2017/12/27 02:37:57 UTC

bigtop git commit: BIGTOP-2959: Add SPARK_DIST_CLASSPATH to include libraries for HDFS and YARN

Repository: bigtop
Updated Branches:
  refs/heads/master 9dd5f5a1d -> 9e78a0365


BIGTOP-2959: Add SPARK_DIST_CLASSPATH to include libraries for HDFS and YARN


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/9e78a036
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/9e78a036
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/9e78a036

Branch: refs/heads/master
Commit: 9e78a03655ee68b8a60258c2a5efd9dbe6dfac30
Parents: 9dd5f5a
Author: Youngwoo Kim <yw...@apache.org>
Authored: Tue Dec 26 15:17:19 2017 +0900
Committer: Youngwoo Kim <yw...@apache.org>
Committed: Wed Dec 27 11:24:13 2017 +0900

----------------------------------------------------------------------
 .../src/common/spark/do-component-build            |  2 +-
 bigtop-packages/src/common/spark/spark-env.sh      |  3 +--
 bigtop-packages/src/deb/spark/rules                | 17 -----------------
 bigtop-packages/src/rpm/spark/SPECS/spark.spec     | 17 -----------------
 4 files changed, 2 insertions(+), 37 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e78a036/bigtop-packages/src/common/spark/do-component-build
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/spark/do-component-build b/bigtop-packages/src/common/spark/do-component-build
index c2a2322..084f9f2 100644
--- a/bigtop-packages/src/common/spark/do-component-build
+++ b/bigtop-packages/src/common/spark/do-component-build
@@ -28,7 +28,7 @@ BUILD_OPTS="-Divy.home=${HOME}/.ivy2 -Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${
             -Dyarn.version=$HADOOP_VERSION \
             -Dprotobuf.version=2.5.0 \
             -DrecompileMode=all \
-            -Pyarn -Phadoop-2.7 \
+            -Pyarn -Phadoop-2.7 -Phadoop-provided \
             -Phive -Phive-thriftserver \
             -Psparkr \
             $SPARK_BUILD_OPTS"

http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e78a036/bigtop-packages/src/common/spark/spark-env.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/spark/spark-env.sh b/bigtop-packages/src/common/spark/spark-env.sh
index f772ef1..1806119 100755
--- a/bigtop-packages/src/common/spark/spark-env.sh
+++ b/bigtop-packages/src/common/spark/spark-env.sh
@@ -28,8 +28,7 @@ export SPARK_LAUNCH_WITH_SCALA=0
 export SPARK_LIBRARY_PATH=${SPARK_LIBRARY_PATH:-${SPARK_HOME}/lib}
 export SCALA_LIBRARY_PATH=${SCALA_LIBRARY_PATH:-${SPARK_HOME}/lib}
 
-# Let's make sure that all needed hadoop libs are added properly
-export CLASSPATH="$CLASSPATH:$HADOOP_HOME/*:$HADOOP_HDFS_HOME/*:$HADOOP_YARN_HOME/*:$HADOOP_MAPRED_HOME/*"
+export SPARK_DIST_CLASSPATH=$(hadoop classpath)
 export SPARK_LIBRARY_PATH=$SPARK_LIBRARY_PATH:${HADOOP_HOME}/lib/native
 
 export STANDALONE_SPARK_MASTER_HOST=`hostname -f`

http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e78a036/bigtop-packages/src/deb/spark/rules
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/spark/rules b/bigtop-packages/src/deb/spark/rules
index d5fdb6d..5bd9833 100644
--- a/bigtop-packages/src/deb/spark/rules
+++ b/bigtop-packages/src/deb/spark/rules
@@ -49,20 +49,3 @@ override_dh_install:
 	dh_install
 	# BIGTOP-2588 drop datanucleus jars from spark-core
 	rm -Rf debian/spark-core/usr/lib/spark/jars/datanucleus*
-
-	rm -f debian/tmp/${lib_spark}/jars/hadoop-*.jar
-	ln -s  ${lib_hadoop_client}/hadoop-annotations.jar debian/tmp/${lib_spark}/jars/
-	ln -s  ${lib_hadoop_client}/hadoop-auth.jar debian/tmp/${lib_spark}/jars/
-	ln -s  ${lib_hadoop_client}/hadoop-client.jar debian/tmp/${lib_spark}/jars/
-	ln -s  ${lib_hadoop_client}/hadoop-common.jar debian/tmp/${lib_spark}/jars/
-	ln -s  ${lib_hadoop_client}/hadoop-hdfs.jar debian/tmp/${lib_spark}/jars/
-	ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-app.jar debian/tmp/${lib_spark}/jars/
-	ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-common.jar debian/tmp/${lib_spark}/jars/
-	ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-core.jar debian/tmp/${lib_spark}/jars/
-	ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-jobclient.jar debian/tmp/${lib_spark}/jars/
-	ln -s  ${lib_hadoop_client}/hadoop-mapreduce-client-shuffle.jar debian/tmp/${lib_spark}/jars/
-	ln -s  ${lib_hadoop_yarn}/hadoop-yarn-api.jar debian/tmp/${lib_spark}/jars/
-	ln -s  ${lib_hadoop_yarn}/hadoop-yarn-client.jar debian/tmp/${lib_spark}/jars/
-	ln -s  ${lib_hadoop_yarn}/hadoop-yarn-common.jar debian/tmp/${lib_spark}/jars/
-	ln -s  ${lib_hadoop_yarn}/hadoop-yarn-server-common.jar debian/tmp/${lib_spark}/jars/
-	ln -s  ${lib_hadoop_yarn}/hadoop-yarn-server-web-proxy.jar debian/tmp/${lib_spark}/jars/

http://git-wip-us.apache.org/repos/asf/bigtop/blob/9e78a036/bigtop-packages/src/rpm/spark/SPECS/spark.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
index c29c68a..8e73cd8 100644
--- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec
+++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
@@ -166,23 +166,6 @@ bash $RPM_SOURCE_DIR/install_spark.sh \
           --prefix=$RPM_BUILD_ROOT  \
           --doc-dir=%{doc_spark}
 
-%__rm -f $RPM_BUILD_ROOT/%{lib_spark}/jars/hadoop-*.jar
-%__ln_s  %{lib_hadoop_client}/hadoop-annotations.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-%__ln_s  %{lib_hadoop_client}/hadoop-auth.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-%__ln_s  %{lib_hadoop_client}/hadoop-client.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-%__ln_s  %{lib_hadoop_client}/hadoop-common.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-%__ln_s  %{lib_hadoop_client}/hadoop-hdfs.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-%__ln_s  %{lib_hadoop_client}/hadoop-mapreduce-client-app.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-%__ln_s  %{lib_hadoop_client}/hadoop-mapreduce-client-common.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-%__ln_s  %{lib_hadoop_client}/hadoop-mapreduce-client-core.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-%__ln_s  %{lib_hadoop_client}/hadoop-mapreduce-client-jobclient.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-%__ln_s  %{lib_hadoop_client}/hadoop-mapreduce-client-shuffle.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-%__ln_s  %{lib_hadoop_yarn}/hadoop-yarn-api.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-%__ln_s  %{lib_hadoop_yarn}/hadoop-yarn-client.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-%__ln_s  %{lib_hadoop_yarn}/hadoop-yarn-common.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-%__ln_s  %{lib_hadoop_yarn}/hadoop-yarn-server-common.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-%__ln_s  %{lib_hadoop_yarn}/hadoop-yarn-server-web-proxy.jar $RPM_BUILD_ROOT/%{lib_spark}/jars/
-
 for service in %{spark_services}
 do
     # Install init script