You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/06/17 13:21:07 UTC
incubator-lens git commit: LENS-515 : Support to run spark in
yarn-client mode in lens-ml module
Repository: incubator-lens
Updated Branches:
refs/heads/master 37d2fc105 -> 44a6f5162
LENS-515 : Support to run spark in yarn-client mode in lens-ml module
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/44a6f516
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/44a6f516
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/44a6f516
Branch: refs/heads/master
Commit: 44a6f5162858e243968280f2fda27ae614f36e4e
Parents: 37d2fc1
Author: Sharad Agarwal <sh...@apache.org>
Authored: Wed Jun 17 16:50:54 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Jun 17 16:50:54 2015 +0530
----------------------------------------------------------------------
lens-docker/lens-test/lens-bootstrap.sh | 6 +++++-
lens-ml-dist/pom.xml | 20 ++++++++++++++++++++
lens-ml-lib/pom.xml | 4 ++++
.../lens/ml/algo/spark/SparkMLDriver.java | 7 +++++++
pom.xml | 5 +++++
tools/conf-pseudo-distr/server/lens-site.xml | 2 +-
tools/scripts/lens-ctl | 4 +++-
7 files changed, 45 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/lens-docker/lens-test/lens-bootstrap.sh
----------------------------------------------------------------------
diff --git a/lens-docker/lens-test/lens-bootstrap.sh b/lens-docker/lens-test/lens-bootstrap.sh
index 11cf1b9..3407dca 100644
--- a/lens-docker/lens-test/lens-bootstrap.sh
+++ b/lens-docker/lens-test/lens-bootstrap.sh
@@ -28,7 +28,11 @@ echo "SPARK_HOME " $SPARK_HOME
LENS_EXT_CLASSPATH=$LENS_EXT_CLASSPATH:`$LENS_ML/bin/lens-ml-classpath.sh`
export LENS_EXT_CLASSPATH
-HIVE_AUX_JARS_PATH=$LENS_ML/lib/lens-ml-lib-2.2.0-beta-incubating-SNAPSHOT.jar,$SPARK_HOME/lib/spark-assembly-1.3.0-hadoop2.4.0.jar
+SPARK_YARN_JAR=$SPARK_HOME/lib/spark-assembly-1.3.0-hadoop2.4.0.jar
+export SPARK_YARN_JAR
+echo "SPARK_YARN_JAR " $SPARK_YARN_JAR
+
+HIVE_AUX_JARS_PATH=$LENS_ML/lib/lens-ml-lib-2.2.0-beta-incubating-SNAPSHOT.jar,$SPARK_YARN_JAR
export HIVE_AUX_JARS_PATH
echo "HIVE_AUX_JARS_PATH " $HIVE_AUX_JARS_PATH
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/lens-ml-dist/pom.xml
----------------------------------------------------------------------
diff --git a/lens-ml-dist/pom.xml b/lens-ml-dist/pom.xml
index 54c9e51..26be98d 100644
--- a/lens-ml-dist/pom.xml
+++ b/lens-ml-dist/pom.xml
@@ -56,6 +56,26 @@
<artifactId>hadoop-common</artifactId>
</exclusion>
<exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-web-proxy</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/lens-ml-lib/pom.xml
----------------------------------------------------------------------
diff --git a/lens-ml-lib/pom.xml b/lens-ml-lib/pom.xml
index f952f8a..63d72da 100644
--- a/lens-ml-lib/pom.xml
+++ b/lens-ml-lib/pom.xml
@@ -88,6 +88,10 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_2.10</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-yarn_2.10</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.hive.hcatalog</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
----------------------------------------------------------------------
diff --git a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
index 6543fed..21ed87d 100644
--- a/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
+++ b/lens-ml-lib/src/main/java/org/apache/lens/ml/algo/spark/SparkMLDriver.java
@@ -172,7 +172,14 @@ public class SparkMLDriver implements MLDriver {
throw new IllegalArgumentException("Spark home is not set");
}
+ // set spark.yarn.jar
+ String yarnJars = System.getenv("SPARK_YARN_JAR");
+ if (StringUtils.isNotBlank(yarnJars)) {
+ sparkConf.set("spark.yarn.jar", yarnJars);
+ }
+
LOG.info("Spark home is set to " + sparkConf.get("spark.home"));
+ LOG.info("spark.yarn.jar is set to " + yarnJars);
}
sparkConf.setAppName("lens-ml");
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 067f9b4..df0b766 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1165,6 +1165,11 @@
<artifactId>spark-mllib_2.10</artifactId>
<version>${spark.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-yarn_2.10</artifactId>
+ <version>${spark.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.hive.hcatalog</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/tools/conf-pseudo-distr/server/lens-site.xml
----------------------------------------------------------------------
diff --git a/tools/conf-pseudo-distr/server/lens-site.xml b/tools/conf-pseudo-distr/server/lens-site.xml
index 6799987..f43d07e 100644
--- a/tools/conf-pseudo-distr/server/lens-site.xml
+++ b/tools/conf-pseudo-distr/server/lens-site.xml
@@ -76,7 +76,7 @@
<property>
<name>lens.ml.sparkdriver.spark.master</name>
- <value>local</value>
+ <value>yarn-client</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/44a6f516/tools/scripts/lens-ctl
----------------------------------------------------------------------
diff --git a/tools/scripts/lens-ctl b/tools/scripts/lens-ctl
index ec6b5fe..1335469 100755
--- a/tools/scripts/lens-ctl
+++ b/tools/scripts/lens-ctl
@@ -116,7 +116,9 @@ start() {
MAPRED_JARS=$MAPRED_JARS:`ls ${HADOOP_HOME}/../hadoop-mapreduce/aws-java-sdk-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
MAPRED_JARS=$MAPRED_JARS:`ls ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
- HADOOP_JARPATH=$CORE_JARS:$LIB_JARS:$COMMON_JARS:$HDFS_JARS:$MAPRED_JARS
+ YARN_JARS=`ls ${HADOOP_HOME}/share/hadoop/yarn/hadoop-yarn*.jar 2>/dev/null | tr "\n" ':' 2>/dev/null`
+
+ HADOOP_JARPATH=$CORE_JARS:$LIB_JARS:$COMMON_JARS:$HDFS_JARS:$MAPRED_JARS:$YARN_JARS
LENSCPPATH=${LENSCPPATH}:$HADOOP_JARPATH
fi