You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zeppelin.apache.org by mo...@apache.org on 2015/03/30 09:55:29 UTC
incubator-zeppelin git commit: [ZEPPELIN-7] Support yarn without
SPARK_YARN_JAR
Repository: incubator-zeppelin
Updated Branches:
refs/heads/master a3a86f883 -> 58fd82b86
[ZEPPELIN-7] Support yarn without SPARK_YARN_JAR
- Removed SPARK_YARN_JAR dependencies
Author: Jongyoul Lee <jo...@gmail.com>
Closes #4 from jongyoul/ZEPPELIN-7 and squashes the following commits:
df697c1 [Jongyoul Lee] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Updated README.md
7f84a5a [Jongyoul Lee] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Reverted to initial commit
b366d16 [Jongyoul Lee] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Fixed 2 spaces indentations
29ea46b [Jongyoul Lee] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Removed unused codes
91066c4 [Jongyoul Lee] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Fixed errors while running
32be4ca [Jongyoul Lee] [ZEPPELIN-7] Support yarn without SPARK_YARN_JAR - Added maven-shade-plugin - Removed copy-dependencies
Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/58fd82b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/58fd82b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/58fd82b8
Branch: refs/heads/master
Commit: 58fd82b86207cc773ced5624a387705ba02aa576
Parents: a3a86f8
Author: Jongyoul Lee <jo...@gmail.com>
Authored: Fri Mar 27 11:34:40 2015 +0900
Committer: Lee moon soo <mo...@apache.org>
Committed: Mon Mar 30 16:55:23 2015 +0900
----------------------------------------------------------------------
README.md | 4 +-
spark/pom.xml | 112 ++++++++++++++++++++++++++++++-----------------------
2 files changed, 65 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/58fd82b8/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 93df312..46e188c 100644
--- a/README.md
+++ b/README.md
@@ -85,10 +85,8 @@ If you set `SPARK_HOME`, you should deploy spark binary on the same location to
Yarn
# ./conf/zeppelin-env.sh
- export SPARK_YARN_JAR=/path/to/spark-assembly-*.jar
export HADOOP_CONF_DIR=/path/to/hadoop_conf_dir
-
-`SPARK_YARN_JAR` is deployed for running executor, this could be a local path or HDFS. HDFS allows YARN to cache it on nodes so that it doesn't need to be distributed each time an application runs. To point to a jar on HDFS, for example, set this configuration to "hdfs:///some/path".
+
`HADOOP_CONF_DIR` should contains yarn-site.xml and core-site.xml.
### Run
http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/58fd82b8/spark/pom.xml
----------------------------------------------------------------------
diff --git a/spark/pom.xml b/spark/pom.xml
index 49b4d08..fcecc1b 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -1,5 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -9,7 +10,6 @@
<version>0.5.0-SNAPSHOT</version>
</parent>
- <groupId>com.nflabs.zeppelin</groupId>
<artifactId>zeppelin-spark</artifactId>
<packaging>jar</packaging>
<version>0.5.0-SNAPSHOT</version>
@@ -67,7 +67,7 @@
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</dependency>
-
+
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-compiler</artifactId>
@@ -77,7 +77,7 @@
<groupId>org.scala-lang</groupId>
<artifactId>scala-reflect</artifactId>
</dependency>
-
+
<dependency>
<groupId>org.eclipse.jetty.aggregate</groupId>
<artifactId>jetty-all-server</artifactId>
@@ -94,27 +94,27 @@
<version>${spark.version}</version>
</dependency>
- <dependency>
+ <dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_2.10</artifactId>
<version>${spark.version}</version>
</dependency>
- <dependency>
+ <dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_2.10</artifactId>
<version>${spark.version}</version>
</dependency>
- <dependency>
+ <dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
<version>${spark.version}</version>
</dependency>
- <dependency>
+ <dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-twitter_2.10</artifactId>
<version>${spark.version}</version>
@@ -125,7 +125,7 @@
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
</dependency>
-
+
<!-- Aether :: maven dependency resolution -->
<dependency>
<groupId>org.apache.maven</groupId>
@@ -139,12 +139,12 @@
<exclusion>
<groupId>org.sonatype.sisu</groupId>
<artifactId>sisu-inject-plexus</artifactId>
- </exclusion>
+ </exclusion>
<exclusion>
<groupId>org.apache.maven</groupId>
<artifactId>maven-model</artifactId>
- </exclusion>
- </exclusions>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.sonatype.aether</groupId>
@@ -161,7 +161,7 @@
<artifactId>aether-impl</artifactId>
<version>1.12</version>
</dependency>
-
+
<dependency>
<groupId>org.apache.maven</groupId>
<artifactId>maven-aether-provider</artifactId>
@@ -170,7 +170,7 @@
<exclusion>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-api</artifactId>
- </exclusion>
+ </exclusion>
<exclusion>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-spi</artifactId>
@@ -178,24 +178,24 @@
<exclusion>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-util</artifactId>
- </exclusion>
+ </exclusion>
<exclusion>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-impl</artifactId>
- </exclusion>
+ </exclusion>
<exclusion>
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-utils</artifactId>
- </exclusion>
- </exclusions>
+ </exclusion>
+ </exclusions>
</dependency>
-
+
<dependency>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-connector-file</artifactId>
<version>1.12</version>
</dependency>
-
+
<dependency>
<groupId>org.sonatype.aether</groupId>
<artifactId>aether-connector-wagon</artifactId>
@@ -205,9 +205,9 @@
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-provider-api</artifactId>
</exclusion>
- </exclusions>
+ </exclusions>
</dependency>
-
+
<dependency>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-provider-api</artifactId>
@@ -216,10 +216,10 @@
<exclusion>
<groupId>org.codehaus.plexus</groupId>
<artifactId>plexus-utils</artifactId>
- </exclusion>
- </exclusions>
+ </exclusion>
+ </exclusions>
</dependency>
-
+
<dependency>
<groupId>org.apache.maven.wagon</groupId>
<artifactId>wagon-http-lightweight</artifactId>
@@ -230,7 +230,7 @@
<artifactId>wagon-http-shared</artifactId>
</exclusion>
</exclusions>
- </dependency>
+ </dependency>
<dependency>
<groupId>org.apache.maven.wagon</groupId>
@@ -259,7 +259,7 @@
<scope>test</scope>
</dependency>
</dependencies>
-
+
<profiles>
<profile>
<id>yarn</id>
@@ -272,7 +272,7 @@
</dependencies>
</profile>
</profiles>
-
+
<build>
<plugins>
<plugin>
@@ -286,11 +286,11 @@
<plugin>
<artifactId>maven-enforcer-plugin</artifactId>
- <version>1.3.1</version>
- <executions>
- <execution>
- <id>enforce</id>
- <phase>none</phase>
+ <version>1.3.1</version>
+ <executions>
+ <execution>
+ <id>enforce</id>
+ <phase>none</phase>
</execution>
</executions>
</plugin>
@@ -307,25 +307,42 @@
</plugin>
<plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.3</version>
+ <configuration>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>reference.conf</resource>
+ </transformer>
+ </transformers>
+ </configuration>
<executions>
<execution>
- <id>copy-dependencies</id>
<phase>package</phase>
<goals>
- <goal>copy-dependencies</goal>
+ <goal>shade</goal>
</goals>
- <configuration>
- <outputDirectory>${project.build.directory}/../../interpreter/spark</outputDirectory>
- <overWriteReleases>false</overWriteReleases>
- <overWriteSnapshots>false</overWriteSnapshots>
- <overWriteIfNewer>true</overWriteIfNewer>
- <includeScope>runtime</includeScope>
- </configuration>
</execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
<execution>
- <id>copy-artifact</id>
<phase>package</phase>
<goals>
<goal>copy</goal>
@@ -335,7 +352,6 @@
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
- <includeScope>runtime</includeScope>
<artifactItems>
<artifactItem>
<groupId>${project.groupId}</groupId>
@@ -343,11 +359,11 @@
<version>${project.version}</version>
<type>${project.packaging}</type>
</artifactItem>
- </artifactItems>
+ </artifactItems>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
- </build>
+ </build>
</project>