You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2014/11/12 22:39:50 UTC
spark git commit: [SPARK-4281][Build] Package Yarn shuffle service
into its own jar
Repository: spark
Updated Branches:
refs/heads/master 6e3c5a296 -> aa43a8da0
[SPARK-4281][Build] Package Yarn shuffle service into its own jar
This is another addendum to #3082, which added the Yarn shuffle service to run inside the NM. This PR makes the feature much more usable by packaging enough dependencies into the jar to run the service inside an NM. After these changes, the user can run `./make-distribution.sh` and find a `spark-network-yarn*.jar` in their `lib` directory. The equivalent change is done in SBT by making the `network-yarn` module an assembly project.
Author: Andrew Or <an...@databricks.com>
Closes #3147 from andrewor14/yarn-shuffle-build and squashes the following commits:
bda58d0 [Andrew Or] Fix line too long
81e9705 [Andrew Or] Merge branch 'master' of github.com:apache/spark into yarn-shuffle-build
fb7f398 [Andrew Or] Rename jar to spark-{VERSION}-yarn-shuffle.jar
65db822 [Andrew Or] Actually mark slf4j as provided
abcefd1 [Andrew Or] Do the same for SBT
c653028 [Andrew Or] Package network-yarn and its dependencies
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/aa43a8da
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/aa43a8da
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/aa43a8da
Branch: refs/heads/master
Commit: aa43a8da012cf0dac7c7fcccde5f028a942599f0
Parents: 6e3c5a2
Author: Andrew Or <an...@databricks.com>
Authored: Wed Nov 12 13:39:45 2014 -0800
Committer: Andrew Or <an...@databricks.com>
Committed: Wed Nov 12 13:39:45 2014 -0800
----------------------------------------------------------------------
make-distribution.sh | 1 +
network/common/pom.xml | 5 +++--
network/shuffle/pom.xml | 5 +++--
network/yarn/pom.xml | 33 +++++++++++++++++++++++++++++++++
project/SparkBuild.scala | 19 +++++++++++++------
5 files changed, 53 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/aa43a8da/make-distribution.sh
----------------------------------------------------------------------
diff --git a/make-distribution.sh b/make-distribution.sh
index d46edbc..55cbdc1 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -181,6 +181,7 @@ echo "Spark $VERSION$GITREVSTRING built for Hadoop $SPARK_HADOOP_VERSION" > "$DI
# Copy jars
cp "$FWDIR"/assembly/target/scala*/*assembly*hadoop*.jar "$DISTDIR/lib/"
cp "$FWDIR"/examples/target/scala*/spark-examples*.jar "$DISTDIR/lib/"
+cp "$FWDIR"/network/yarn/target/scala*/spark-*-yarn-shuffle.jar "$DISTDIR/lib/"
# Copy example sources (needed for python and SQL)
mkdir -p "$DISTDIR/examples/src/main"
http://git-wip-us.apache.org/repos/asf/spark/blob/aa43a8da/network/common/pom.xml
----------------------------------------------------------------------
diff --git a/network/common/pom.xml b/network/common/pom.xml
index 8b24ebf..a6bee7e 100644
--- a/network/common/pom.xml
+++ b/network/common/pom.xml
@@ -41,12 +41,13 @@
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
</dependency>
+
+ <!-- Provided dependencies -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
+ <scope>provided</scope>
</dependency>
-
- <!-- Provided dependencies -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
http://git-wip-us.apache.org/repos/asf/spark/blob/aa43a8da/network/shuffle/pom.xml
----------------------------------------------------------------------
diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml
index a180a5e..be78331 100644
--- a/network/shuffle/pom.xml
+++ b/network/shuffle/pom.xml
@@ -42,12 +42,13 @@
<artifactId>spark-network-common_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <!-- Provided dependencies -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
+ <scope>provided</scope>
</dependency>
-
- <!-- Provided dependencies -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
http://git-wip-us.apache.org/repos/asf/spark/blob/aa43a8da/network/yarn/pom.xml
----------------------------------------------------------------------
diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml
index 85960eb..7845011 100644
--- a/network/yarn/pom.xml
+++ b/network/yarn/pom.xml
@@ -54,5 +54,38 @@
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <configuration>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <outputFile>${project.build.directory}/scala-${scala.binary.version}/spark-${project.version}-yarn-shuffle.jar</outputFile>
+ <artifactSet>
+ <includes>
+ <include>*:*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/spark/blob/aa43a8da/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 5eb3ed4..d95d50a 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -38,12 +38,12 @@ object BuildCommons {
"streaming-flume", "streaming-kafka", "streaming-mqtt", "streaming-twitter",
"streaming-zeromq").map(ProjectRef(buildLocation, _))
- val optionallyEnabledProjects@Seq(yarn, yarnStable, yarnAlpha, networkYarn, java8Tests,
- sparkGangliaLgpl, sparkKinesisAsl) = Seq("yarn", "yarn-stable", "yarn-alpha", "network-yarn",
+ val optionallyEnabledProjects@Seq(yarn, yarnStable, yarnAlpha, java8Tests,
+ sparkGangliaLgpl, sparkKinesisAsl) = Seq("yarn", "yarn-stable", "yarn-alpha",
"java8-tests", "ganglia-lgpl", "kinesis-asl").map(ProjectRef(buildLocation, _))
- val assemblyProjects@Seq(assembly, examples) = Seq("assembly", "examples")
- .map(ProjectRef(buildLocation, _))
+ val assemblyProjects@Seq(assembly, examples, networkYarn) =
+ Seq("assembly", "examples", "network-yarn").map(ProjectRef(buildLocation, _))
val tools = ProjectRef(buildLocation, "tools")
// Root project.
@@ -289,8 +289,15 @@ object Assembly {
lazy val settings = assemblySettings ++ Seq(
test in assembly := {},
- jarName in assembly <<= (version, moduleName) map { (v, mName) => mName + "-"+v + "-hadoop" +
- Option(System.getProperty("hadoop.version")).getOrElse("1.0.4") + ".jar" },
+ jarName in assembly <<= (version, moduleName) map { (v, mName) =>
+ if (mName.contains("network-yarn")) {
+ // This must match the same name used in maven (see network/yarn/pom.xml)
+ "spark-" + v + "-yarn-shuffle.jar"
+ } else {
+ mName + "-" + v + "-hadoop" +
+ Option(System.getProperty("hadoop.version")).getOrElse("1.0.4") + ".jar"
+ }
+ },
mergeStrategy in assembly := {
case PathList("org", "datanucleus", xs @ _*) => MergeStrategy.discard
case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org