You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2015/02/02 06:14:06 UTC
spark git commit: [SPARK-3996]: Shade Jetty in Spark deliverables
Repository: spark
Updated Branches:
refs/heads/master 859f7249a -> a15f6e31f
[SPARK-3996]: Shade Jetty in Spark deliverables
(v2 of this patch with a fix that was only relevant for the maven build).
This patch piggy-back's on vanzin's work to simplify the Guava shading,
and adds Jetty as a shaded library in Spark. Other than adding Jetty,
it consilidates the <artifactSet>'s into the root pom. I found it was
a bit easier to follow that way, since you don't need to look into
child pom's to find out specific artifact sets included in shading.
Author: Patrick Wendell <pa...@databricks.com>
Closes #4285 from pwendell/jetty and squashes the following commits:
d3e7f4e [Patrick Wendell] Fix for shaded deps causing compile errors
19f0710 [Patrick Wendell] More code review feedback
961452d [Patrick Wendell] Responding to feedback from Marcello
6df25ca [Patrick Wendell] [WIP] [SPARK-3996]: Shade Jetty in Spark deliverables
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a15f6e31
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a15f6e31
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a15f6e31
Branch: refs/heads/master
Commit: a15f6e31fc216d0d39bc1578e0da11b068b7821c
Parents: 859f724
Author: Patrick Wendell <pa...@databricks.com>
Authored: Sun Feb 1 21:13:57 2015 -0800
Committer: Patrick Wendell <pa...@databricks.com>
Committed: Sun Feb 1 21:13:57 2015 -0800
----------------------------------------------------------------------
bin/compute-classpath.sh | 4 +++-
core/pom.xml | 22 ++++++++++++++++++++--
network/common/pom.xml | 12 ------------
pom.xml | 32 ++++++++++++++++++++++++++++++++
repl/pom.xml | 24 ++++++++++++++++++++++++
streaming/pom.xml | 16 ++++++++++++++++
6 files changed, 95 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a15f6e31/bin/compute-classpath.sh
----------------------------------------------------------------------
diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh
index 9e8d0b7..a8c344b 100755
--- a/bin/compute-classpath.sh
+++ b/bin/compute-classpath.sh
@@ -50,8 +50,8 @@ fi
if [ -n "$SPARK_PREPEND_CLASSES" ]; then
echo "NOTE: SPARK_PREPEND_CLASSES is set, placing locally compiled Spark"\
"classes ahead of assembly." >&2
+ # Spark classes
CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SPARK_SCALA_VERSION/classes"
- CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*"
CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SPARK_SCALA_VERSION/classes"
CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SPARK_SCALA_VERSION/classes"
CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SPARK_SCALA_VERSION/classes"
@@ -63,6 +63,8 @@ if [ -n "$SPARK_PREPEND_CLASSES" ]; then
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SPARK_SCALA_VERSION/classes"
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive-thriftserver/target/scala-$SPARK_SCALA_VERSION/classes"
CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SPARK_SCALA_VERSION/classes"
+ # Jars for shaded deps in their original form (copied here during build)
+ CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*"
fi
# Use spark-assembly jar from either RELEASE or assembly directory
http://git-wip-us.apache.org/repos/asf/spark/blob/a15f6e31/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 6fce10a..4d24ae9 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -94,23 +94,36 @@
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
</dependency>
+
+ <!-- Jetty dependencies promoted to compile here so they are shaded
+ and inlined into spark-core jar -->
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-plus</artifactId>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-security</artifactId>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
+ <scope>compile</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
+ <scope>compile</scope>
</dependency>
<dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-http</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
@@ -348,19 +361,24 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
+ <!-- When using SPARK_PREPEND_CLASSES Spark classes compiled locally don't use
+ shaded deps. So here we store jars in their original form which are added
+ when the classpath is computed. -->
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
- <configuration>
+ <configuration>
<outputDirectory>${project.build.directory}</outputDirectory>
<overWriteReleases>false</overWriteReleases>
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<useSubDirectoryPerType>true</useSubDirectoryPerType>
- <includeArtifactIds>guava</includeArtifactIds>
+ <includeArtifactIds>
+ guava,jetty-io,jetty-http,jetty-plus,jetty-util,jetty-server
+ </includeArtifactIds>
<silent>true</silent>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/spark/blob/a15f6e31/network/common/pom.xml
----------------------------------------------------------------------
diff --git a/network/common/pom.xml b/network/common/pom.xml
index 5a9bbe1..8f7c924 100644
--- a/network/common/pom.xml
+++ b/network/common/pom.xml
@@ -101,18 +101,6 @@
</execution>
</executions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <configuration>
- <shadedArtifactAttached>false</shadedArtifactAttached>
- <artifactSet>
- <includes>
- <include>com.google.guava:guava</include>
- </includes>
- </artifactSet>
- </configuration>
- </plugin>
</plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/spark/blob/a15f6e31/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b855f23..d4112b0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -337,25 +337,39 @@
</exclusion>
</exclusions>
</dependency>
+
+ <!-- Shaded deps marked as provided. These are promoted to compile scope
+ in the modules where we want the shaded classes to appear in the
+ associated jar. -->
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-http</artifactId>
+ <version>${jetty.version}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
<version>${jetty.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-security</artifactId>
<version>${jetty.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-plus</artifactId>
<version>${jetty.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>${jetty.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
@@ -363,6 +377,8 @@
<version>14.0.1</version>
<scope>provided</scope>
</dependency>
+ <!-- End of shaded deps -->
+
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
@@ -1276,11 +1292,27 @@
<shadedArtifactAttached>false</shadedArtifactAttached>
<artifactSet>
<includes>
+ <!-- At a minimum we must include this to force effective pom generation -->
<include>org.spark-project.spark:unused</include>
+
+ <include>org.eclipse.jetty:jetty-io</include>
+ <include>org.eclipse.jetty:jetty-http</include>
+ <include>org.eclipse.jetty:jetty-plus</include>
+ <include>org.eclipse.jetty:jetty-security</include>
+ <include>org.eclipse.jetty:jetty-util</include>
+ <include>org.eclipse.jetty:jetty-server</include>
+ <include>com.google.guava:guava</include>
</includes>
</artifactSet>
<relocations>
<relocation>
+ <pattern>org.eclipse.jetty</pattern>
+ <shadedPattern>org.spark-project.jetty</shadedPattern>
+ <includes>
+ <include>org.eclipse.jetty.**</include>
+ </includes>
+ </relocation>
+ <relocation>
<pattern>com.google.common</pattern>
<shadedPattern>org.spark-project.guava</shadedPattern>
<excludes>
http://git-wip-us.apache.org/repos/asf/spark/blob/a15f6e31/repl/pom.xml
----------------------------------------------------------------------
diff --git a/repl/pom.xml b/repl/pom.xml
index ae7c31a..bd39b90 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -87,6 +87,30 @@
<artifactId>scalacheck_${scala.binary.version}</artifactId>
<scope>test</scope>
</dependency>
+
+ <!-- Explicit listing of transitive deps that are shaded. Otherwise, odd compiler crashes. -->
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-plus</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-http</artifactId>
+ </dependency>
+ <!-- End of shaded deps. -->
+
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ </dependency>
</dependencies>
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
http://git-wip-us.apache.org/repos/asf/spark/blob/a15f6e31/streaming/pom.xml
----------------------------------------------------------------------
diff --git a/streaming/pom.xml b/streaming/pom.xml
index d032491..5efed16 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -40,6 +40,8 @@
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <!-- Explicit listing of transitive deps that are shaded. Otherwise, odd compiler crashes. -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
@@ -49,6 +51,20 @@
<artifactId>jetty-server</artifactId>
</dependency>
<dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-plus</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-http</artifactId>
+ </dependency>
+ <!-- End of shaded deps. -->
+
+ <dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</dependency>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org