You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2014/04/18 19:03:21 UTC
git commit: [SPARK-1520] remove fastutil from dependencies
Repository: spark
Updated Branches:
refs/heads/master 89f47434e -> aa17f022c
[SPARK-1520] remove fastutil from dependencies
A quick fix for https://issues.apache.org/jira/browse/SPARK-1520
By excluding fastutil, we bring the number of files in the assembly jar back under 65536, so Java 7 won't create the assembly jar in zip64 format, which cannot be read by Java 6.
With this change, the assembly jar now has about 60000 entries (58000 files), tested with both sbt and maven.
Author: Xiangrui Meng <me...@databricks.com>
Closes #437 from mengxr/remove-fastutil and squashes the following commits:
00f9beb [Xiangrui Meng] remove fastutil from dependencies
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/aa17f022
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/aa17f022
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/aa17f022
Branch: refs/heads/master
Commit: aa17f022c59af02b04b977da9017671ef14d664a
Parents: 89f4743
Author: Xiangrui Meng <me...@databricks.com>
Authored: Fri Apr 18 10:03:15 2014 -0700
Committer: Reynold Xin <rx...@apache.org>
Committed: Fri Apr 18 10:03:15 2014 -0700
----------------------------------------------------------------------
pom.xml | 7 +++++++
project/SparkBuild.scala | 4 ++--
2 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/aa17f022/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index cd20437..4ff18af 100644
--- a/pom.xml
+++ b/pom.xml
@@ -263,6 +263,13 @@
<groupId>com.clearspring.analytics</groupId>
<artifactId>stream</artifactId>
<version>2.5.1</version>
+ <exclusions>
+ <!-- Only HyperLogLog is used, which doesn't depend on fastutil -->
+ <exclusion>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- In theory we need not directly depend on protobuf since Spark does not directly
use it. However, when building with Hadoop/YARN 2.2 Maven doesn't correctly bump
http://git-wip-us.apache.org/repos/asf/spark/blob/aa17f022/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 4f5a3a2..33f9d64 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -304,7 +304,7 @@ object SparkBuild extends Build {
val excludeHadoop = ExclusionRule(organization = "org.apache.hadoop")
val excludeCurator = ExclusionRule(organization = "org.apache.curator")
val excludePowermock = ExclusionRule(organization = "org.powermock")
-
+ val excludeFastutil = ExclusionRule(organization = "it.unimi.dsi")
def sparkPreviousArtifact(id: String, organization: String = "org.apache.spark",
version: String = "0.9.0-incubating", crossVersion: String = "2.10"): Option[sbt.ModuleID] = {
@@ -343,7 +343,7 @@ object SparkBuild extends Build {
"com.twitter" %% "chill" % chillVersion excludeAll(excludeAsm),
"com.twitter" % "chill-java" % chillVersion excludeAll(excludeAsm),
"org.tachyonproject" % "tachyon" % "0.4.1-thrift" excludeAll(excludeHadoop, excludeCurator, excludeEclipseJetty, excludePowermock),
- "com.clearspring.analytics" % "stream" % "2.5.1",
+ "com.clearspring.analytics" % "stream" % "2.5.1" excludeAll(excludeFastutil),
"org.spark-project" % "pyrolite" % "2.0"
),
libraryDependencies ++= maybeAvro