You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/11/01 23:21:42 UTC
git commit: [SPARK-4121] Set commons-math3 version based on hadoop
profiles, instead of shading
Repository: spark
Updated Branches:
refs/heads/master 7894de276 -> d8176b1c2
[SPARK-4121] Set commons-math3 version based on hadoop profiles, instead of shading
In #2928 , we shade commons-math3 to prevent future conflicts with hadoop. It caused problems with our Jenkins master build with maven. Some tests used local-cluster mode, where the assembly jar contains relocated math3 classes, while mllib test code still compiles with core and the untouched math3 classes.
This PR sets commons-math3 version based on hadoop profiles.
pwendell JoshRosen srowen
Author: Xiangrui Meng <me...@databricks.com>
Closes #3023 from mengxr/SPARK-4121-alt and squashes the following commits:
580f6d9 [Xiangrui Meng] replace tab by spaces
7f71f08 [Xiangrui Meng] revert changes to PoissonSampler to avoid conflicts
d3353d9 [Xiangrui Meng] do not shade commons-math3
b4180dc [Xiangrui Meng] temp work
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d8176b1c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d8176b1c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d8176b1c
Branch: refs/heads/master
Commit: d8176b1c2f22247ee724041aefa1af9118cf861d
Parents: 7894de2
Author: Xiangrui Meng <me...@databricks.com>
Authored: Sat Nov 1 15:21:36 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Sat Nov 1 15:21:36 2014 -0700
----------------------------------------------------------------------
assembly/pom.xml | 4 ----
mllib/pom.xml | 4 ++++
pom.xml | 9 ++++++++-
3 files changed, 12 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d8176b1c/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 9e8525d..31a01e4 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -146,10 +146,6 @@
<exclude>com/google/common/base/Present*</exclude>
</excludes>
</relocation>
- <relocation>
- <pattern>org.apache.commons.math3</pattern>
- <shadedPattern>org.spark-project.commons.math3</shadedPattern>
- </relocation>
</relocations>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
http://git-wip-us.apache.org/repos/asf/spark/blob/d8176b1c/mllib/pom.xml
----------------------------------------------------------------------
diff --git a/mllib/pom.xml b/mllib/pom.xml
index de062a4..fb7239e 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -65,6 +65,10 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-math3</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/spark/blob/d8176b1c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 61a508a..6191cd3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -145,6 +145,7 @@
<aws.java.sdk.version>1.8.3</aws.java.sdk.version>
<aws.kinesis.client.version>1.1.0</aws.kinesis.client.version>
<commons.httpclient.version>4.2.6</commons.httpclient.version>
+ <commons.math3.version>3.1.1</commons.math3.version>
<PermGen>64m</PermGen>
<MaxPermGen>512m</MaxPermGen>
@@ -318,7 +319,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
- <version>3.3</version>
+ <version>${commons.math3.version}</version>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
@@ -1175,6 +1176,10 @@
<!-- A series of build profiles where customizations for particular Hadoop releases can be made -->
+ <!-- Hadoop-a.b.c dependencies can be found at
+ http://hadoop.apache.org/docs/ra.b.c/hadoop-project-dist/hadoop-common/dependency-analysis.html
+ -->
+
<profile>
<id>hadoop-0.23</id>
<!-- SPARK-1121: Adds an explicit dependency on Avro to work around a Hadoop 0.23.X issue -->
@@ -1204,6 +1209,7 @@
<hadoop.version>2.3.0</hadoop.version>
<protobuf.version>2.5.0</protobuf.version>
<jets3t.version>0.9.0</jets3t.version>
+ <commons.math3.version>3.1.1</commons.math3.version>
<avro.mapred.classifier>hadoop2</avro.mapred.classifier>
</properties>
</profile>
@@ -1214,6 +1220,7 @@
<hadoop.version>2.4.0</hadoop.version>
<protobuf.version>2.5.0</protobuf.version>
<jets3t.version>0.9.0</jets3t.version>
+ <commons.math3.version>3.1.1</commons.math3.version>
<avro.mapred.classifier>hadoop2</avro.mapred.classifier>
</properties>
</profile>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org