You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/09/23 22:42:06 UTC
git commit: [SPARK-3647] Add more exceptions to Guava relocation.
Repository: spark
Updated Branches:
refs/heads/master a08153f8a -> 8dfe79ffb
[SPARK-3647] Add more exceptions to Guava relocation.
Guava's Optional refers to some package private classes / methods, and
when those are relocated the code stops working, throwing exceptions.
So add the affected classes to the exception list too, and add a unit
test.
(Note that this unit test only really makes sense in maven, since we
don't relocate in the sbt build. Also, JavaAPISuite doesn't seem to
be run by "mvn test" - I had to manually add command line options to
enable it.)
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #2496 from vanzin/SPARK-3647 and squashes the following commits:
84f58d7 [Marcelo Vanzin] [SPARK-3647] Add more exceptions to Guava relocation.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8dfe79ff
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8dfe79ff
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8dfe79ff
Branch: refs/heads/master
Commit: 8dfe79ffb204807945e3c09b75c7255b09ad2a97
Parents: a08153f
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Tue Sep 23 13:42:00 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Tue Sep 23 13:42:00 2014 -0700
----------------------------------------------------------------------
assembly/pom.xml | 4 ++-
core/pom.xml | 2 ++
.../java/org/apache/spark/JavaAPISuite.java | 26 ++++++++++++++++++++
3 files changed, 31 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8dfe79ff/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 604b1ab..5ec9da2 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -141,7 +141,9 @@
<include>com.google.common.**</include>
</includes>
<excludes>
- <exclude>com.google.common.base.Optional**</exclude>
+ <exclude>com/google/common/base/Absent*</exclude>
+ <exclude>com/google/common/base/Optional*</exclude>
+ <exclude>com/google/common/base/Present*</exclude>
</excludes>
</relocation>
</relocations>
http://git-wip-us.apache.org/repos/asf/spark/blob/8dfe79ff/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 2a81f6d..e012c5e 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -343,7 +343,9 @@
<filter>
<artifact>com.google.guava:guava</artifact>
<includes>
+ <include>com/google/common/base/Absent*</include>
<include>com/google/common/base/Optional*</include>
+ <include>com/google/common/base/Present*</include>
</includes>
</filter>
</filters>
http://git-wip-us.apache.org/repos/asf/spark/blob/8dfe79ff/core/src/test/java/org/apache/spark/JavaAPISuite.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/spark/JavaAPISuite.java b/core/src/test/java/org/apache/spark/JavaAPISuite.java
index b8574df..b8c23d5 100644
--- a/core/src/test/java/org/apache/spark/JavaAPISuite.java
+++ b/core/src/test/java/org/apache/spark/JavaAPISuite.java
@@ -1307,4 +1307,30 @@ public class JavaAPISuite implements Serializable {
SomeCustomClass[] collected = (SomeCustomClass[]) rdd.rdd().retag(SomeCustomClass.class).collect();
Assert.assertEquals(data.size(), collected.length);
}
+
+ /**
+ * Test for SPARK-3647. This test needs to use the maven-built assembly to trigger the issue,
+ * since that's the only artifact where Guava classes have been relocated.
+ */
+ @Test
+ public void testGuavaOptional() {
+ // Stop the context created in setUp() and start a local-cluster one, to force usage of the
+ // assembly.
+ sc.stop();
+ JavaSparkContext localCluster = new JavaSparkContext("local-cluster[1,1,512]", "JavaAPISuite");
+ try {
+ JavaRDD<Integer> rdd1 = localCluster.parallelize(Arrays.asList(1, 2, null), 3);
+ JavaRDD<Optional<Integer>> rdd2 = rdd1.map(
+ new Function<Integer, Optional<Integer>>() {
+ @Override
+ public Optional<Integer> call(Integer i) {
+ return Optional.fromNullable(i);
+ }
+ });
+ rdd2.collect();
+ } finally {
+ localCluster.stop();
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org