You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2013/09/11 19:26:42 UTC
[03/13] git commit: Fix HDFS access bug with assembly build.
Fix HDFS access bug with assembly build.
Due to this change in HDFS:
https://issues.apache.org/jira/browse/HADOOP-7549
there is a bug when using the new assembly builds. The symptom is that any HDFS access
results in an exception saying "No filesystem for scheme 'hdfs'". This adds a merge
strategy in the assembly build which fixes the problem.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/0c1985b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/0c1985b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/0c1985b1
Branch: refs/heads/branch-0.8
Commit: 0c1985b153a2dc2c891ae61c1ee67506926384ae
Parents: 2425eb8
Author: Patrick Wendell <pw...@gmail.com>
Authored: Tue Sep 10 21:38:22 2013 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Tue Sep 10 22:05:13 2013 -0700
----------------------------------------------------------------------
assembly/pom.xml | 6 ++++++
project/SparkBuild.scala | 1 +
2 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0c1985b1/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index d19f44d..808a829 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -106,6 +106,12 @@
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>META-INF/services/org.apache.hadoop.fs.FileSystem</resource>
+ </transformer>
+ </transformers>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
</transformers>
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0c1985b1/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index a60b553..0736ec3 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -303,6 +303,7 @@ object SparkBuild extends Build {
mergeStrategy in assembly := {
case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard
case m if m.toLowerCase.matches("meta-inf.*\\.sf$") => MergeStrategy.discard
+ case "META-INF/services/org.apache.hadoop.fs.FileSystem" => MergeStrategy.concat
case "reference.conf" => MergeStrategy.concat
case _ => MergeStrategy.first
}