You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vb...@apache.org on 2019/09/17 19:38:56 UTC
[incubator-hudi] branch master updated: [HUDI-254]: Bundle and
shade databricks/avro with spark bundle
This is an automated email from the ASF dual-hosted git repository.
vbalaji pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new e217db5 [HUDI-254]: Bundle and shade databricks/avro with spark bundle
e217db5 is described below
commit e217db56ab30ec9630411aefc16516747b3112e0
Author: Vinoth Chandar <vi...@apache.org>
AuthorDate: Tue Sep 17 12:00:41 2019 -0700
[HUDI-254]: Bundle and shade databricks/avro with spark bundle
- spark 2.4 onwards, spark has built in support. shading to avoid conflicts
- spark 2.3 still needs this bundled, so that dropping bundle into jars folder would work
---
hudi-spark/pom.xml | 1 -
packaging/hudi-spark-bundle/pom.xml | 6 ++++++
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/hudi-spark/pom.xml b/hudi-spark/pom.xml
index eb86e75..2701ac4 100644
--- a/hudi-spark/pom.xml
+++ b/hudi-spark/pom.xml
@@ -216,7 +216,6 @@
<groupId>com.databricks</groupId>
<artifactId>spark-avro_2.11</artifactId>
<version>4.0.0</version>
- <scope>provided</scope>
</dependency>
<!-- Hadoop -->
diff --git a/packaging/hudi-spark-bundle/pom.xml b/packaging/hudi-spark-bundle/pom.xml
index 153e69b..e8ef205 100644
--- a/packaging/hudi-spark-bundle/pom.xml
+++ b/packaging/hudi-spark-bundle/pom.xml
@@ -84,6 +84,8 @@
<include>org.apache.hive:hive-service-rpc</include>
<include>org.apache.hive:hive-metastore</include>
<include>org.apache.hive:hive-jdbc</include>
+
+ <include>com.databricks:spark-avro_2.11</include>
</includes>
</artifactSet>
<relocations>
@@ -127,6 +129,10 @@
<pattern>org.apache.commons.codec.</pattern>
<shadedPattern>org.apache.hudi.org.apache.commons.codec.</shadedPattern>
</relocation>
+ <relocation>
+ <pattern>com.databricks.</pattern>
+ <shadedPattern>org.apache.hudi.com.databricks.</shadedPattern>
+ </relocation>
<!-- TODO: Revisit GH ISSUE #533 & PR#633-->
<!--
<relocation>