You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vb...@apache.org on 2020/05/12 18:40:43 UTC
[incubator-hudi] branch master updated: [HUDI-884] Shade avro and
parquet-avro in hudi-hive-sync-bundle (#1618)
This is an automated email from the ASF dual-hosted git repository.
vbalaji pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 404c7e8 [HUDI-884] Shade avro and parquet-avro in hudi-hive-sync-bundle (#1618)
404c7e8 is described below
commit 404c7e82d9b538c356f93ffbbcb2c20d23310fd0
Author: Udit Mehrotra <um...@illinois.edu>
AuthorDate: Tue May 12 11:40:31 2020 -0700
[HUDI-884] Shade avro and parquet-avro in hudi-hive-sync-bundle (#1618)
Co-authored-by: Mehrotra <ud...@amazon.com>
---
.../java/org/apache/hudi/integ/ITTestHoodieDemo.java | 11 +++++++++--
packaging/hudi-hive-sync-bundle/pom.xml | 19 +++++++++++++++++++
2 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java
index ad18eae..4657092 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java
@@ -125,8 +125,15 @@ public class ITTestHoodieDemo extends ITTestBase {
+ " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
+ " --target-base-path " + COW_BASE_PATH + " --target-table " + COW_TABLE_NAME
+ " --props /var/demo/config/dfs-source.properties"
- + " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
- + String.format(HIVE_SYNC_CMD_FMT, "dt", COW_TABLE_NAME),
+ + " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider ",
+ "spark-submit --class org.apache.hudi.hive.HiveSyncTool " + HUDI_HIVE_SYNC_BUNDLE
+ + " --database default"
+ + " --table " + COW_TABLE_NAME
+ + " --base-path " + COW_BASE_PATH
+ + " --user hive"
+ + " --pass hive"
+ + " --jdbc-url jdbc:hive2://hiveserver:10000"
+ + " --partitioned-by dt",
("spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE
+ " --table-type MERGE_ON_READ "
+ " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
diff --git a/packaging/hudi-hive-sync-bundle/pom.xml b/packaging/hudi-hive-sync-bundle/pom.xml
index 0e9d894..0c2a087 100644
--- a/packaging/hudi-hive-sync-bundle/pom.xml
+++ b/packaging/hudi-hive-sync-bundle/pom.xml
@@ -69,12 +69,19 @@
<include>org.apache.hudi:hudi-hive-sync</include>
<include>com.beust:jcommander</include>
+ <include>org.apache.avro:avro</include>
<include>org.apache.parquet:parquet-avro</include>
<include>com.esotericsoftware:kryo-shaded</include>
<include>org.objenesis:objenesis</include>
<include>com.esotericsoftware:minlog</include>
</includes>
</artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.avro.</pattern>
+ <shadedPattern>org.apache.hudi.org.apache.avro.</shadedPattern>
+ </relocation>
+ </relocations>
<createDependencyReducedPom>false</createDependencyReducedPom>
<filters>
<filter>
@@ -120,5 +127,17 @@
<artifactId>hudi-hive-sync</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-avro</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
</project>