You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by su...@apache.org on 2019/09/10 03:53:33 UTC
[incubator-gobblin] branch master updated: [GOBBLIN-877] Add column
metadata for partition for inline hive registration
This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new febcf2f [GOBBLIN-877] Add column metadata for partition for inline hive registration
febcf2f is described below
commit febcf2f461f5596ab99cf1dee9ab5beee1054992
Author: Zihan Li <zi...@zihli-mn1.linkedin.biz>
AuthorDate: Mon Sep 9 20:53:26 2019 -0700
[GOBBLIN-877] Add column metadata for partition for inline hive registration
Closes #2731 from ZihanLi58/GOBBLIN-877
---
.../java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtils.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtils.java b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtils.java
index efd0b9c..bb6a8b8 100644
--- a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtils.java
+++ b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtils.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeUtils;
@@ -211,7 +212,9 @@ public class HiveMetaStoreUtils {
State props = unit.getStorageProps();
StorageDescriptor sd = new StorageDescriptor();
sd.setParameters(getParameters(props));
- if (unit.isRegisterSchema()) {
+ //Treat AVRO and other formats differently. Details can be found in GOBBLIN-877
+ if (unit.isRegisterSchema() ||
+ (unit.getInputFormat().isPresent() && !unit.getInputFormat().get().equals(AvroContainerInputFormat.class.getName()))) {
sd.setCols(getFieldSchemas(unit));
}
if (unit.getLocation().isPresent()) {