You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by hu...@apache.org on 2019/08/16 16:36:33 UTC
[incubator-gobblin] branch master updated: [GOBBLIN-851] Provide
capability to disable Hive partition schema registration.
This is an automated email from the ASF dual-hosted git repository.
hutran pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new c210e92 [GOBBLIN-851] Provide capability to disable Hive partition schema registration.
c210e92 is described below
commit c210e9281da3c4a65d8836e8010b1b8331e3f38f
Author: Kuai Yu <ku...@linkedin.com>
AuthorDate: Fri Aug 16 09:36:25 2019 -0700
[GOBBLIN-851] Provide capability to disable Hive partition schema registration.
Closes #2707 from yukuai518/schema
---
.../org/apache/gobblin/hive/HiveRegistrationUnit.java | 8 ++++++++
.../gobblin/hive/avro/HiveAvroSerDeManager.java | 19 ++++++++++---------
.../gobblin/hive/metastore/HiveMetaStoreUtils.java | 4 +++-
.../apache/gobblin/metrics/event/EventSubmitter.java | 4 +++-
4 files changed, 24 insertions(+), 11 deletions(-)
diff --git a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveRegistrationUnit.java b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveRegistrationUnit.java
index 8995a4b..adb024b 100644
--- a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveRegistrationUnit.java
+++ b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/HiveRegistrationUnit.java
@@ -50,6 +50,7 @@ public class HiveRegistrationUnit {
protected final String dbName;
protected final String tableName;
+ protected final boolean registerSchema;
protected final List<Column> columns = Lists.newArrayList();
protected final State props = new State();
protected final State storageProps = new State();
@@ -85,6 +86,7 @@ public class HiveRegistrationUnit {
this.dbName = builder.dbName;
this.tableName = builder.tableName;
+ this.registerSchema = builder.registerSchema;
this.columns.addAll(builder.columns);
this.props.addAll(builder.props);
this.storageProps.addAll(builder.storageProps);
@@ -400,6 +402,7 @@ public class HiveRegistrationUnit {
static abstract class Builder<T extends Builder<?>> {
private String dbName;
private String tableName;
+ private boolean registerSchema = true;
private List<Column> columns = Lists.newArrayList();
private State props = new State();
private State storageProps = new State();
@@ -418,6 +421,11 @@ public class HiveRegistrationUnit {
return (T) this;
}
+ public T withRegisterSchema(boolean registerSchema) {
+ this.registerSchema = registerSchema;
+ return (T) this;
+ }
+
@SuppressWarnings("unchecked")
public T withColumns(List<Column> columns) {
this.columns = columns;
diff --git a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/avro/HiveAvroSerDeManager.java b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/avro/HiveAvroSerDeManager.java
index 6a8f1f6..9c4e960 100644
--- a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/avro/HiveAvroSerDeManager.java
+++ b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/avro/HiveAvroSerDeManager.java
@@ -122,8 +122,7 @@ public class HiveAvroSerDeManager extends HiveSerDeManager {
hiveUnit.setSerDeType(this.serDeWrapper.getSerDe().getClass().getName());
hiveUnit.setInputFormat(this.serDeWrapper.getInputFormatClassName());
hiveUnit.setOutputFormat(this.serDeWrapper.getOutputFormatClassName());
-
- addSchemaProperties(path, hiveUnit, schema);
+ addSchemaPropertiesIfRequired(path, hiveUnit, schema);
}
@Override
@@ -145,13 +144,15 @@ public class HiveAvroSerDeManager extends HiveSerDeManager {
}
}
- private void addSchemaProperties(Path path, HiveRegistrationUnit hiveUnit, Schema schema) throws IOException {
- Path schemaFile = new Path(path, this.schemaFileName);
- if (this.useSchemaFile) {
- hiveUnit.setSerDeProp(SCHEMA_URL, schemaFile.toString());
- } else {
- try (Timer.Context context = metricContext.timer(HIVE_SPEC_SCHEMA_WRITING_TIMER).time()) {
- addSchemaFromAvroFile(schema, schemaFile, hiveUnit);
+ private void addSchemaPropertiesIfRequired(Path path, HiveRegistrationUnit hiveUnit, Schema schema) throws IOException {
+ if (hiveUnit.isRegisterSchema()) {
+ Path schemaFile = new Path(path, this.schemaFileName);
+ if (this.useSchemaFile) {
+ hiveUnit.setSerDeProp(SCHEMA_URL, schemaFile.toString());
+ } else {
+ try (Timer.Context context = metricContext.timer(HIVE_SPEC_SCHEMA_WRITING_TIMER).time()) {
+ addSchemaFromAvroFile(schema, schemaFile, hiveUnit);
+ }
}
}
}
diff --git a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtils.java b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtils.java
index b05f697..3064c35 100644
--- a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtils.java
+++ b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtils.java
@@ -211,7 +211,9 @@ public class HiveMetaStoreUtils {
State props = unit.getStorageProps();
StorageDescriptor sd = new StorageDescriptor();
sd.setParameters(getParameters(props));
- sd.setCols(getFieldSchemas(unit));
+ if (unit.isRegisterSchema()) {
+ sd.setCols(getFieldSchemas(unit));
+ }
if (unit.getLocation().isPresent()) {
sd.setLocation(unit.getLocation().get());
}
diff --git a/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/event/EventSubmitter.java b/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/event/EventSubmitter.java
index d3c0d1f..891f980 100644
--- a/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/event/EventSubmitter.java
+++ b/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/event/EventSubmitter.java
@@ -87,7 +87,9 @@ public class EventSubmitter {
if(eventBuilder.namespace == null) {
eventBuilder.setNamespace(this.namespace);
}
- this.metricContext.get().submitEvent(eventBuilder.build());
+ if (metricContext.isPresent()) {
+ this.metricContext.get().submitEvent(eventBuilder.build());
+ }
}
/**