You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/01/31 02:22:18 UTC
hive git commit: HIVE-15714 : backport HIVE-11985 (and HIVE-12601) to
branch-1 (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/branch-1 f746dc6ab -> 70eb1f6e2
HIVE-15714 : backport HIVE-11985 (and HIVE-12601) to branch-1 (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/70eb1f6e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/70eb1f6e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/70eb1f6e
Branch: refs/heads/branch-1
Commit: 70eb1f6e2b9b15f74c65ee480d07e3ef35b672dc
Parents: f746dc6
Author: Sergey Shelukhin <se...@apache.org>
Authored: Mon Jan 30 18:22:18 2017 -0800
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Mon Jan 30 18:22:18 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 14 +++++---
.../hadoop/hive/metastore/MetaStoreUtils.java | 24 ++++++++++---
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 25 +++++++++----
.../apache/hadoop/hive/ql/metadata/Hive.java | 17 +++++++--
.../hadoop/hive/ql/metadata/Partition.java | 15 +++++++-
.../apache/hadoop/hive/ql/metadata/Table.java | 38 +++++++++++++++++---
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 5 +++
.../hadoop/hive/serde2/AbstractSerDe.java | 9 +++++
.../hadoop/hive/serde2/avro/AvroSerDe.java | 19 ++++++++--
9 files changed, 141 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 93cf71c..10b0644 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -874,11 +874,15 @@ public class HiveConf extends Configuration {
"The default SerDe Hive will use for storage formats that do not specify a SerDe."),
SERDESUSINGMETASTOREFORSCHEMA("hive.serdes.using.metastore.for.schema",
- "org.apache.hadoop.hive.ql.io.orc.OrcSerde,org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe," +
- "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe,org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe," +
- "org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe,org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe," +
- "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe,org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe",
- "SerDes retriving schema from metastore. This an internal parameter. Check with the hive dev. team"),
+ "org.apache.hadoop.hive.ql.io.orc.OrcSerde," +
+ "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe," +
+ "org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe," +
+ "org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe," +
+ "org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe," +
+ "org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe," +
+ "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe," +
+ "org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe",
+ "SerDes retrieving schema from metastore. This is an internal parameter."),
HIVEHISTORYFILELOC("hive.querylog.location",
"${system:java.io.tmpdir}" + File.separator + "${system:user.name}",
http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 961359e..7878f16 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -385,6 +385,12 @@ public class MetaStoreUtils {
if (lib == null) {
return null;
}
+ return getDeserializer(conf, table, skipConfError, lib);
+ }
+
+ public static Deserializer getDeserializer(Configuration conf,
+ org.apache.hadoop.hive.metastore.api.Table table, boolean skipConfError,
+ String lib) throws MetaException {
try {
Deserializer deserializer = ReflectionUtil.newInstance(conf.getClassByName(lib).
asSubclass(Deserializer.class), conf);
@@ -549,8 +555,9 @@ public class MetaStoreUtils {
if (!validateColumnName(fieldSchema.getName())) {
return "name: " + fieldSchema.getName();
}
- if (!validateColumnType(fieldSchema.getType())) {
- return "type: " + fieldSchema.getType();
+ String typeError = validateColumnType(fieldSchema.getType());
+ if (typeError != null) {
+ return typeError;
}
}
return null;
@@ -625,6 +632,9 @@ public class MetaStoreUtils {
return false;
}
+ public static final int MAX_MS_TYPENAME_LENGTH = 2000; // 4000/2, for an unlikely unicode case
+
+ public static final String TYPE_FROM_DESERIALIZER = "<derived from deserializer>";
/**
* validate column type
*
@@ -632,7 +642,11 @@ public class MetaStoreUtils {
* @param name
* @return
*/
- static public boolean validateColumnType(String type) {
+ static public String validateColumnType(String type) {
+ if (type.equals(TYPE_FROM_DESERIALIZER)) return null;
+ if (type.length() > MAX_MS_TYPENAME_LENGTH) {
+ return "type name is too long: " + type;
+ }
int last = 0;
boolean lastAlphaDigit = isValidTypeChar(type.charAt(last));
for (int i = 1; i <= type.length(); i++) {
@@ -641,12 +655,12 @@ public class MetaStoreUtils {
String token = type.substring(last, i);
last = i;
if (!hiveThriftTypeMap.contains(token)) {
- return false;
+ return "type: " + type;
}
break;
}
}
- return true;
+ return null;
}
private static boolean isValidTypeChar(char c) {
http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index cb667fc..7d972e5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -204,6 +204,7 @@ import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.SerDeSpec;
import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe;
import org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe;
@@ -3112,7 +3113,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
cols.addAll(tbl.getPartCols());
}
} else {
- cols = Hive.getFieldsFromDeserializer(colPath, deserializer);
+ cols = Hive.getFieldsFromDeserializer(colPath, deserializer); // TODO#: here - desc
if (descTbl.isFormatted()) {
// when column name is specified in describe table DDL, colPath will
// will be table_name.column_name
@@ -3366,7 +3367,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
tbl.setDbName(Utilities.getDatabaseName(alterTbl.getNewName()));
tbl.setTableName(Utilities.getTableName(alterTbl.getNewName()));
} else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDCOLS) {
- List<FieldSchema> oldCols = (part == null ? tbl.getCols() : part.getCols());
+ List<FieldSchema> oldCols = (part == null
+ ? tbl.getColsForMetastore() : part.getColsForMetastore());
StorageDescriptor sd = (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd());
List<FieldSchema> newCols = alterTbl.getNewCols();
String serializationLib = sd.getSerdeInfo().getSerializationLib();
@@ -3394,7 +3396,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
sd.setCols(oldCols);
}
} else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.RENAMECOLUMN) {
- List<FieldSchema> oldCols = (part == null ? tbl.getCols() : part.getCols());
+ List<FieldSchema> oldCols = (part == null
+ ? tbl.getColsForMetastore() : part.getColsForMetastore());
StorageDescriptor sd = (part == null ? tbl.getTTable().getSd() : part.getTPartition().getSd());
List<FieldSchema> newCols = new ArrayList<FieldSchema>();
Iterator<FieldSchema> iterOldCols = oldCols.iterator();
@@ -3536,11 +3539,21 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
sd.getSerdeInfo().getParameters().putAll(alterTbl.getProps());
}
if (part != null) {
+ // TODO: wtf? This doesn't do anything.
part.getTPartition().getSd().setCols(part.getTPartition().getSd().getCols());
} else {
- if (!Table.hasMetastoreBasedSchema(conf, serdeName)) {
- tbl.setFields(Hive.getFieldsFromDeserializer(tbl.getTableName(), tbl.
- getDeserializer()));
+ if (Table.shouldStoreFieldsInMetastore(conf, serdeName, tbl.getParameters())
+ && !Table.hasMetastoreBasedSchema(conf, oldSerdeName)) {
+ // If new SerDe needs to store fields in metastore, but the old serde doesn't, save
+ // the fields so that new SerDe could operate. Note that this may fail if some fields
+ // from old SerDe are too long to be stored in metastore, but there's nothing we can do.
+ try {
+ Deserializer oldSerde = MetaStoreUtils.getDeserializer(
+ conf, tbl.getTTable(), false, oldSerdeName);
+ tbl.setFields(Hive.getFieldsFromDeserializer(tbl.getTableName(), oldSerde));
+ } catch (MetaException ex) {
+ throw new HiveException(ex);
+ }
}
}
} else if (alterTbl.getOp() == AlterTableDesc.AlterTableTypes.ADDFILEFORMAT) {
http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index c2ab4bf..48b781d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -724,6 +724,16 @@ public class Hive {
}
}
+ public static List<FieldSchema> getFieldsFromDeserializerForMsStorage(
+ Table tbl, Deserializer deserializer) throws SerDeException, MetaException {
+ List<FieldSchema> schema = MetaStoreUtils.getFieldsFromDeserializer(
+ tbl.getTableName(), deserializer);
+ for (FieldSchema field : schema) {
+ field.setType(MetaStoreUtils.TYPE_FROM_DESERIALIZER);
+ }
+ return schema;
+ }
+
/**
*
* @param tableName
@@ -848,8 +858,11 @@ public class Hive {
List<Order> sortCols = new ArrayList<Order>();
int k = 0;
Table metaBaseTbl = new Table(baseTbl);
- for (int i = 0; i < metaBaseTbl.getCols().size(); i++) {
- FieldSchema col = metaBaseTbl.getCols().get(i);
+ // Even though we are storing these in metastore, get regular columns. Indexes on lengthy
+ // types from e.g. Avro schema will just fail to create the index table (by design).
+ List<FieldSchema> cols = metaBaseTbl.getCols();
+ for (int i = 0; i < cols.size(); i++) {
+ FieldSchema col = cols.get(i);
if (indexedCols.contains(col.getName())) {
indexTblCols.add(col);
sortCols.add(new Order(col.getName(), 1));
http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
index 08ff2e9..21a55dd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
@@ -492,10 +492,23 @@ public class Partition implements Serializable {
}
public List<FieldSchema> getCols() {
+ return getColsInternal(false);
+ }
+
+ public List<FieldSchema> getColsForMetastore() {
+ return getColsInternal(true);
+ }
+
+ private List<FieldSchema> getColsInternal(boolean forMs) {
try {
- if (Table.hasMetastoreBasedSchema(SessionState.getSessionConf(), tPartition.getSd())) {
+ String serializationLib = tPartition.getSd().getSerdeInfo().getSerializationLib();
+ // Do the lightweight check for general case.
+ if (Table.hasMetastoreBasedSchema(SessionState.getSessionConf(), serializationLib)) {
return tPartition.getSd().getCols();
+ } else if (forMs && !Table.shouldStoreFieldsInMetastore(
+ SessionState.getSessionConf(), serializationLib, table.getParameters())) {
+ return Hive.getFieldsFromDeserializerForMsStorage(table, getDeserializer());
}
return MetaStoreUtils.getFieldsFromDeserializer(table.getTableName(), getDeserializer());
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
index e53933e..fddb6d5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
@@ -64,6 +65,7 @@ import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.OutputFormat;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hive.common.util.ReflectionUtil;
/**
* A Hive Table: is a fundamental unit of data in Hive that shares a common schema/DDL.
@@ -596,11 +598,22 @@ public class Table implements Serializable {
}
public List<FieldSchema> getCols() {
+ return getColsInternal(false);
+ }
+
+ public List<FieldSchema> getColsForMetastore() {
+ return getColsInternal(true);
+ }
+ private List<FieldSchema> getColsInternal(boolean forMs) {
String serializationLib = getSerializationLib();
try {
+ // Do the lightweight check for general case.
if (hasMetastoreBasedSchema(SessionState.getSessionConf(), serializationLib)) {
return tTable.getSd().getCols();
+ } else if (forMs && !shouldStoreFieldsInMetastore(
+ SessionState.getSessionConf(), serializationLib, tTable.getParameters())) {
+ return Hive.getFieldsFromDeserializerForMsStorage(this, getDeserializer());
} else {
return MetaStoreUtils.getFieldsFromDeserializer(getTableName(), getDeserializer());
}
@@ -934,15 +947,28 @@ public class Table implements Serializable {
return tTable.isTemporary();
}
- public static boolean hasMetastoreBasedSchema(HiveConf conf, StorageDescriptor serde) {
- return hasMetastoreBasedSchema(conf, serde.getSerdeInfo().getSerializationLib());
- }
-
public static boolean hasMetastoreBasedSchema(HiveConf conf, String serdeLib) {
return StringUtils.isEmpty(serdeLib) ||
conf.getStringCollection(ConfVars.SERDESUSINGMETASTOREFORSCHEMA.varname).contains(serdeLib);
}
+ public static boolean shouldStoreFieldsInMetastore(
+ HiveConf conf, String serdeLib, Map<String, String> tableParams) {
+ if (hasMetastoreBasedSchema(conf, serdeLib)) return true;
+ // Table may or may not be using metastore. Only the SerDe can tell us.
+ AbstractSerDe deserializer = null;
+ try {
+ Class<?> clazz = conf.getClassByName(serdeLib);
+ if (!AbstractSerDe.class.isAssignableFrom(clazz)) return true; // The default.
+ deserializer = ReflectionUtil.newInstance(
+ conf.getClassByName(serdeLib).asSubclass(AbstractSerDe.class), conf);
+ } catch (Exception ex) {
+ LOG.warn("Cannot initialize SerDe: " + serdeLib + ", ignoring", ex);
+ return true;
+ }
+ return deserializer.shouldStoreFieldsInMetastore(tableParams);
+ }
+
public static void validateColumns(List<FieldSchema> columns, List<FieldSchema> partCols)
throws HiveException {
List<String> colNames = new ArrayList<String>();
@@ -982,4 +1008,8 @@ public class Table implements Serializable {
this.tableSpec = tableSpec;
}
+ public boolean hasDeserializer() {
+ return deserializer != null;
+ }
+
};
http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 74f5867..98d3fc5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -10433,6 +10433,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
FieldSchema fieldSchema = derivedSchema.get(i);
// Modify a copy, not the original
fieldSchema = new FieldSchema(fieldSchema);
+ // TODO: there's a potential problem here if some table uses external schema like Avro,
+ // with a very large type name. It seems like the view does not derive the SerDe from
+ // the table, so it won't be able to just get the type from the deserializer like the
+ // table does; we won't be able to properly store the type in the RDBMS metastore.
+ // Not sure if these large cols could be in resultSchema. Ignore this for now 0_o
derivedSchema.set(i, fieldSchema);
sb.append(HiveUtils.unparseIdentifier(fieldSchema.getName(), conf));
sb.append(" AS ");
http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java
index c5e78c5..9269ff4 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/AbstractSerDe.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.serde2;
+import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
@@ -115,4 +116,12 @@ public abstract class AbstractSerDe implements SerDe {
public String getConfigurationErrors() {
return configErrors == null ? "" : configErrors;
}
+
+ /**
+ * @rturn Whether the SerDe that can store schema both inside and outside of metastore
+ * does, in fact, store it inside metastore, based on table parameters.
+ */
+ public boolean shouldStoreFieldsInMetastore(Map<String, String> tableParams) {
+ return false; // The default, unless SerDe overrides it. TODO#
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/70eb1f6e/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
index efff663..7271095 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroSerDe.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.serde2.avro;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
import org.apache.avro.Schema;
@@ -96,8 +97,7 @@ public class AvroSerDe extends AbstractSerDe {
final String columnTypeProperty = properties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
final String columnCommentProperty = properties.getProperty(LIST_COLUMN_COMMENTS,"");
- if (properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName()) != null
- || properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_URL.getPropName()) != null
+ if (hasExternalSchema(properties)
|| columnNameProperty == null || columnNameProperty.isEmpty()
|| columnTypeProperty == null || columnTypeProperty.isEmpty()) {
schema = determineSchemaOrReturnErrorSchema(configuration, properties);
@@ -127,6 +127,16 @@ public class AvroSerDe extends AbstractSerDe {
this.oi = aoig.getObjectInspector();
}
+ private boolean hasExternalSchema(Properties properties) {
+ return properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName()) != null
+ || properties.getProperty(AvroSerdeUtils.AvroTableProperties.SCHEMA_URL.getPropName()) != null;
+ }
+
+ private boolean hasExternalSchema(Map<String, String> tableParams) {
+ return tableParams.containsKey(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName())
+ || tableParams.containsKey(AvroSerdeUtils.AvroTableProperties.SCHEMA_URL.getPropName());
+ }
+
public static Schema getSchemaFromCols(Properties properties,
List<String> columnNames, List<TypeInfo> columnTypes, String columnCommentProperty) {
List<String> columnComments;
@@ -227,4 +237,9 @@ public class AvroSerDe extends AbstractSerDe {
return avroSerializer;
}
+
+ @Override
+ public boolean shouldStoreFieldsInMetastore(Map<String, String> tableParams) {
+ return !hasExternalSchema(tableParams);
+ }
}