You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2017/09/08 20:03:27 UTC
hive git commit: HIVE-17359 Deal with TypeInfo dependencies in the
metastore. This closes #239. (Alan Gates, reviewed by Owen O'Malley)
Repository: hive
Updated Branches:
refs/heads/master 65480cecf -> 5e57a2a86
HIVE-17359 Deal with TypeInfo dependencies in the metastore. This closes #239. (Alan Gates, reviewed by Owen O'Malley)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5e57a2a8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5e57a2a8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5e57a2a8
Branch: refs/heads/master
Commit: 5e57a2a8675760659a7a6a48c8cd39c2b2bf5dc0
Parents: 65480ce
Author: Alan Gates <ga...@hortonworks.com>
Authored: Fri Sep 8 13:02:21 2017 -0700
Committer: Alan Gates <ga...@hortonworks.com>
Committed: Fri Sep 8 13:02:21 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/metastore/HiveAlterHandler.java | 24 +-
.../hive/metastore/MetaStoreDirectSql.java | 7 +-
.../hadoop/hive/metastore/ObjectStore.java | 11 +-
.../metastore/PartitionExpressionProxy.java | 10 +-
.../hive/metastore/cache/CachedStore.java | 12 +-
.../hive/metastore/parser/ExpressionTree.java | 6 +-
.../MockPartitionExpressionForMetastore.java | 5 +-
.../hadoop/hive/metastore/TestObjectStore.java | 11 +-
.../hadoop/hive/metastore/TestOldSchema.java | 6 +-
.../ppr/PartitionExpressionForMetastore.java | 14 +-
.../hadoop/hive/metastore/ColumnType.java | 242 +++++++++++++++++++
11 files changed, 297 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5e57a2a8/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 28c3cfe..b279e1d 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -156,8 +156,7 @@ public class HiveAlterHandler implements AlterHandler {
false)) {
// Throws InvalidOperationException if the new column types are not
// compatible with the current column types.
- MetaStoreUtils.throwExceptionIfIncompatibleColTypeChange(
- oldt.getSd().getCols(), newt.getSd().getCols());
+ checkColTypeChangeCompatible(oldt.getSd().getCols(), newt.getSd().getCols());
}
//check that partition keys have not changed, except for virtual views
@@ -852,4 +851,25 @@ public class HiveAlterHandler implements AlterHandler {
return newPartsColStats;
}
+
+ private void checkColTypeChangeCompatible(List<FieldSchema> oldCols, List<FieldSchema> newCols)
+ throws InvalidOperationException {
+ List<String> incompatibleCols = new ArrayList<>();
+ int maxCols = Math.min(oldCols.size(), newCols.size());
+ for (int i = 0; i < maxCols; i++) {
+ if (!ColumnType.areColTypesCompatible(
+ ColumnType.getTypeName(oldCols.get(i).getType()),
+ ColumnType.getTypeName(newCols.get(i).getType()))) {
+ incompatibleCols.add(newCols.get(i).getName());
+ }
+ }
+ if (!incompatibleCols.isEmpty()) {
+ throw new InvalidOperationException(
+ "The following columns have types incompatible with the existing " +
+ "columns in their respective positions :\n" +
+ org.apache.commons.lang.StringUtils.join(incompatibleCols, ',')
+ );
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e57a2a8/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 2dd7b68..b122c43 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -76,7 +76,6 @@ import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LogicalOperator;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeVisitor;
-import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hive.common.util.BloomFilter;
import org.datanucleus.store.rdbms.query.ForwardQueryResult;
import org.slf4j.Logger;
@@ -1112,11 +1111,11 @@ class MetaStoreDirectSql {
Invalid;
static FilterType fromType(String colTypeStr) {
- if (colTypeStr.equals(serdeConstants.STRING_TYPE_NAME)) {
+ if (colTypeStr.equals(ColumnType.STRING_TYPE_NAME)) {
return FilterType.String;
- } else if (colTypeStr.equals(serdeConstants.DATE_TYPE_NAME)) {
+ } else if (colTypeStr.equals(ColumnType.DATE_TYPE_NAME)) {
return FilterType.Date;
- } else if (serdeConstants.IntegralTypes.contains(colTypeStr)) {
+ } else if (ColumnType.IntegralTypes.contains(colTypeStr)) {
return FilterType.Integral;
}
return FilterType.Invalid;
http://git-wip-us.apache.org/repos/asf/hive/blob/5e57a2a8/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 0db1bc0..b01e59e 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -163,8 +163,6 @@ import org.apache.hadoop.hive.metastore.model.MMetastoreDBProperties;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree;
import org.apache.hadoop.hive.metastore.parser.ExpressionTree.FilterBuilder;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.util.StringUtils;
import org.apache.hive.common.util.HiveStringUtils;
import org.apache.thrift.TException;
@@ -2558,17 +2556,10 @@ public class ObjectStore implements RawStore, Configurable {
String defaultPartName, short maxParts, List<String> result) throws MetaException {
result.addAll(getPartitionNamesNoTxn(
table.getDbName(), table.getTableName(), maxParts));
- List<String> columnNames = new ArrayList<String>();
- List<PrimitiveTypeInfo> typeInfos = new ArrayList<PrimitiveTypeInfo>();
- for (FieldSchema fs : table.getPartitionKeys()) {
- columnNames.add(fs.getName());
- typeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()));
- }
if (defaultPartName == null || defaultPartName.isEmpty()) {
defaultPartName = HiveConf.getVar(getConf(), HiveConf.ConfVars.DEFAULTPARTITIONNAME);
}
- return expressionProxy.filterPartitionsByExpr(
- columnNames, typeInfos, expr, defaultPartName, result);
+ return expressionProxy.filterPartitionsByExpr(table.getPartitionKeys(), expr, defaultPartName, result);
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/5e57a2a8/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java b/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
index 8d6dc12..af0a6bd 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/PartitionExpressionProxy.java
@@ -20,10 +20,10 @@ package org.apache.hadoop.hive.metastore;
import java.util.List;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
/**
* The proxy interface that metastore uses for variety of QL operations (metastore can't depend
@@ -41,16 +41,14 @@ public interface PartitionExpressionProxy {
/**
* Filters the partition names via serialized Hive expression.
- * @param partColumnNames Partition column names in the underlying table.
- * @param partColumnTypeInfos Partition column types in the underlying table
+ * @param partColumns Partition columns in the underlying table.
* @param expr Serialized expression.
* @param defaultPartitionName Default partition name from job or server configuration.
* @param partitionNames Partition names; the list is modified in place.
* @return Whether there were any unknown partitions preserved in the name list.
*/
- boolean filterPartitionsByExpr(List<String> partColumnNames,
- List<PrimitiveTypeInfo> partColumnTypeInfos, byte[] expr,
- String defaultPartitionName, List<String> partitionNames) throws MetaException;
+ boolean filterPartitionsByExpr(List<FieldSchema> partColumns,
+ byte[] expr, String defaultPartitionName, List<String> partitionNames) throws MetaException;
/**
* Determines the file metadata type from input format of the source table or partition.
http://git-wip-us.apache.org/repos/asf/hive/blob/5e57a2a8/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
index 93d1ba6..3ba81ce 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/cache/CachedStore.java
@@ -50,7 +50,6 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
import org.apache.hadoop.hive.metastore.api.Function;
import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege;
@@ -84,8 +83,6 @@ import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
import org.apache.hadoop.hive.metastore.api.UnknownTableException;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hive.common.util.HiveStringUtils;
import org.apache.thrift.TException;
import org.slf4j.Logger;
@@ -1048,17 +1045,10 @@ public class CachedStore implements RawStore, Configurable {
for (Partition part : parts) {
result.add(Warehouse.makePartName(table.getPartitionKeys(), part.getValues()));
}
- List<String> columnNames = new ArrayList<String>();
- List<PrimitiveTypeInfo> typeInfos = new ArrayList<PrimitiveTypeInfo>();
- for (FieldSchema fs : table.getPartitionKeys()) {
- columnNames.add(fs.getName());
- typeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()));
- }
if (defaultPartName == null || defaultPartName.isEmpty()) {
defaultPartName = HiveConf.getVar(getConf(), HiveConf.ConfVars.DEFAULTPARTITIONNAME);
}
- return expressionProxy.filterPartitionsByExpr(
- columnNames, typeInfos, expr, defaultPartName, result);
+ return expressionProxy.filterPartitionsByExpr(table.getPartitionKeys(), expr, defaultPartName, result);
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/5e57a2a8/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java b/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
index 8b12899..12773ac 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java
@@ -26,13 +26,13 @@ import java.util.Stack;
import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.CharStream;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.ColumnType;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
-import org.apache.hadoop.hive.serde.serdeConstants;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
@@ -432,8 +432,8 @@ public class ExpressionTree {
boolean isIntegralSupported = canPushDownIntegral && canJdoUseStringsWithIntegral();
String colType = table.getPartitionKeys().get(partColIndex).getType();
// Can only support partitions whose types are string, or maybe integers
- if (!colType.equals(serdeConstants.STRING_TYPE_NAME)
- && (!isIntegralSupported || !serdeConstants.IntegralTypes.contains(colType))) {
+ if (!colType.equals(ColumnType.STRING_TYPE_NAME)
+ && (!isIntegralSupported || !ColumnType.IntegralTypes.contains(colType))) {
filterBuilder.setError("Filtering is supported only on partition keys of type " +
"string" + (isIntegralSupported ? ", or integral types" : ""));
return null;
http://git-wip-us.apache.org/repos/asf/hive/blob/5e57a2a8/metastore/src/test/org/apache/hadoop/hive/metastore/MockPartitionExpressionForMetastore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/MockPartitionExpressionForMetastore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/MockPartitionExpressionForMetastore.java
index 032af69..12a862d 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/MockPartitionExpressionForMetastore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/MockPartitionExpressionForMetastore.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.metastore;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
@@ -35,8 +36,8 @@ public class MockPartitionExpressionForMetastore implements PartitionExpressionP
}
@Override
- public boolean filterPartitionsByExpr(List<String> partColumnNames,
- List<PrimitiveTypeInfo> partColumnTypeInfos, byte[] expr, String defaultPartitionName,
+ public boolean filterPartitionsByExpr(List<FieldSchema> partColumns,
+ byte[] expr, String defaultPartitionName,
List<String> partitionNames) throws MetaException {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e57a2a8/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
index 08228af..5448d0d 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java
@@ -50,8 +50,6 @@ import org.apache.hadoop.hive.metastore.messaging.EventMessage;
import org.apache.hadoop.hive.metastore.metrics.Metrics;
import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -86,9 +84,8 @@ public class TestObjectStore {
}
@Override
- public boolean filterPartitionsByExpr(List<String> partColumnNames,
- List<PrimitiveTypeInfo> partColumnTypeInfos, byte[] expr,
- String defaultPartitionName, List<String> partitionNames)
+ public boolean filterPartitionsByExpr(List<FieldSchema> partColumns,
+ byte[] expr, String defaultPartitionName, List<String> partitionNames)
throws MetaException {
return false;
}
@@ -278,8 +275,8 @@ public class TestObjectStore {
StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null);
HashMap<String,String> tableParams = new HashMap<String,String>();
tableParams.put("EXTERNAL", "false");
- FieldSchema partitionKey1 = new FieldSchema("Country", serdeConstants.STRING_TYPE_NAME, "");
- FieldSchema partitionKey2 = new FieldSchema("State", serdeConstants.STRING_TYPE_NAME, "");
+ FieldSchema partitionKey1 = new FieldSchema("Country", ColumnType.STRING_TYPE_NAME, "");
+ FieldSchema partitionKey2 = new FieldSchema("State", ColumnType.STRING_TYPE_NAME, "");
Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, Arrays.asList(partitionKey1, partitionKey2), tableParams, null, null, "MANAGED_TABLE");
objectStore.createTable(tbl1);
HashMap<String, String> partitionParams = new HashMap<String, String>();
http://git-wip-us.apache.org/repos/asf/hive/blob/5e57a2a8/metastore/src/test/org/apache/hadoop/hive/metastore/TestOldSchema.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestOldSchema.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestOldSchema.java
index 8312b34..8409d9b 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestOldSchema.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestOldSchema.java
@@ -62,9 +62,9 @@ public class TestOldSchema {
}
@Override
- public boolean filterPartitionsByExpr(List<String> partColumnNames,
- List<PrimitiveTypeInfo> partColumnTypeInfos, byte[] expr, String defaultPartitionName,
- List<String> partitionNames) throws MetaException {
+ public boolean filterPartitionsByExpr(List<FieldSchema> partColumns, byte[] expr,
+ String defaultPartitionName,
+ List<String> partitionNames) throws MetaException {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5e57a2a8/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
index 6555187..3d8852e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionExpressionForMetastore.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.hive.ql.optimizer.ppr;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.FileMetadataExprType;
+import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hive.metastore.FileFormatProxy;
@@ -35,6 +37,7 @@ import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,9 +53,14 @@ public class PartitionExpressionForMetastore implements PartitionExpressionProxy
}
@Override
- public boolean filterPartitionsByExpr(List<String> partColumnNames,
- List<PrimitiveTypeInfo> partColumnTypeInfos, byte[] exprBytes,
- String defaultPartitionName, List<String> partitionNames) throws MetaException {
+ public boolean filterPartitionsByExpr(List<FieldSchema> partColumns,
+ byte[] exprBytes, String defaultPartitionName, List<String> partitionNames) throws MetaException {
+ List<String> partColumnNames = new ArrayList<>();
+ List<PrimitiveTypeInfo> partColumnTypeInfos = new ArrayList<>();
+ for (FieldSchema fs : partColumns) {
+ partColumnNames.add(fs.getName());
+ partColumnTypeInfos.add(TypeInfoFactory.getPrimitiveTypeInfo(fs.getType()));
+ }
ExprNodeGenericFuncDesc expr = deserializeExpr(exprBytes);
try {
ExprNodeDescUtils.replaceEqualDefaultPartition(expr, defaultPartitionName);
http://git-wip-us.apache.org/repos/asf/hive/blob/5e57a2a8/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java
new file mode 100644
index 0000000..a41b5ee
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hive.metastore.utils.StringUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Constants and utility functions for column types. This is explicitly done as constants in the
+ * class rather than an enum in order to interoperate with Hive's old serdeConstants. All type
+ * names in this class match the type names in Hive's serdeConstants class. They must continue
+ * to do so.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class ColumnType {
+ public static final String VOID_TYPE_NAME = "void";
+
+ public static final String BOOLEAN_TYPE_NAME = "boolean";
+
+ public static final String TINYINT_TYPE_NAME = "tinyint";
+
+ public static final String SMALLINT_TYPE_NAME = "smallint";
+
+ public static final String INT_TYPE_NAME = "int";
+
+ public static final String BIGINT_TYPE_NAME = "bigint";
+
+ public static final String FLOAT_TYPE_NAME = "float";
+
+ public static final String DOUBLE_TYPE_NAME = "double";
+
+ public static final String STRING_TYPE_NAME = "string";
+
+ public static final String CHAR_TYPE_NAME = "char";
+
+ public static final String VARCHAR_TYPE_NAME = "varchar";
+
+ public static final String DATE_TYPE_NAME = "date";
+
+ public static final String DATETIME_TYPE_NAME = "datetime";
+
+ public static final String TIMESTAMP_TYPE_NAME = "timestamp";
+
+ public static final String DECIMAL_TYPE_NAME = "decimal";
+
+ public static final String BINARY_TYPE_NAME = "binary";
+
+ public static final String INTERVAL_YEAR_MONTH_TYPE_NAME = "interval_year_month";
+
+ public static final String INTERVAL_DAY_TIME_TYPE_NAME = "interval_day_time";
+
+ public static final String TIMESTAMPTZ_TYPE_NAME = "timestamp with time zone";
+
+ public static final String LIST_TYPE_NAME = "array";
+
+ public static final String MAP_TYPE_NAME = "map";
+
+ public static final String STRUCT_TYPE_NAME = "struct";
+
+ public static final String UNION_TYPE_NAME = "uniontype";
+
+ public static final String LIST_COLUMNS = "columns";
+
+ public static final String LIST_COLUMN_TYPES = "columns.types";
+
+ public static final String COLUMN_NAME_DELIMITER = "column.name.delimiter";
+
+ public static final String SERIALIZATION_FORMAT = "serialization.format";
+
+ public static final Set<String> PrimitiveTypes = StringUtils.asSet(
+ VOID_TYPE_NAME,
+ BOOLEAN_TYPE_NAME,
+ TINYINT_TYPE_NAME,
+ SMALLINT_TYPE_NAME,
+ INT_TYPE_NAME,
+ BIGINT_TYPE_NAME,
+ FLOAT_TYPE_NAME,
+ DOUBLE_TYPE_NAME,
+ STRING_TYPE_NAME,
+ VARCHAR_TYPE_NAME,
+ CHAR_TYPE_NAME,
+ DATE_TYPE_NAME,
+ DATETIME_TYPE_NAME,
+ TIMESTAMP_TYPE_NAME,
+ INTERVAL_YEAR_MONTH_TYPE_NAME,
+ INTERVAL_DAY_TIME_TYPE_NAME,
+ DECIMAL_TYPE_NAME,
+ BINARY_TYPE_NAME,
+ TIMESTAMPTZ_TYPE_NAME);
+
+ public static final Set<String> StringTypes = StringUtils.asSet(
+ STRING_TYPE_NAME,
+ VARCHAR_TYPE_NAME,
+ CHAR_TYPE_NAME
+ );
+
+ public static final Set<String> NumericTypes = StringUtils.asSet(
+ TINYINT_TYPE_NAME,
+ SMALLINT_TYPE_NAME,
+ INT_TYPE_NAME,
+ BIGINT_TYPE_NAME,
+ FLOAT_TYPE_NAME,
+ DOUBLE_TYPE_NAME,
+ DECIMAL_TYPE_NAME
+ );
+
+ // This intentionally does not include interval types.
+ public static final Set<String> DateTimeTypes = StringUtils.asSet(
+ DATE_TYPE_NAME,
+ DATETIME_TYPE_NAME,
+ TIMESTAMP_TYPE_NAME,
+ TIMESTAMPTZ_TYPE_NAME
+ );
+
+ // This map defines the progression of up casts in numeric types.
+ public static final Map<String, Integer> NumericCastOrder = new HashMap<>();
+
+ static {
+ NumericCastOrder.put(TINYINT_TYPE_NAME, 1);
+ NumericCastOrder.put(SMALLINT_TYPE_NAME, 2);
+ NumericCastOrder.put(INT_TYPE_NAME, 3);
+ NumericCastOrder.put(BIGINT_TYPE_NAME, 4);
+ NumericCastOrder.put(DECIMAL_TYPE_NAME, 5);
+ NumericCastOrder.put(FLOAT_TYPE_NAME, 6);
+ NumericCastOrder.put(DOUBLE_TYPE_NAME, 7);
+ }
+
+ private static final Map<String, String> alternateTypeNames = new HashMap<>();
+
+ static {
+ alternateTypeNames.put("integer", INT_TYPE_NAME);
+ alternateTypeNames.put("numeric", DECIMAL_TYPE_NAME);
+ }
+
+ public static final Set<String> CollectionTypes = StringUtils.asSet(
+ LIST_TYPE_NAME,
+ MAP_TYPE_NAME);
+
+ public static final Set<String> IntegralTypes = StringUtils.asSet(
+ TINYINT_TYPE_NAME,
+ SMALLINT_TYPE_NAME,
+ INT_TYPE_NAME,
+ BIGINT_TYPE_NAME);
+
+ public static final Set<String> AllTypes = StringUtils.asSet(
+ VOID_TYPE_NAME,
+ BOOLEAN_TYPE_NAME,
+ TINYINT_TYPE_NAME,
+ SMALLINT_TYPE_NAME,
+ INT_TYPE_NAME,
+ BIGINT_TYPE_NAME,
+ FLOAT_TYPE_NAME,
+ DOUBLE_TYPE_NAME,
+ STRING_TYPE_NAME,
+ CHAR_TYPE_NAME,
+ VARCHAR_TYPE_NAME,
+ DATE_TYPE_NAME,
+ DATETIME_TYPE_NAME,
+ TIMESTAMP_TYPE_NAME,
+ DECIMAL_TYPE_NAME,
+ BINARY_TYPE_NAME,
+ INTERVAL_YEAR_MONTH_TYPE_NAME,
+ INTERVAL_DAY_TIME_TYPE_NAME,
+ TIMESTAMPTZ_TYPE_NAME,
+ LIST_TYPE_NAME,
+ MAP_TYPE_NAME,
+ STRUCT_TYPE_NAME,
+ UNION_TYPE_NAME,
+ LIST_COLUMNS,
+ LIST_COLUMN_TYPES,
+ COLUMN_NAME_DELIMITER
+ );
+
+ /**
+ * Given a type string return the type name. For example, passing in the type string
+ * <tt>varchar(256)</tt> will return <tt>varchar</tt>.
+ * @param typeString Type string
+ * @return type name, guaranteed to be in lower case
+ */
+ public static String getTypeName(String typeString) {
+ if (typeString == null) return null;
+ String protoType = typeString.toLowerCase().split("\\W")[0];
+ String realType = alternateTypeNames.get(protoType);
+ return realType == null ? protoType : realType;
+ }
+
+ public static boolean areColTypesCompatible(String from, String to) {
+ if (from.equals(to)) return true;
+
+ if (PrimitiveTypes.contains(from) && PrimitiveTypes.contains(to)) {
+ // They aren't the same, but we may be able to do a cast
+
+ // If they are both types of strings, that should be fine
+ if (StringTypes.contains(from) && StringTypes.contains(to)) return true;
+
+ // If both are numeric, make sure the new type is larger than the old.
+ if (NumericTypes.contains(from) && NumericTypes.contains(to)) {
+ return NumericCastOrder.get(from) < NumericCastOrder.get(to);
+ }
+
+ // Allow string to double conversion
+ if (StringTypes.contains(from) && to.equals(DOUBLE_TYPE_NAME)) return true;
+
+ // Void can go to anything
+ if (from.equals(VOID_TYPE_NAME)) return true;
+
+ // Allow date to string casts. NOTE: I suspect this is the reverse of what we actually
+ // want, but it matches the code in o.a.h.h.serde2.typeinfo.TypeInfoUtils. I can't see how
+ // users would be altering date columns into string columns. The other I easily see since
+ // Hive did not originally support datetime types. Also, the comment in the Hive code
+ // says string to date, even though the code does the opposite. But for now I'm keeping
+ // this as is so the functionality matches.
+ if (DateTimeTypes.contains(from) && StringTypes.contains(to)) return true;
+
+ // Allow numeric to string
+ if (NumericTypes.contains(from) && StringTypes.contains(to)) return true;
+
+ }
+ return false;
+ }
+}