You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2018/12/06 09:16:10 UTC
[kylin] branch master updated: KYLIN-1111 Ignore unsupported hive
column types when sync hive table
This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new b8ad201 KYLIN-1111 Ignore unsupported hive column types when sync hive table
b8ad201 is described below
commit b8ad201e7797c909cb2069ec664e51452f85654e
Author: whuwb <sk...@gmail.com>
AuthorDate: Wed Dec 5 15:02:05 2018 +0800
KYLIN-1111 Ignore unsupported hive column types when sync hive table
---
.../apache/kylin/metadata/datatype/DataType.java | 13 +++++++++-
.../kylin/metadata/datatype/DataTypeTest.java | 7 ++++++
.../kylin/source/hive/HiveMetadataExplorer.java | 29 ++++++++++++++--------
3 files changed, 37 insertions(+), 12 deletions(-)
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index 7580ad0..e261e77 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -152,6 +152,17 @@ public class DataType implements Serializable {
return cached;
}
+ public static boolean isKylinSupported(String typeName) {
+ if (typeName == null) {
+ return false;
+ }
+
+ String formattedTypeName = typeName.trim().toLowerCase(Locale.ROOT);
+ formattedTypeName = replaceLegacy(formattedTypeName);
+
+ return TYPE_PATTERN.matcher(formattedTypeName).matches();
+ }
+
public static boolean isNumberFamily(String name) {
return NUMBER_FAMILY.contains(name);
}
@@ -237,7 +248,7 @@ public class DataType implements Serializable {
return getOrder().compare(value1, value2);
}
- private String replaceLegacy(String str) {
+ private static String replaceLegacy(String str) {
String replace = LEGACY_TYPE_MAP.get(str);
return replace == null ? str : replace;
}
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/DataTypeTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/DataTypeTest.java
index 902920d..071ad12 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/DataTypeTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/DataTypeTest.java
@@ -31,4 +31,11 @@ public class DataTypeTest {
Assert.assertTrue(DataType.isComplexType(arrayInt));
Assert.assertTrue(DataType.isComplexType(arrayString));
}
+
+ @Test
+ public void testIsSupportedType() {
+ Assert.assertFalse(DataType.isKylinSupported("array<int>"));
+ Assert.assertFalse(DataType.isKylinSupported("map<string, int>"));
+ Assert.assertFalse(DataType.isKylinSupported(null));
+ }
}
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
index 0407ac6..6893742 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
@@ -31,6 +31,7 @@ import org.apache.kylin.common.util.HiveCmdBuilder;
import org.apache.kylin.common.util.Pair;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.metadata.TableMetadataManager;
+import org.apache.kylin.metadata.datatype.DataType;
import org.apache.kylin.metadata.model.ColumnDesc;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.model.TableExtDesc;
@@ -238,21 +239,27 @@ public class HiveMetadataExplorer implements ISourceMetadataExplorer, ISampleDat
for (int i = 0; i < columnNumber; i++) {
HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i);
- ColumnDesc cdesc = new ColumnDesc();
- cdesc.setName(field.name.toUpperCase(Locale.ROOT));
- // use "double" in kylin for "float"
- if ("float".equalsIgnoreCase(field.dataType)) {
- cdesc.setDatatype("double");
+ // skip unsupported fields, e.g. map<string, int>
+ if (DataType.isKylinSupported(field.dataType)) {
+ ColumnDesc cdesc = new ColumnDesc();
+ cdesc.setName(field.name.toUpperCase(Locale.ROOT));
+
+ // use "double" in kylin for "float"
+ if ("float".equalsIgnoreCase(field.dataType)) {
+ cdesc.setDatatype("double");
+ } else {
+ cdesc.setDatatype(field.dataType);
+ }
+
+ cdesc.setId(String.valueOf(i + 1));
+ cdesc.setComment(field.comment);
+ columns.add(cdesc);
} else {
- cdesc.setDatatype(field.dataType);
+ logger.warn("Unsupported data type {}, excluding the field '{}'.", field.dataType, field.name);
}
-
- cdesc.setId(String.valueOf(i + 1));
- cdesc.setComment(field.comment);
- columns.add(cdesc);
}
- return columns.toArray(new ColumnDesc[columnNumber]);
+ return columns.toArray(new ColumnDesc[0]);
}
}