[GitHub] shaofengshi closed pull request #372: KYLIN-1111 Ignore unsupported hive column types when sync hive table

[GitBox <gi...@apache.org>]

shaofengshi closed pull request #372: KYLIN-1111 Ignore unsupported hive column types when sync hive table
URL: https://github.com/apache/kylin/pull/372
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
index 7580ad0202..d3f2804136 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/datatype/DataType.java
@@ -152,6 +152,17 @@ public static DataType getType(String type) {
         return cached;
     }
 
+    public static boolean isSupportedType(String typeName) {
+        if (typeName == null) {
+            return false;
+        }
+
+        String formattedTypeName = typeName.trim().toLowerCase(Locale.ROOT);
+        formattedTypeName = replaceLegacy(formattedTypeName);
+
+        return TYPE_PATTERN.matcher(formattedTypeName).matches() || COMPLEX_TYPE_PATTERN.matcher(formattedTypeName).matches();
+    }
+
     public static boolean isNumberFamily(String name) {
         return NUMBER_FAMILY.contains(name);
     }
@@ -237,7 +248,7 @@ public int compare(String value1, String value2) {
         return getOrder().compare(value1,  value2);
     }
 
-    private String replaceLegacy(String str) {
+    private static String replaceLegacy(String str) {
         String replace = LEGACY_TYPE_MAP.get(str);
         return replace == null ? str : replace;
     }
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/DataTypeTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/DataTypeTest.java
index 902920dacf..3b3ef6c31c 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/DataTypeTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/datatype/DataTypeTest.java
@@ -31,4 +31,11 @@ public void testComplexType() {
         Assert.assertTrue(DataType.isComplexType(arrayInt));
         Assert.assertTrue(DataType.isComplexType(arrayString));
     }
+
+    @Test
+    public void testIsSupportedType() {
+        Assert.assertTrue(DataType.isSupportedType("array<int>"));
+        Assert.assertFalse(DataType.isSupportedType("map<string, int>"));
+        Assert.assertFalse(DataType.isSupportedType(null));
+    }
 }
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
index 0407ac6e26..301b229e72 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMetadataExplorer.java
@@ -31,6 +31,7 @@
 import org.apache.kylin.common.util.Pair;
 import org.apache.kylin.common.util.RandomUtil;
 import org.apache.kylin.metadata.TableMetadataManager;
+import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.metadata.model.ColumnDesc;
 import org.apache.kylin.metadata.model.TableDesc;
 import org.apache.kylin.metadata.model.TableExtDesc;
@@ -238,21 +239,25 @@ public void validateSQL(String query) throws Exception {
 
         for (int i = 0; i < columnNumber; i++) {
             HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i);
-            ColumnDesc cdesc = new ColumnDesc();
-            cdesc.setName(field.name.toUpperCase(Locale.ROOT));
-
-            // use "double" in kylin for "float"
-            if ("float".equalsIgnoreCase(field.dataType)) {
-                cdesc.setDatatype("double");
-            } else {
-                cdesc.setDatatype(field.dataType);
-            }
 
-            cdesc.setId(String.valueOf(i + 1));
-            cdesc.setComment(field.comment);
-            columns.add(cdesc);
+            // skip unsupported fields, e.g. map<string, int>
+            if (DataType.isSupportedType(field.dataType)) {
+                ColumnDesc cdesc = new ColumnDesc();
+                cdesc.setName(field.name.toUpperCase(Locale.ROOT));
+
+                // use "double" in kylin for "float"
+                if ("float".equalsIgnoreCase(field.dataType)) {
+                    cdesc.setDatatype("double");
+                } else {
+                    cdesc.setDatatype(field.dataType);
+                }
+
+                cdesc.setId(String.valueOf(i + 1));
+                cdesc.setComment(field.comment);
+                columns.add(cdesc);
+            }
         }
 
-        return columns.toArray(new ColumnDesc[columnNumber]);
+        return  columns.toArray(new ColumnDesc[0]);
     }
 }


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services