[carbondata] branch master updated: [CARBONDATA-3969] Fix Deserialization issue with DataType class

[aj...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

ajantha pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new 82ea971  [CARBONDATA-3969] Fix Deserialization issue with DataType class
82ea971 is described below

commit 82ea97114aaf0edafaef62963a46c88aab93df92
Author: Indhumathi27 <in...@gmail.com>
AuthorDate: Tue Sep 1 17:46:11 2020 +0530

    [CARBONDATA-3969] Fix Deserialization issue with DataType class
    
    Why is this PR needed?
    When DataType(For eg, StringType) object is serialized and deserialized using Gson, DataType object after deserialization, gets changed to Main class (DataType) instance instead of child type(StringType in this example).
    
    Because of this, presto query on table with complex columns fails with NPE, because while filing DimensionAndMeasureDetails(SegmentProperties.fillDimensionAndMeasureDetails) from CarbonLocalInputSplit.getDetailInfo, list of child dimensions will be left unfilled for Parent Carbon Dimension.
    
    What changes were proposed in this PR?
    Override JsonDeserializer method for DataType and deserialize the jsonObject based on its DataType child class instance.
    Add registerTypeAdapter for DataType class to gsonObject, while deserializing carbonLocalInputSplit.detailInfo.
    
    Does this PR introduce any user interface change?
    No
    
    Is any new testcase added?
    Yes
    
    This closes #3910
---
 .../metadata/datatype/DataTypeDeserializer.java    | 46 +++++++++++++++++++
 .../metadata/schema/table/ColumnSchemaTest.java    | 51 ++++++++++++++++++++++
 .../presto/impl/CarbonLocalInputSplit.java         |  8 +++-
 3 files changed, 104 insertions(+), 1 deletion(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeDeserializer.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeDeserializer.java
new file mode 100644
index 0000000..9ed49f5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeDeserializer.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.datatype;
+
+import java.lang.reflect.Type;
+
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+
+/**
+ * This class will deserialize DataType json string based on name
+ */
+public class DataTypeDeserializer implements JsonDeserializer<DataType> {
+  @Override
+  public DataType deserialize(JsonElement jsonElement, Type type,
+      JsonDeserializationContext context) throws JsonParseException {
+    JsonObject jsonObject = jsonElement.getAsJsonObject();
+    // deserialize dataType object based on dataType name
+    JsonElement jsonNameElement = jsonObject.get("name");
+    if (null != jsonNameElement) {
+      return context
+          .deserialize(jsonObject, DataTypeUtil.valueOf(jsonNameElement.getAsString()).getClass());
+    }
+    return context.deserialize(jsonObject, DataType.class);
+  }
+}
diff --git a/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/ColumnSchemaTest.java b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/ColumnSchemaTest.java
new file mode 100644
index 0000000..36b0f21
--- /dev/null
+++ b/core/src/test/java/org/apache/carbondata/core/metadata/schema/table/ColumnSchemaTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.table;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypeDeserializer;
+import org.apache.carbondata.core.metadata.datatype.DataTypes;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class ColumnSchemaTest {
+
+  @Test
+  public void testDataTypeOnColumnSchemaObject() {
+    ColumnSchema columnSchema = new ColumnSchema();
+    columnSchema.setDataType(DataTypes.createDefaultArrayType());
+
+    // without DataTypeDeserializer, after deserialization, data type is main class type instead
+    // of child class type
+    Gson gson = new Gson();
+    String serializedColumnSchema = gson.toJson(columnSchema);
+    ColumnSchema newColumnSchema = gson.fromJson(serializedColumnSchema, ColumnSchema.class);
+    Assert.assertFalse(newColumnSchema.getDataType().isComplexType());
+
+    // using DataTypeDeserializer
+    GsonBuilder gsonBuilder = new GsonBuilder();
+    gsonBuilder.registerTypeAdapter(DataType.class, new DataTypeDeserializer());
+    Gson newGson = gsonBuilder.create();
+    ColumnSchema newColumnSchemaObj = newGson.fromJson(serializedColumnSchema, ColumnSchema.class);
+    Assert.assertTrue(newColumnSchemaObj.getDataType().isComplexType());
+  }
+
+}
diff --git a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
index 815775c..1b934f4 100755
--- a/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
+++ b/integration/presto/src/main/java/org/apache/carbondata/presto/impl/CarbonLocalInputSplit.java
@@ -22,12 +22,15 @@ import java.util.List;
 
 import org.apache.carbondata.core.indexstore.BlockletDetailInfo;
 import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypeDeserializer;
 import org.apache.carbondata.core.statusmanager.FileFormat;
 import org.apache.carbondata.hadoop.CarbonInputSplit;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
 
 /**
  * CarbonLocalInputSplit represents a block, it contains a set of blocklet.
@@ -156,7 +159,10 @@ public class CarbonLocalInputSplit {
     if (FileFormat.COLUMNAR_V3.ordinal() == inputSplit.getFileFormat().ordinal()
         && null != carbonLocalInputSplit.detailInfo && !carbonLocalInputSplit.detailInfo
         .equalsIgnoreCase("null")) {
-      Gson gson = new Gson();
+      GsonBuilder gsonBuilder = new GsonBuilder();
+      // add typeAdapter for DataType Class for deserialization
+      gsonBuilder.registerTypeAdapter(DataType.class, new DataTypeDeserializer());
+      Gson gson = gsonBuilder.create();
       BlockletDetailInfo blockletDetailInfo =
           gson.fromJson(carbonLocalInputSplit.detailInfo, BlockletDetailInfo.class);
       if (null == blockletDetailInfo) {