You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2018/01/07 03:05:26 UTC
[18/50] [abbrv] carbondata git commit: [CARBONDATA-1935]fix the
backword compatibility issue for tableInfo deserialization
[CARBONDATA-1935]fix the backword compatibility issue for tableInfo deserialization
This closes #1720
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/03ddcc85
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/03ddcc85
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/03ddcc85
Branch: refs/heads/carbonstore
Commit: 03ddcc85db37cba37aedd3feb2f99f43d14b0c8a
Parents: be5134e
Author: akashrn5 <ak...@gmail.com>
Authored: Fri Dec 22 18:37:20 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Tue Jan 2 16:40:39 2018 +0800
----------------------------------------------------------------------
.../core/metadata/datatype/DataTypeAdapter.java | 51 ++++++++++++++++++++
.../core/metadata/datatype/DataTypes.java | 16 +++---
.../apache/carbondata/core/util/CarbonUtil.java | 11 ++++-
.../carbondata/core/util/DataTypeUtil.java | 40 +++++++++++++++
.../table/CarbonDescribeFormattedCommand.scala | 2 +-
5 files changed, 110 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeAdapter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeAdapter.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeAdapter.java
new file mode 100644
index 0000000..08fbcca
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeAdapter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.datatype;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+import com.google.gson.Gson;
+import com.google.gson.TypeAdapter;
+import com.google.gson.stream.JsonReader;
+import com.google.gson.stream.JsonToken;
+import com.google.gson.stream.JsonWriter;
+
+/**
+ * This class is added to support backward compatibility with table info object, where DATATYPE
+ * is string in old version(1.2) and OBJECT in new version(1.3)
+ */
+public class DataTypeAdapter extends TypeAdapter<Object> {
+
+ private Gson fallBack_original = new Gson();
+
+ @Override public void write(JsonWriter jsonWriter, Object o) throws IOException {
+ }
+
+ @Override public Object read(JsonReader jsonReader) throws IOException {
+ JsonToken token = jsonReader.peek();
+ if (token == JsonToken.STRING) {
+ return DataTypeUtil.valueOf(jsonReader.nextString());
+ } else {
+ // use original deserializer logic
+ return fallBack_original.fromJson(jsonReader, DataType.class);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
index e96917f..ad21eaa 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
@@ -58,14 +58,14 @@ public class DataTypes {
static final int LONG_TYPE_ID = 7;
static final int DOUBLE_TYPE_ID = 8;
static final int NULL_TYPE_ID = 9;
- static final int BYTE_TYPE_ID = 10;
- static final int BYTE_ARRAY_TYPE_ID = 11;
- static final int SHORT_INT_TYPE_ID = 12;
- static final int LEGACY_LONG_TYPE_ID = 13;
- static final int DECIMAL_TYPE_ID = 20;
- static final int ARRAY_TYPE_ID = 21;
- static final int STRUCT_TYPE_ID = 22;
- static final int MAP_TYPE_ID = 23;
+ static final int BYTE_TYPE_ID = 14;
+ static final int BYTE_ARRAY_TYPE_ID = 15;
+ static final int SHORT_INT_TYPE_ID = 16;
+ static final int LEGACY_LONG_TYPE_ID = 17;
+ static final int DECIMAL_TYPE_ID = 10;
+ static final int ARRAY_TYPE_ID = 11;
+ static final int STRUCT_TYPE_ID = 12;
+ static final int MAP_TYPE_ID = 13;
/**
* create a DataType instance from uniqueId of the DataType
http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index bd01772..ffe4654 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -65,6 +65,7 @@ import org.apache.carbondata.core.metadata.ValueEncoderMeta;
import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypeAdapter;
import org.apache.carbondata.core.metadata.datatype.DataTypes;
import org.apache.carbondata.core.metadata.encoder.Encoding;
import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema;
@@ -90,6 +91,8 @@ import org.apache.carbondata.format.DataChunk2;
import org.apache.carbondata.format.DataChunk3;
import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
@@ -1924,7 +1927,6 @@ public final class CarbonUtil {
// TODO: move this to carbon store API as it is related to TableInfo creation
public static TableInfo convertGsonToTableInfo(Map<String, String> properties) {
- Gson gson = new Gson();
String partsNo = properties.get("carbonSchemaPartsNo");
if (partsNo == null) {
return null;
@@ -1938,6 +1940,13 @@ public final class CarbonUtil {
}
builder.append(part);
}
+
+ // Datatype GSON adapter is added to support backward compatibility for tableInfo
+ // deserialization
+ GsonBuilder gsonBuilder = new GsonBuilder();
+ gsonBuilder.registerTypeAdapter(DataType.class, new DataTypeAdapter());
+
+ Gson gson = gsonBuilder.create();
TableInfo tableInfo = gson.fromJson(builder.toString(), TableInfo.class);
// The tableInfo is deserialized from GSON string, need to update the scale and
http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 0e819c4..65ea63e 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -735,4 +735,44 @@ public final class DataTypeUtil {
return converter;
}
+ /**
+ * @param name datatype string extracted from the json data
+ * @return returns the datatype based on the input string from json to deserialize the tableInfo
+ */
+ public static DataType valueOf(String name) {
+ if (DataTypes.STRING.getName().equalsIgnoreCase(name)) {
+ return DataTypes.STRING;
+ } else if (DataTypes.DATE.getName().equalsIgnoreCase(name)) {
+ return DataTypes.DATE;
+ } else if (DataTypes.TIMESTAMP.getName().equalsIgnoreCase(name)) {
+ return DataTypes.TIMESTAMP;
+ } else if (DataTypes.BOOLEAN.getName().equalsIgnoreCase(name)) {
+ return DataTypes.BOOLEAN;
+ } else if (DataTypes.BYTE.getName().equalsIgnoreCase(name)) {
+ return DataTypes.BYTE;
+ } else if (DataTypes.SHORT.getName().equalsIgnoreCase(name)) {
+ return DataTypes.SHORT;
+ } else if (DataTypes.SHORT_INT.getName().equalsIgnoreCase(name)) {
+ return DataTypes.SHORT_INT;
+ } else if (DataTypes.INT.getName().equalsIgnoreCase(name)) {
+ return DataTypes.INT;
+ } else if (DataTypes.LONG.getName().equalsIgnoreCase(name)) {
+ return DataTypes.LONG;
+ } else if (DataTypes.LEGACY_LONG.getName().equalsIgnoreCase(name)) {
+ return DataTypes.LEGACY_LONG;
+ } else if (DataTypes.FLOAT.getName().equalsIgnoreCase(name)) {
+ return DataTypes.FLOAT;
+ } else if (DataTypes.DOUBLE.getName().equalsIgnoreCase(name)) {
+ return DataTypes.DOUBLE;
+ } else if (DataTypes.NULL.getName().equalsIgnoreCase(name)) {
+ return DataTypes.NULL;
+ } else if (DataTypes.BYTE_ARRAY.getName().equalsIgnoreCase(name)) {
+ return DataTypes.BYTE_ARRAY;
+ } else if (DataTypes.BYTE_ARRAY.getName().equalsIgnoreCase(name)) {
+ return DataTypes.BYTE_ARRAY;
+ } else {
+ throw new RuntimeException("create DataType with invalid name: " + name);
+ }
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
index 5d19e0a..41656e5 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
@@ -109,7 +109,7 @@ private[sql] case class CarbonDescribeFormattedCommand(
results ++= Seq(("Streaming", isStreaming, ""))
val tblProps = carbonTable.getTableInfo.getFactTable.getTableProperties
- results ++= Seq(("SORT_SCOPE", tblProps.getOrDefault("sort_scope", CarbonCommonConstants
+ results ++= Seq(("SORT_SCOPE", tblProps.asScala.getOrElse("sort_scope", CarbonCommonConstants
.LOAD_SORT_SCOPE_DEFAULT), CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
// show table level compaction options