You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/07/17 01:56:57 UTC
[11/15] carbondata git commit: [CARBONDATA-1307] Fix bug for
TableInfo serialization problem in cluster mode
[CARBONDATA-1307] Fix bug for TableInfo serialization problem in cluster mode
TableInfo serialization has problem when there are scala list in ColumnSchema, this PR fixed this issue
This closes #1172
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8d3c9bf1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8d3c9bf1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8d3c9bf1
Branch: refs/heads/datamap
Commit: 8d3c9bf1b09fa3c15ff5d4e707f5a0e657d1ff74
Parents: 5b49afc
Author: jackylk <ja...@huawei.com>
Authored: Fri Jul 14 16:25:12 2017 +0800
Committer: Ravindra Pesala <ra...@gmail.com>
Committed: Sat Jul 15 15:26:11 2017 +0530
----------------------------------------------------------------------
.../carbondata/hadoop/CarbonInputFormat.java | 15 +++++++++++++--
.../hadoop/util/ObjectSerializationUtil.java | 20 ++++++++++++--------
2 files changed, 25 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d3c9bf1/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 787f571..1673193 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -16,6 +16,8 @@
*/
package org.apache.carbondata.hadoop;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
@@ -114,7 +116,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
public static void setTableInfo(Configuration configuration, TableInfo tableInfo)
throws IOException {
if (null != tableInfo) {
- configuration.set(TABLE_INFO, ObjectSerializationUtil.convertObjectToString(tableInfo));
+ configuration.set(TABLE_INFO, ObjectSerializationUtil.encodeToString(tableInfo.serialize()));
}
}
@@ -123,7 +125,16 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
*/
private TableInfo getTableInfo(Configuration configuration) throws IOException {
String tableInfoStr = configuration.get(TABLE_INFO);
- return (TableInfo) ObjectSerializationUtil.convertStringToObject(tableInfoStr);
+ if (tableInfoStr == null) {
+ return null;
+ } else {
+ TableInfo output = new TableInfo();
+ output.readFields(
+ new DataInputStream(
+ new ByteArrayInputStream(
+ ObjectSerializationUtil.decodeStringToBytes(tableInfoStr))));
+ return output;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/8d3c9bf1/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
index fab729e..ea3246f 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
@@ -16,11 +16,7 @@
*/
package org.apache.carbondata.hadoop.util;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
+import java.io.*;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@@ -70,7 +66,11 @@ public class ObjectSerializationUtil {
}
}
- return new String(Base64.encodeBase64(baos.toByteArray()),
+ return encodeToString(baos.toByteArray());
+ }
+
+ public static String encodeToString(byte[] bytes) throws UnsupportedEncodingException {
+ return new String(Base64.encodeBase64(bytes),
CarbonCommonConstants.DEFAULT_CHARSET);
}
@@ -86,8 +86,7 @@ public class ObjectSerializationUtil {
return null;
}
- byte[] bytes =
- Base64.decodeBase64(objectString.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));
+ byte[] bytes = decodeStringToBytes(objectString);
ByteArrayInputStream bais = null;
GZIPInputStream gis = null;
@@ -116,4 +115,9 @@ public class ObjectSerializationUtil {
}
}
}
+
+ public static byte[] decodeStringToBytes(String objectString)
+ throws UnsupportedEncodingException {
+ return Base64.decodeBase64(objectString.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));
+ }
}