You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/07/25 17:37:46 UTC
[09/15] carbondata git commit: [CARBONDATA-1307] Fix bug for
TableInfo serialization problem in cluster mode
[CARBONDATA-1307] Fix bug for TableInfo serialization problem in cluster mode
TableInfo serialization has problem when there are scala list in ColumnSchema, this PR fixed this issue
This closes #1172
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a3a12f0c
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a3a12f0c
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a3a12f0c
Branch: refs/heads/master
Commit: a3a12f0ceb9ab2a7bd3690961035fd7c186c2216
Parents: 7d1cde5
Author: jackylk <ja...@huawei.com>
Authored: Fri Jul 14 16:25:12 2017 +0800
Committer: Raghunandan S <ca...@gmail.com>
Committed: Tue Jul 25 17:35:42 2017 +0800
----------------------------------------------------------------------
.../carbondata/hadoop/CarbonInputFormat.java | 15 +++++++++++++--
.../hadoop/util/ObjectSerializationUtil.java | 20 ++++++++++++--------
2 files changed, 25 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a3a12f0c/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 6d14424..b81fdf4 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -16,6 +16,8 @@
*/
package org.apache.carbondata.hadoop;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
@@ -112,7 +114,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
public static void setTableInfo(Configuration configuration, TableInfo tableInfo)
throws IOException {
if (null != tableInfo) {
- configuration.set(TABLE_INFO, ObjectSerializationUtil.convertObjectToString(tableInfo));
+ configuration.set(TABLE_INFO, ObjectSerializationUtil.encodeToString(tableInfo.serialize()));
}
}
@@ -121,7 +123,16 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
*/
private TableInfo getTableInfo(Configuration configuration) throws IOException {
String tableInfoStr = configuration.get(TABLE_INFO);
- return (TableInfo) ObjectSerializationUtil.convertStringToObject(tableInfoStr);
+ if (tableInfoStr == null) {
+ return null;
+ } else {
+ TableInfo output = new TableInfo();
+ output.readFields(
+ new DataInputStream(
+ new ByteArrayInputStream(
+ ObjectSerializationUtil.decodeStringToBytes(tableInfoStr))));
+ return output;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a3a12f0c/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
index fab729e..ea3246f 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/ObjectSerializationUtil.java
@@ -16,11 +16,7 @@
*/
package org.apache.carbondata.hadoop.util;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
+import java.io.*;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
@@ -70,7 +66,11 @@ public class ObjectSerializationUtil {
}
}
- return new String(Base64.encodeBase64(baos.toByteArray()),
+ return encodeToString(baos.toByteArray());
+ }
+
+ public static String encodeToString(byte[] bytes) throws UnsupportedEncodingException {
+ return new String(Base64.encodeBase64(bytes),
CarbonCommonConstants.DEFAULT_CHARSET);
}
@@ -86,8 +86,7 @@ public class ObjectSerializationUtil {
return null;
}
- byte[] bytes =
- Base64.decodeBase64(objectString.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));
+ byte[] bytes = decodeStringToBytes(objectString);
ByteArrayInputStream bais = null;
GZIPInputStream gis = null;
@@ -116,4 +115,9 @@ public class ObjectSerializationUtil {
}
}
}
+
+ public static byte[] decodeStringToBytes(String objectString)
+ throws UnsupportedEncodingException {
+ return Base64.decodeBase64(objectString.getBytes(CarbonCommonConstants.DEFAULT_CHARSET));
+ }
}