You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2017/09/01 07:16:31 UTC
[2/2] carbondata git commit: [CARBONDATA-1443] Schema storage has
issue while converting table object to json string
[CARBONDATA-1443] Schema storage has issue while converting table object to json string
Every table object converts to json and splits to small splits as there is limit in hive metastore of 4000. While splitting the json string code has problem so it could not get table and throws NPE
This closes #1314
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/baca6f92
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/baca6f92
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/baca6f92
Branch: refs/heads/branch-1.2
Commit: baca6f925ad973864ff432dbaccd6a99df5ea363
Parents: bb0b347
Author: Ravindra Pesala <ra...@gmail.com>
Authored: Fri Sep 1 10:51:04 2017 +0530
Committer: chenliang613 <ch...@apache.org>
Committed: Fri Sep 1 14:25:00 2017 +0800
----------------------------------------------------------------------
.../apache/carbondata/core/util/CarbonUtil.java | 51 +++++++++++++++++---
.../carbondata/core/util/CarbonUtilTest.java | 47 +++++++++++++++++-
2 files changed, 91 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/baca6f92/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 67c1d81..8b6e44a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -1803,10 +1803,31 @@ public final class CarbonUtil {
return !(null == badRecordsLocation || badRecordsLocation.length() == 0);
}
+ /**
+ * Converts Tableinfo object to json multi string objects of size 4000
+ * @param tableInfo
+ * @param seperator separator between each string
+ * @param quote Quote to be used for string
+ * @param prefix Prefix to be added before generated string
+ * @return
+ */
public static String convertToMultiGsonStrings(TableInfo tableInfo, String seperator,
String quote, String prefix) {
Gson gson = new Gson();
String schemaString = gson.toJson(tableInfo);
+ return splitSchemaStringToMultiString(seperator, quote, prefix, schemaString);
+ }
+
+ /**
+ * Converts Json String to multi string objects of size 4000
+ * @param schemaString Json string
+ * @param seperator separator between each string
+ * @param quote Quote to be used for string
+ * @param prefix Prefix to be added before generated string
+ * @return
+ */
+ public static String splitSchemaStringToMultiString(String seperator, String quote,
+ String prefix, String schemaString) {
int schemaLen = schemaString.length();
int splitLen = 4000;
int parts = schemaLen / splitLen;
@@ -1817,10 +1838,12 @@ public final class CarbonUtil {
new StringBuilder(prefix).append(quote).append("carbonSchemaPartsNo").append(quote)
.append(seperator).append("'").append(parts).append("',");
int runningLen = 0;
- int endLen = splitLen;
+ int endLen = schemaLen > splitLen ? splitLen : schemaLen;
for (int i = 0; i < parts; i++) {
if (i == parts - 1) {
- endLen = schemaLen % splitLen;
+ if (schemaLen % splitLen > 0) {
+ endLen = schemaLen % splitLen;
+ }
}
builder.append(quote).append("carbonSchema").append(i).append(quote).append(seperator);
builder.append("'").append(schemaString.substring(runningLen, runningLen + endLen))
@@ -1833,10 +1856,25 @@ public final class CarbonUtil {
return builder.toString();
}
+ /**
+ * Converts Tableinfo object to json multi string objects of size 4000 and stored in map
+ * @param tableInfo
+ * @return
+ */
public static Map<String, String> convertToMultiStringMap(TableInfo tableInfo) {
- Map<String, String> map = new HashMap<>();
Gson gson = new Gson();
String schemaString = gson.toJson(tableInfo);
+ return splitSchemaStringToMap(schemaString);
+ }
+
+ /**
+ * Converts Json string to multi string objects of size 4000 and stored in map
+ *
+ * @param schemaString
+ * @return
+ */
+ public static Map<String, String> splitSchemaStringToMap(String schemaString) {
+ Map<String, String> map = new HashMap<>();
int schemaLen = schemaString.length();
int splitLen = 4000;
int parts = schemaLen / splitLen;
@@ -1845,10 +1883,12 @@ public final class CarbonUtil {
}
map.put("carbonSchemaPartsNo", parts + "");
int runningLen = 0;
- int endLen = splitLen;
+ int endLen = schemaLen > splitLen ? splitLen : schemaLen;
for (int i = 0; i < parts; i++) {
if (i == parts - 1) {
- endLen = schemaLen % splitLen;
+ if (schemaLen % splitLen > 0) {
+ endLen = schemaLen % splitLen;
+ }
}
map.put("carbonSchema" + i, schemaString.substring(runningLen, runningLen + endLen));
runningLen += splitLen;
@@ -1856,7 +1896,6 @@ public final class CarbonUtil {
return map;
}
-
public static TableInfo convertGsonToTableInfo(Map<String, String> properties) {
Gson gson = new Gson();
String partsNo = properties.get("carbonSchemaPartsNo");
http://git-wip-us.apache.org/repos/asf/carbondata/blob/baca6f92/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
index 776059f..4a8483f 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonUtilTest.java
@@ -27,6 +27,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import org.apache.carbondata.core.datastore.block.TableBlockInfo;
import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
@@ -1002,6 +1003,50 @@ public class CarbonUtilTest {
}
+ @Test
+ public void testSplitSchemaStringToMapWithLessThanSplitLen() {
+ String schema = generateString(399);
+ Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
+ assert (map.size() == 2);
+ String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
+ assert (schemaString.length() > schema.length());
+ }
+
+ @Test
+ public void testSplitSchemaStringToMapWithEqualThanSplitLen() {
+ String schema = generateString(4000);
+ Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
+ assert (map.size() == 2);
+ String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
+ assert (schemaString.length() > schema.length());
+ }
+
+ @Test
+ public void testSplitSchemaStringToMapWithMoreThanSplitLen() {
+ String schema = generateString(7999);
+ Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
+ assert (map.size() == 3);
+ String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
+ assert (schemaString.length() > schema.length());
+ }
+
+ @Test
+ public void testSplitSchemaStringToMapWithMultiplesOfSplitLen() {
+ String schema = generateString(12000);
+ Map<String, String> map = CarbonUtil.splitSchemaStringToMap(schema);
+ assert (map.size() == 4);
+ String schemaString = CarbonUtil.splitSchemaStringToMultiString(" ", "'", ",", schema);
+ assert (schemaString.length() > schema.length());
+ }
+
+ private String generateString(int length) {
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < length; i++) {
+ builder.append("a");
+ }
+ return builder.toString();
+ }
+
private void assertRangeIndex(byte[][] dataArr, byte[] dataChunk,
FixedLengthDimensionDataChunk fixedLengthDimensionDataChunk, byte[] keyWord, int[] expectRangeIndex) {
int[] range;
@@ -1013,7 +1058,7 @@ public class CarbonUtilTest {
int index = CarbonUtil.binarySearch(dataArr, 0, dataChunk.length / keyWord.length - 1, keyWord);
assertTrue(expectRangeIndex[0] <= index && index <= range[1]);
}
-
+
@AfterClass public static void testcleanUp() {
new File("../core/src/test/resources/testFile.txt").deleteOnExit();