You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/07/25 17:37:51 UTC
[14/15] carbondata git commit: [CARBONDATA-1326] Fixed high priority
findbug issues
[CARBONDATA-1326] Fixed high priority findbug issues
Fixed high priority findbug issues in the code
This closes #1191
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2b66476d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2b66476d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2b66476d
Branch: refs/heads/master
Commit: 2b66476dd643bad15aa4713b84bc84afe2233c35
Parents: aadbd5c
Author: manishgupta88 <to...@gmail.com>
Authored: Fri Jul 21 21:48:20 2017 +0800
Committer: Raghunandan S <ca...@gmail.com>
Committed: Tue Jul 25 17:38:57 2017 +0800
----------------------------------------------------------------------
.../cache/dictionary/ColumnDictionaryInfo.java | 10 +++++---
.../dictionary/DoubleArrayTrieDictionary.java | 7 ++++--
.../core/constants/CarbonCommonConstants.java | 13 +++++-----
.../core/datastore/block/BlockInfo.java | 4 ++-
.../block/SegmentTaskIndexWrapper.java | 17 -------------
.../core/datastore/block/TableBlockInfo.java | 7 ++++--
.../core/datastore/page/ColumnPage.java | 2 +-
.../generator/key/DictionaryMessage.java | 16 +++++++-----
.../core/indexstore/DataMapStoreManager.java | 2 +-
.../datatype/DecimalConverterFactory.java | 2 +-
.../core/mutate/DeleteDeltaBlockletDetails.java | 6 +++--
.../scan/filter/FilterExpressionProcessor.java | 26 +++++++++++++++++---
.../scan/filter/intf/FilterOptimizerImpl.java | 2 +-
.../RowLevelRangeFilterResolverImpl.java | 2 +-
.../core/scan/result/AbstractScannedResult.java | 7 ++++--
.../statusmanager/SegmentStatusManager.java | 5 +++-
.../SegmentUpdateStatusManager.java | 3 ---
.../CarbonDictionarySortInfoPreparator.java | 4 +--
.../carbondata/hadoop/CarbonInputSplit.java | 7 ++++--
.../internal/index/impl/InMemoryBTreeIndex.java | 2 +-
.../carbondata/spark/util/CarbonScalaUtil.scala | 4 ++-
.../org/apache/spark/util/TableAPIUtil.scala | 2 +-
.../processing/csvload/CSVInputFormat.java | 5 +++-
.../newflow/AbstractDataLoadProcessorStep.java | 26 ++++++++++----------
.../CarbonRowDataWriterProcessorStepImpl.java | 1 +
.../steps/DataConverterProcessorStepImpl.java | 1 +
...ConverterProcessorWithBucketingStepImpl.java | 1 +
.../steps/DataWriterBatchProcessorStepImpl.java | 1 +
.../steps/DataWriterProcessorStepImpl.java | 1 +
.../newflow/steps/InputProcessorStepImpl.java | 5 ++--
.../newflow/steps/SortProcessorStepImpl.java | 1 +
.../writer/v2/CarbonFactDataWriterImplV2.java | 4 ++-
32 files changed, 117 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
index d3599da..260ba90 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
@@ -194,8 +194,9 @@ public class ColumnDictionaryInfo extends AbstractColumnDictionaryInfo {
byte[] dictionaryValue = getDictionaryBytesFromSurrogate(surrogateKey);
int cmp = -1;
if (this.getDataType() != DataType.STRING) {
- cmp = compareFilterKeyWithDictionaryKey(new String(dictionaryValue), filterKey,
- this.getDataType());
+ cmp = compareFilterKeyWithDictionaryKey(
+ new String(dictionaryValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)),
+ filterKey, this.getDataType());
} else {
cmp = ByteUtil.UnsafeComparer.INSTANCE.compareTo(dictionaryValue, key);
@@ -240,8 +241,9 @@ public class ColumnDictionaryInfo extends AbstractColumnDictionaryInfo {
if (null == dictionaryValue) {
cmp = -1;
} else if (this.getDataType() != DataType.STRING) {
- cmp = compareFilterKeyWithDictionaryKey(new String(dictionaryValue), filterKey,
- this.getDataType());
+ cmp = compareFilterKeyWithDictionaryKey(
+ new String(dictionaryValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)),
+ filterKey, this.getDataType());
} else {
cmp =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
index f738008..cab1926 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
@@ -21,9 +21,12 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.PrintStream;
+import java.nio.charset.Charset;
import java.util.SortedSet;
import java.util.TreeSet;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
/**
* A dictionary based on DoubleArrayTrie data structure that maps enumerations
* of byte[] to int IDs. With DoubleArrayTrie the memory footprint of the mapping
@@ -116,7 +119,7 @@ public class DoubleArrayTrieDictionary {
*/
public int getValue(String key) {
String k = key + '\0';
- byte[] bKeys = k.getBytes();
+ byte[] bKeys = k.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
return getValue(bKeys);
}
@@ -315,7 +318,7 @@ public class DoubleArrayTrieDictionary {
*/
public boolean insert(String key) {
String k = key + '\0';
- byte[] bKeys = k.getBytes();
+ byte[] bKeys = k.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
if (!insert(bKeys)) {
return false;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 8c61305..8427a22 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -176,9 +176,15 @@ public final class CarbonCommonConstants {
public static final String MEMBER_DEFAULT_VAL = "@NU#LL$!";
/**
+ * default charset to be used for reading and writing
+ */
+ public static final String DEFAULT_CHARSET = "UTF-8";
+
+ /**
* MEMBER_DEFAULT_VAL_ARRAY
*/
- public static final byte[] MEMBER_DEFAULT_VAL_ARRAY = MEMBER_DEFAULT_VAL.getBytes();
+ public static final byte[] MEMBER_DEFAULT_VAL_ARRAY =
+ MEMBER_DEFAULT_VAL.getBytes(Charset.forName(DEFAULT_CHARSET));
/**
* Bytes for string 0, it is used in codegen in case of null values.
@@ -772,11 +778,6 @@ public final class CarbonCommonConstants {
public static final String ENABLE_XXHASH_DEFAULT = "true";
/**
- * default charset to be used for reading and writing
- */
- public static final String DEFAULT_CHARSET = "UTF-8";
-
- /**
* default charset class to be used for reading and writing
*/
public static final Charset DEFAULT_CHARSET_CLASS = Charset.forName(DEFAULT_CHARSET);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java
index 2833316..d2eae8b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java
@@ -16,6 +16,8 @@
*/
package org.apache.carbondata.core.datastore.block;
+import java.io.Serializable;
+
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.util.path.CarbonTablePath;
@@ -26,7 +28,7 @@ import org.apache.carbondata.core.util.path.CarbonTablePath;
* block will be loaded together. This class will be used to store table block info
* and equals and hash code method is used to identify blocklet belongs to same block
*/
-public class BlockInfo {
+public class BlockInfo implements Serializable {
/**
* table block info, stores all the details
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
index 516cd6a..4fe6d1b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
@@ -23,7 +23,6 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.carbondata.core.cache.Cacheable;
import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
-import org.apache.carbondata.core.mutate.UpdateVO;
/**
* SegmentTaskIndexWrapper class holds the taskIdToTableSegmentMap
@@ -45,7 +44,6 @@ public class SegmentTaskIndexWrapper implements Cacheable {
protected AtomicLong memorySize = new AtomicLong();
private Long refreshedTimeStamp;
- private UpdateVO invalidTaskKey;
public SegmentTaskIndexWrapper(
Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> taskIdToTableSegmentMap) {
this.taskIdToTableSegmentMap = taskIdToTableSegmentMap;
@@ -128,19 +126,4 @@ public class SegmentTaskIndexWrapper implements Cacheable {
this.refreshedTimeStamp = refreshedTimeStamp;
}
- public void removeEntryFromCacheAndRefresh(String taskId) {
- AbstractIndex blockEntry = this.getTaskIdToTableSegmentMap().remove(taskId);
- if (null != blockEntry) {
- memorySize.set(memorySize.get() - blockEntry.getMemorySize());
- }
- }
-
- public void setLastUpdateVO(UpdateVO invalidTaskKey) {
- this.invalidTaskKey = invalidTaskKey;
- }
-
- public UpdateVO getInvalidTaskKey() {
- return invalidTaskKey;
- }
-
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index 44347cf..1da6699 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -17,6 +17,7 @@
package org.apache.carbondata.core.datastore.block;
import java.io.Serializable;
+import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
@@ -221,8 +222,10 @@ public class TableBlockInfo implements Distributable, Serializable {
// offset of
// the file
if (CarbonTablePath.isCarbonDataFile(filePath)) {
- int compare = ByteUtil.compare(DataFileUtil.getTaskNo(filePath).getBytes(),
- DataFileUtil.getTaskNo(((TableBlockInfo) other).filePath).getBytes());
+ int compare = ByteUtil.compare(DataFileUtil.getTaskNo(filePath)
+ .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)),
+ DataFileUtil.getTaskNo(((TableBlockInfo) other).filePath)
+ .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
if (compare != 0) {
return compare;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index 2c43165..ef2c9ec 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -109,7 +109,7 @@ public abstract class ColumnPage {
}
private static ColumnPage createPage(DataType dataType, int pageSize, int scale, int precision) {
- if (dataType.equals(BYTE_ARRAY) | dataType.equals(DECIMAL)) {
+ if (dataType.equals(BYTE_ARRAY) || dataType.equals(DECIMAL)) {
return createVarLengthPage(dataType, pageSize, scale, precision);
} else {
return createFixLengthPage(dataType, pageSize, scale, precision);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
index 2ae0521..0bb82f9 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
@@ -16,6 +16,8 @@
*/
package org.apache.carbondata.core.dictionary.generator.key;
+import java.nio.charset.Charset;
+
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import io.netty.buffer.ByteBuf;
@@ -53,11 +55,12 @@ public class DictionaryMessage {
public void readData(ByteBuf byteBuf) {
byte[] tableBytes = new byte[byteBuf.readInt()];
byteBuf.readBytes(tableBytes);
- tableUniqueName = new String(tableBytes);
+ tableUniqueName =
+ new String(tableBytes, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
byte[] colBytes = new byte[byteBuf.readInt()];
byteBuf.readBytes(colBytes);
- columnName = new String(colBytes);
+ columnName = new String(colBytes, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
byte typeByte = byteBuf.readByte();
type = getKeyType(typeByte);
@@ -68,7 +71,7 @@ public class DictionaryMessage {
} else {
byte[] dataBytes = new byte[byteBuf.readInt()];
byteBuf.readBytes(dataBytes);
- data = new String(dataBytes);
+ data = new String(dataBytes, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
}
}
@@ -77,11 +80,12 @@ public class DictionaryMessage {
// Just reserve the bytes to add length of header at last.
byteBuf.writeShort(Short.MAX_VALUE);
- byte[] tableBytes = tableUniqueName.getBytes();
+ byte[] tableBytes =
+ tableUniqueName.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
byteBuf.writeInt(tableBytes.length);
byteBuf.writeBytes(tableBytes);
- byte[] colBytes = columnName.getBytes();
+ byte[] colBytes = columnName.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
byteBuf.writeInt(colBytes.length);
byteBuf.writeBytes(colBytes);
@@ -92,7 +96,7 @@ public class DictionaryMessage {
byteBuf.writeInt(dictionaryValue);
} else {
byteBuf.writeByte(1);
- byte[] dataBytes = data.getBytes();
+ byte[] dataBytes = data.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
byteBuf.writeInt(dataBytes.length);
byteBuf.writeBytes(dataBytes);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
index 06638ad..64c6e20 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
@@ -86,7 +86,7 @@ public class DataMapStoreManager {
} catch (Exception e) {
LOGGER.error(e);
}
- dataMap.init(identifier, dataMapName);
+ // TODO: Initialize a data map by calling init method on the data map
map.put(dataMapName, dataMap);
return dataMap;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
index 555df1c..459eb24 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
@@ -29,7 +29,7 @@ import org.apache.carbondata.core.util.DataTypeUtil;
*/
public final class DecimalConverterFactory {
- public static DecimalConverterFactory INSTANCE = new DecimalConverterFactory();
+ public static final DecimalConverterFactory INSTANCE = new DecimalConverterFactory();
private int[] minBytesForPrecision = minBytesForPrecision();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
index 0f54f3a..22785fa 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
@@ -69,9 +69,11 @@ public class DeleteDeltaBlockletDetails implements Serializable {
if (obj == null || !(obj instanceof DeleteDeltaBlockletDetails)) {
return false;
}
-
DeleteDeltaBlockletDetails that = (DeleteDeltaBlockletDetails) obj;
- return id.equals(that.id) && pageId == that.pageId;
+ if (null == pageId || null == that.pageId) {
+ return false;
+ }
+ return id.equals(that.id) && (pageId.intValue() == that.pageId.intValue());
}
@Override public int hashCode() {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index 21c7bf6..02052aa 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -131,11 +131,14 @@ public class FilterExpressionProcessor implements FilterProcessor {
}
}
if (LOGGER.isDebugEnabled()) {
+ char delimiter = ',';
LOGGER.debug(
- "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey
- .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey
- .getNoDictionaryKeys() + "Dictionary End Key: " + searchEndKey.getDictionaryKeys()
- + "No Dictionary End Key " + searchEndKey.getNoDictionaryKeys());
+ "Successfully retrieved the start and end key" + "Dictionary Start Key: " + joinByteArray(
+ searchStartKey.getDictionaryKeys(), delimiter) + "No Dictionary Start Key "
+ + joinByteArray(searchStartKey.getNoDictionaryKeys(), delimiter)
+ + "Dictionary End Key: " + joinByteArray(searchEndKey.getDictionaryKeys(), delimiter)
+ + "No Dictionary End Key " + joinByteArray(searchEndKey.getNoDictionaryKeys(),
+ delimiter));
}
long startTimeInMillis = System.currentTimeMillis();
DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
@@ -159,6 +162,21 @@ public class FilterExpressionProcessor implements FilterProcessor {
return listOfDataBlocksToScan;
}
+ private String joinByteArray(byte[] bytes, char delimiter) {
+ String byteArrayAsString = "";
+ if (null != bytes) {
+ for (int i = 0; i < bytes.length; i++) {
+ byteArrayAsString = byteArrayAsString + delimiter + bytes[i];
+ }
+ if (byteArrayAsString.length() > 0) {
+ byteArrayAsString = byteArrayAsString.substring(1);
+ }
+ } else {
+ byteArrayAsString = null;
+ }
+ return byteArrayAsString;
+ }
+
/**
* Get the map of required partitions
* The value of "1" in BitSet represent the required partition
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
index 145fc09..cb9495b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
@@ -30,6 +30,6 @@ public class FilterOptimizerImpl implements FilterOptimizer {
}
@Override public Expression optimizeFilter() {
- return this.optimizeFilter();
+ return filterExpression;
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 6d4e528..47ca7e1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -145,7 +145,7 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
for (ExpressionResult result : listOfExpressionResults) {
try {
if (result.getString() == null) {
- filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes());
+ filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY);
continue;
}
filterValuesList.add(DataTypeUtil
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
index e78383d..21cf6c8 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
@@ -21,6 +21,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
import java.util.Map;
import org.apache.carbondata.common.logging.LogService;
@@ -304,7 +305,8 @@ public abstract class AbstractScannedResult {
j :
rowMapping[pageCounter][j]);
}
- vector.putBytes(vectorOffset++, data.getBytes());
+ vector.putBytes(vectorOffset++,
+ data.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
}
}
}
@@ -394,7 +396,8 @@ public abstract class AbstractScannedResult {
int position = 0;
for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
noDictionaryColumnsKeys[position++] = new String(
- dimensionDataChunks[noDictionaryColumnBlockIndexes[i]][pageCounter].getChunkData(rowId));
+ dimensionDataChunks[noDictionaryColumnBlockIndexes[i]][pageCounter].getChunkData(rowId),
+ Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
}
return noDictionaryColumnsKeys;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
index beef0c6..28d3f18 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
@@ -34,6 +34,7 @@ import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.row.LoadStatusType;
import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
import org.apache.carbondata.core.fileoperations.FileWriteOperation;
@@ -526,7 +527,9 @@ public class SegmentStatusManager {
+ "as the segment has been compacted.");
continue;
}
- if (!CarbonCommonConstants.MARKED_FOR_DELETE.equals(loadMetadata.getLoadStatus())) {
+ if (!CarbonCommonConstants.MARKED_FOR_DELETE.equals(loadMetadata.getLoadStatus())
+ && !LoadStatusType.IN_PROGRESS.getMessage().equals(loadMetadata.getLoadStatus())
+ && !LoadStatusType.INSERT_OVERWRITE.getMessage().equals(loadMetadata.getLoadStatus())) {
loadFound = true;
updateSegmentMetadataDetails(loadMetadata);
LOG.info("Info: " +
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index 3e0a2cb..9abc58f 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -788,9 +788,6 @@ public class SegmentUpdateStatusManager {
CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.BLOCK_ID)));
String segmentId =
CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.SEGMENT_ID);
- CarbonTablePath carbonTablePath = CarbonStorePath
- .getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
- absoluteTableIdentifier.getCarbonTableIdentifier());
SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray =
readLoadMetadata();
for (SegmentUpdateDetails block : listOfSegmentUpdateDetailsArray) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
index 61ae6c2..8c7d922 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
@@ -123,8 +123,8 @@ public class CarbonDictionarySortInfoPreparator {
// for new distinct values
Iterator<String> distinctValue = distinctValues.iterator();
while (distinctValue.hasNext()) {
- dictionarySortModels[surrogate - 1] =
- createDictionarySortModel(surrogate, dataType, distinctValue.next().getBytes());
+ dictionarySortModels[surrogate - 1] = createDictionarySortModel(surrogate, dataType,
+ distinctValue.next().getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
surrogate++;
}
return dictionarySortModels;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index 631bc2c..1cdbb26 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -20,6 +20,7 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.io.Serializable;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -261,8 +262,10 @@ public class CarbonInputSplit extends FileSplit
String filePath1 = this.getPath().getName();
String filePath2 = other.getPath().getName();
if (CarbonTablePath.isCarbonDataFile(filePath1)) {
- byte[] firstTaskId = CarbonTablePath.DataFileUtil.getTaskNo(filePath1).getBytes();
- byte[] otherTaskId = CarbonTablePath.DataFileUtil.getTaskNo(filePath2).getBytes();
+ byte[] firstTaskId = CarbonTablePath.DataFileUtil.getTaskNo(filePath1)
+ .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+ byte[] otherTaskId = CarbonTablePath.DataFileUtil.getTaskNo(filePath2)
+ .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
int compare = ByteUtil.compare(firstTaskId, otherTaskId);
if (compare != 0) {
return compare;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
index ed98871..4268ee2 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -79,7 +79,7 @@ class InMemoryBTreeIndex implements Index {
FilterExpressionProcessor filterExpressionProcessor = new FilterExpressionProcessor();
- AbsoluteTableIdentifier identifier = null;
+ AbsoluteTableIdentifier identifier = AbsoluteTableIdentifier.from(segment.getPath(), "", "");
//for this segment fetch blocks matching filter in BTree
List<DataRefNode> dataRefNodes =
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index fb01b27..65f2ba2 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -17,6 +17,7 @@
package org.apache.carbondata.spark.util
+import java.nio.charset.Charset
import java.text.SimpleDateFormat
import org.apache.spark.sql._
@@ -109,7 +110,8 @@ object CarbonScalaUtil {
case b: java.lang.Boolean => b.toString
case s: java.lang.Short => s.toString
case f: java.lang.Float => f.toString
- case bs: Array[Byte] => new String(bs)
+ case bs: Array[Byte] => new String(bs,
+ Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET))
case s: scala.collection.Seq[Any] =>
val delimiter = if (level == 1) {
delimiterLevel1
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
index a57ab10..bc62902 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
@@ -33,7 +33,7 @@ object TableAPIUtil {
def parseSchemaName(tableName: String): (String, String) = {
if (tableName.contains(".")) {
- val parts = tableName.split(".")
+ val parts = tableName.split("\\.")
(parts(0), parts(1))
} else {
("default", tableName)
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java
index 3a6428d..c793126 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java
@@ -20,9 +20,11 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
+import java.nio.charset.Charset;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import com.univocity.parsers.csv.CsvParser;
import com.univocity.parsers.csv.CsvParserSettings;
@@ -225,7 +227,8 @@ public class CSVInputFormat extends FileInputFormat<NullWritable, StringArrayWri
filePosition = fileIn;
inputStream = boundedInputStream;
}
- reader = new InputStreamReader(inputStream);
+ reader = new InputStreamReader(inputStream,
+ Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
csvParser = new CsvParser(extractCsvParserSettings(job));
csvParser.beginParsing(reader);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java
index a5887e6..9e0aa02 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java
@@ -56,7 +56,20 @@ public abstract class AbstractDataLoadProcessorStep {
this.child = child;
this.rowCounter = new AtomicLong();
this.closed = false;
+ }
+
+ /**
+ * The output meta for this step. The data returns from this step is as per this meta.
+ *
+ */
+ public abstract DataField[] getOutput();
+ /**
+ * Initialization process for this step.
+ *
+ * @throws IOException
+ */
+ public void initialize() throws IOException {
if (LOGGER.isInfoEnabled()) {
// This thread prints the rows processed in each step for every 10 seconds.
new Thread() {
@@ -76,19 +89,6 @@ public abstract class AbstractDataLoadProcessorStep {
}
/**
- * The output meta for this step. The data returns from this step is as per this meta.
- *
- */
- public abstract DataField[] getOutput();
-
- /**
- * Initialization process for this step.
- *
- * @throws IOException
- */
- public abstract void initialize() throws IOException;
-
- /**
* Tranform the data as per the implementation.
*
* @return Array of Iterator with data. It can be processed parallel if implementation class wants
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
index 71e5727..4ed588d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -84,6 +84,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
}
@Override public void initialize() throws IOException {
+ super.initialize();
child.initialize();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
index 62d6c94..c46ea25 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
@@ -61,6 +61,7 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
@Override
public void initialize() throws IOException {
+ super.initialize();
child.initialize();
converters = new ArrayList<>();
badRecordLogger = createBadRecordLogger(configuration);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
index c6f83ed..fe36f82 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
@@ -68,6 +68,7 @@ public class DataConverterProcessorWithBucketingStepImpl extends AbstractDataLoa
@Override
public void initialize() throws IOException {
+ super.initialize();
child.initialize();
converters = new ArrayList<>();
badRecordLogger = createBadRecordLogger();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java
index 46c1020..00cb302 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java
@@ -55,6 +55,7 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS
}
@Override public void initialize() throws IOException {
+ super.initialize();
child.initialize();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
index 087b0c7..afb3062 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
@@ -62,6 +62,7 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep {
}
@Override public void initialize() throws IOException {
+ super.initialize();
child.initialize();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java
index 4a78e8f..cbeb20a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java
@@ -16,6 +16,7 @@
*/
package org.apache.carbondata.processing.newflow.steps;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -32,7 +33,6 @@ import org.apache.carbondata.core.util.CarbonProperties;
import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
import org.apache.carbondata.processing.newflow.DataField;
-import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
import org.apache.carbondata.processing.newflow.parser.RowParser;
import org.apache.carbondata.processing.newflow.parser.impl.RowParserImpl;
import org.apache.carbondata.processing.newflow.row.CarbonRowBatch;
@@ -61,7 +61,8 @@ public class InputProcessorStepImpl extends AbstractDataLoadProcessorStep {
return configuration.getDataFields();
}
- @Override public void initialize() throws CarbonDataLoadingException {
+ @Override public void initialize() throws IOException {
+ super.initialize();
rowParser = new RowParserImpl(getOutput(), configuration);
executorService = Executors.newCachedThreadPool();
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
index 987cb15..0b93b7a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
@@ -49,6 +49,7 @@ public class SortProcessorStepImpl extends AbstractDataLoadProcessorStep {
@Override
public void initialize() throws IOException {
+ super.initialize();
child.initialize();
SortParameters sortParameters = SortParameters.createSortParameters(configuration);
sorter = SorterFactory.createSorter(configuration, rowCounter);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/2b66476d/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
index 82e83d5..c835332 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.processing.store.writer.v2;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
@@ -107,7 +108,8 @@ public class CarbonFactDataWriterImplV2 extends CarbonFactDataWriterImplV1 {
try {
if (fileChannel.size() == 0) {
ColumnarFormatVersion version = CarbonProperties.getInstance().getFormatVersion();
- byte[] header = (CarbonCommonConstants.CARBON_DATA_VERSION_HEADER + version).getBytes();
+ byte[] header = (CarbonCommonConstants.CARBON_DATA_VERSION_HEADER + version).getBytes(
+ Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
ByteBuffer buffer = ByteBuffer.allocate(header.length);
buffer.put(header);
buffer.rewind();