You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ja...@apache.org on 2017/08/19 01:53:11 UTC
[3/3] carbondata git commit: [Review][CARBONDATA-1386] fixed findbugs
errors in carbondata-core
[Review][CARBONDATA-1386] fixed findbugs errors in carbondata-core
fixed findbugs errors in carbondata-core
This closes #1263
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/500654e6
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/500654e6
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/500654e6
Branch: refs/heads/master
Commit: 500654e60d4fb7b1ab5eda39f29a415933838006
Parents: 2ee7775
Author: sraghunandan <ca...@gmail.com>
Authored: Thu Aug 17 19:14:48 2017 +0530
Committer: Jacky Li <ja...@qq.com>
Committed: Sat Aug 19 09:52:52 2017 +0800
----------------------------------------------------------------------
.../impl/ExtendedRollingFileAppender.java | 4 +
.../dictionary/AbstractDictionaryCache.java | 2 +-
.../dictionary/DoubleArrayTrieDictionary.java | 12 +-
.../dictionary/ReverseDictionaryCache.java | 2 +-
.../core/constants/CarbonCommonConstants.java | 3 +-
.../core/datamap/DataMapStoreManager.java | 4 +-
.../carbondata/core/datamap/TableDataMap.java | 6 +-
.../datastore/AbstractBlockIndexStoreCache.java | 8 +-
.../core/datastore/BlockIndexStore.java | 2 +-
.../core/datastore/block/TableBlockInfo.java | 12 +-
.../core/datastore/block/TableTaskInfo.java | 23 ++++
.../impl/FixedLengthDimensionDataChunk.java | 8 ++
.../UnsafeAbstractDimensionDataChunkStore.java | 10 +-
...nsafeFixedLengthDimensionDataChunkStore.java | 24 ++--
...afeVariableLengthDimesionDataChunkStore.java | 28 ++---
.../columnar/BlockIndexerStorageForShort.java | 2 +-
.../core/datastore/impl/FileFactory.java | 3 +
.../core/datastore/impl/FileHolderImpl.java | 8 +-
.../core/datastore/page/ColumnPage.java | 2 +
.../page/UnsafeFixLengthColumnPage.java | 87 +++++++------
.../page/UnsafeVarLengthColumnPage.java | 10 +-
.../page/encoding/DeltaIntegralCodec.java | 9 +-
.../core/datastore/page/encoding/RLECodec.java | 2 +
.../statistics/PrimitivePageStatsCollector.java | 13 ++
.../core/datastore/row/LoadStatusType.java | 4 -
.../client/DictionaryClientHandler.java | 2 +-
.../generator/ServerDictionaryGenerator.java | 11 +-
.../generator/TableDictionaryGenerator.java | 35 +++---
.../dictionary/server/DictionaryServer.java | 2 +-
.../carbondata/core/indexstore/Blocklet.java | 11 +-
.../core/indexstore/UnsafeMemoryDMStore.java | 39 +++---
.../blockletindex/BlockletDataMap.java | 19 +--
.../BlockletDataRefNodeWrapper.java | 2 +-
.../core/indexstore/row/UnsafeDataMapRow.java | 25 ++--
.../MultiDimKeyVarLengthEquiSplitGenerator.java | 23 ----
...tiDimKeyVarLengthVariableSplitGenerator.java | 23 ----
.../DateDirectDictionaryGenerator.java | 2 +-
.../TimeStampDirectDictionaryGenerator.java | 2 +-
.../carbondata/core/locks/HdfsFileLock.java | 2 +-
.../carbondata/core/locks/ZooKeeperLocking.java | 12 +-
.../carbondata/core/memory/CarbonUnsafe.java | 6 +-
.../core/memory/IntPointerBuffer.java | 4 +-
.../carbondata/core/memory/MemoryBlock.java | 7 --
.../core/memory/UnsafeMemoryAllocator.java | 6 +-
.../core/memory/UnsafeMemoryManager.java | 12 +-
.../core/memory/UnsafeSortMemoryManager.java | 12 +-
.../core/metadata/ColumnPageCodecMeta.java | 2 +
.../datatype/DecimalConverterFactory.java | 6 -
.../core/metadata/schema/table/TableInfo.java | 10 +-
.../core/mutate/CarbonUpdateUtil.java | 4 -
.../RestructureBasedRawResultCollector.java | 3 +-
.../scan/complextypes/ComplexQueryType.java | 3 -
.../core/scan/executor/util/QueryUtil.java | 3 +-
.../scan/executor/util/RestructureUtil.java | 4 +-
.../expression/RangeExpressionEvaluator.java | 20 +--
.../expression/conditional/ListExpression.java | 11 +-
.../scan/filter/FilterExpressionProcessor.java | 15 ++-
.../carbondata/core/scan/filter/FilterUtil.java | 8 +-
.../RestructureExcludeFilterExecutorImpl.java | 12 +-
.../RestructureIncludeFilterExecutorImpl.java | 13 +-
.../executer/RowLevelFilterExecuterImpl.java | 4 +-
.../scan/filter/intf/FilterOptimizerImpl.java | 4 +-
.../filter/optimizer/RangeFilterOptmizer.java | 2 +-
.../MeasureColumnResolvedFilterInfo.java | 6 -
.../visitor/CustomTypeDictionaryVisitor.java | 39 +++---
.../visitor/DictionaryColumnVisitor.java | 48 +++----
.../visitor/MeasureColumnVisitor.java | 50 ++++----
.../visitor/NoDictionaryTypeVisitor.java | 48 +++----
.../visitor/RangeDictionaryColumnVisitor.java | 28 +++--
.../visitor/RangeDirectDictionaryVisitor.java | 53 ++++----
.../visitor/RangeNoDictionaryTypeVisitor.java | 52 ++++----
.../scan/result/vector/ColumnVectorInfo.java | 23 ++++
.../core/stats/QueryStatisticsRecorderImpl.java | 14 ++-
.../apache/carbondata/core/util/ByteUtil.java | 49 ++++----
.../carbondata/core/util/CarbonProperties.java | 28 ++---
.../apache/carbondata/core/util/CarbonUtil.java | 11 +-
.../core/util/DataTypeConverterImpl.java | 2 +-
.../carbondata/core/util/DataTypeUtil.java | 6 +-
.../core/util/comparator/Comparator.java | 8 +-
dev/findbugs-exclude.xml | 124 ++++---------------
pom.xml | 2 +-
.../sort/unsafe/UnsafeCarbonRowPage.java | 72 +++++------
.../newflow/sort/unsafe/UnsafeSortDataRows.java | 2 +-
.../unsafe/comparator/UnsafeRowComparator.java | 24 ++--
.../UnsafeRowComparatorForNormalDIms.java | 4 +-
85 files changed, 663 insertions(+), 699 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java b/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java
index 1261726..828dd14 100644
--- a/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java
+++ b/common/src/main/java/org/apache/carbondata/common/logging/impl/ExtendedRollingFileAppender.java
@@ -74,6 +74,10 @@ public class ExtendedRollingFileAppender extends RollingFileAppender {
}
});
+ if (null == files) {
+ return;
+ }
+
int backupFiles = files.length - 1;
if (backupFiles <= maxBackupIndex) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
index f0b8c78..6910888 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/AbstractDictionaryCache.java
@@ -173,7 +173,7 @@ public abstract class AbstractDictionaryCache<K extends DictionaryColumnUniqueId
protected long getSortIndexSize(long numOfRecords) {
// sort index has sort index and reverse sort index,each is 4 byte integer.
// 32 byte is the array header of both the integer arrays
- return numOfRecords * ObjectSizeCalculator.estimate(new Integer(0), 16) * 2 + 32;
+ return numOfRecords * ObjectSizeCalculator.estimate(0, 16) * 2 + 32;
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
index cab1926..ef36d7a 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
@@ -35,10 +35,10 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants;
*/
public class DoubleArrayTrieDictionary {
- public static final byte[] HEAD_MAGIC = new byte[]{
+ private static final byte[] HEAD_MAGIC = new byte[]{
0x44, 0x41, 0x54, 0x54, 0x72, 0x69, 0x65, 0x44, 0x69, 0x63, 0x74
}; // "DATTrieDict"
- public static final int HEAD_LEN = HEAD_MAGIC.length;
+ private static final int HEAD_LEN = HEAD_MAGIC.length;
private static final int INIT_CAPA_VALUE = 256; // init len of double array
private static final int BASE_ROOT_VALUE = 1; // root base value of trie root
@@ -168,7 +168,7 @@ public class DoubleArrayTrieDictionary {
return null;
}
if (check[cpos] == pos) {
- children.add(new Integer(i));
+ children.add(i);
}
}
return children;
@@ -227,9 +227,9 @@ public class DoubleArrayTrieDictionary {
private int conflict(int start, int bKey) {
int from = start;
TreeSet<Integer> children = getChildren(from);
- children.add(new Integer(bKey));
+ children.add(bKey);
int newBasePos = findFreeRoom(children);
- children.remove(new Integer(bKey));
+ children.remove(bKey);
int oldBasePos = base[start];
base[start] = newBasePos;
@@ -351,7 +351,7 @@ public class DoubleArrayTrieDictionary {
*/
public void read(DataInputStream in) throws IOException {
byte[] header = new byte[HEAD_LEN];
- in.read(header);
+ in.readFully(header);
int comp = 0;
for (int i = 0; i < HEAD_LEN; i++) {
comp = HEAD_MAGIC[i] - header[i];
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
index 28568b5..fd5a954 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ReverseDictionaryCache.java
@@ -58,7 +58,7 @@ public class ReverseDictionaryCache<K extends DictionaryColumnUniqueIdentifier,
private static final long sizeOfHashMapNode = ObjectSizeCalculator.estimate(new
DictionaryByteArrayWrapper(new byte[0]), 16) +
- ObjectSizeCalculator.estimate(new Integer(0), 16);
+ ObjectSizeCalculator.estimate(0, 16);
private static final long byteArraySize = ObjectSizeCalculator.estimate(new byte[0], 16);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index 8939a7e..944abc4 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -189,8 +189,7 @@ public final class CarbonCommonConstants {
/**
* Bytes for string 0, it is used in codegen in case of null values.
*/
- public static final byte[] ZERO_BYTE_ARRAY = "0".getBytes();
-
+ public static final byte[] ZERO_BYTE_ARRAY = "0".getBytes(Charset.forName(DEFAULT_CHARSET));
/**
* FILE STATUS IN-PROGRESS
*/
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
index f5bc22f..54318b5 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/DataMapStoreManager.java
@@ -92,7 +92,7 @@ public final class DataMapStoreManager {
try {
DataMapFactory dataMapFactory = factoryClass.newInstance();
dataMapFactory.init(identifier, dataMapName);
- dataMap = new TableDataMap(identifier, dataMapName, dataMapFactory);
+ dataMap = new TableDataMap(dataMapName, dataMapFactory);
} catch (Exception e) {
LOGGER.error(e);
throw new RuntimeException(e);
@@ -119,7 +119,7 @@ public final class DataMapStoreManager {
* @param dataMapName
*/
public void clearDataMap(AbsoluteTableIdentifier identifier, String dataMapName) {
- List<TableDataMap> tableDataMaps = allDataMaps.get(identifier);
+ List<TableDataMap> tableDataMaps = allDataMaps.get(identifier.uniqueName());
if (tableDataMaps != null) {
int i = 0;
for (TableDataMap tableDataMap: tableDataMaps) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
index b55c5d9..5571538 100644
--- a/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/datamap/TableDataMap.java
@@ -25,7 +25,6 @@ import org.apache.carbondata.core.datamap.dev.DataMapFactory;
import org.apache.carbondata.core.events.ChangeEvent;
import org.apache.carbondata.core.events.EventListener;
import org.apache.carbondata.core.indexstore.Blocklet;
-import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
/**
@@ -34,8 +33,6 @@ import org.apache.carbondata.core.scan.filter.resolver.FilterResolverIntf;
*/
public final class TableDataMap implements EventListener {
- private AbsoluteTableIdentifier identifier;
-
private String dataMapName;
private DataMapFactory dataMapFactory;
@@ -43,9 +40,8 @@ public final class TableDataMap implements EventListener {
/**
* It is called to initialize and load the required table datamap metadata.
*/
- public TableDataMap(AbsoluteTableIdentifier identifier, String dataMapName,
+ public TableDataMap(String dataMapName,
DataMapFactory dataMapFactory) {
- this.identifier = identifier;
this.dataMapName = dataMapName;
this.dataMapFactory = dataMapFactory;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java b/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java
index 8cfc602..3a62783 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/AbstractBlockIndexStoreCache.java
@@ -41,10 +41,7 @@ import org.apache.carbondata.core.util.ObjectSizeCalculator;
*/
public abstract class AbstractBlockIndexStoreCache<K, V>
implements Cache<TableBlockUniqueIdentifier, AbstractIndex> {
- /**
- * carbon store path
- */
- protected String carbonStorePath;
+
/**
* CarbonLRU cache
*/
@@ -70,8 +67,7 @@ public abstract class AbstractBlockIndexStoreCache<K, V>
*/
protected Map<String, Object> segmentIDLock;
- public AbstractBlockIndexStoreCache(String carbonStorePath, CarbonLRUCache lruCache) {
- this.carbonStorePath = carbonStorePath;
+ public AbstractBlockIndexStoreCache(CarbonLRUCache lruCache) {
this.lruCache = lruCache;
blockInfoLock = new ConcurrentHashMap<BlockInfo, Object>();
segmentIDLock = new ConcurrentHashMap<String, Object>();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
index 32ca096..fad6ed6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/BlockIndexStore.java
@@ -59,7 +59,7 @@ public class BlockIndexStore<K, V> extends AbstractBlockIndexStoreCache<K, V> {
private static final LogService LOGGER =
LogServiceFactory.getLogService(BlockIndexStore.class.getName());
public BlockIndexStore(String carbonStorePath, CarbonLRUCache lruCache) {
- super(carbonStorePath, lruCache);
+ super(lruCache);
}
/**
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index 316e202..6389528 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -190,13 +190,15 @@ public class TableBlockInfo implements Distributable, Serializable {
if (blockLength != other.blockLength) {
return false;
}
- if (filePath == null && other.filePath != null) {
- return false;
- } else if (filePath != null && other.filePath == null) {
- return false;
- } else if (!filePath.equals(other.filePath)) {
+
+ if (null == filePath || null == other.filePath) {
+ return false;
+ }
+
+ if (!filePath.equals(other.filePath)) {
return false;
}
+
if (blockletInfos.getStartBlockletNumber() != other.blockletInfos.getStartBlockletNumber()) {
return false;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java
index 73aa810..89a7551 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableTaskInfo.java
@@ -60,6 +60,29 @@ public class TableTaskInfo implements Distributable {
return taskId.compareTo(((TableTaskInfo)o).getTaskId());
}
+ @Override public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+
+ if (null == obj) {
+ return false;
+ }
+
+ if (!(obj instanceof TableTaskInfo)) {
+ return false;
+ }
+
+ return 0 == taskId.compareTo(((TableTaskInfo)obj).getTaskId());
+ }
+
+ @Override public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((taskId == null) ? 0 : taskId.hashCode());
+ return result;
+ }
+
/**
* Finding which node has the maximum number of blocks for it.
* @param blockList
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
index bb76bbf..97fd226 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/impl/FixedLengthDimensionDataChunk.java
@@ -110,6 +110,10 @@ public class FixedLengthDimensionDataChunk extends AbstractDimensionDataChunk {
case LONG:
vector.putLong(vectorOffset++, (long) valueFromSurrogate);
break;
+ default:
+ throw new IllegalArgumentException(
+ "unsupported data type: " + columnVectorInfo.directDictionaryGenerator
+ .getReturnType());
}
}
}
@@ -150,6 +154,10 @@ public class FixedLengthDimensionDataChunk extends AbstractDimensionDataChunk {
case LONG:
vector.putLong(vectorOffset++, (long) valueFromSurrogate);
break;
+ default:
+ throw new IllegalArgumentException(
+ "unsupported data type: " + columnVectorInfo.directDictionaryGenerator
+ .getReturnType());
}
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
index 3752fb6..704f2d3 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeAbstractDimensionDataChunkStore.java
@@ -92,16 +92,16 @@ public abstract class UnsafeAbstractDimensionDataChunkStore implements Dimension
invertedIndex.length * CarbonCommonConstants.INT_SIZE_IN_BYTE;
}
// copy the data to memory
- CarbonUnsafe.unsafe
+ CarbonUnsafe.getUnsafe()
.copyMemory(data, CarbonUnsafe.BYTE_ARRAY_OFFSET, dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset(), this.dataLength);
// if inverted index is present then copy the inverted index
// and reverse inverted index to memory
if (isExplicitSorted) {
- CarbonUnsafe.unsafe.copyMemory(invertedIndex, CarbonUnsafe.INT_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(invertedIndex, CarbonUnsafe.INT_ARRAY_OFFSET,
dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + dataLength,
invertedIndex.length * CarbonCommonConstants.INT_SIZE_IN_BYTE);
- CarbonUnsafe.unsafe.copyMemory(invertedIndexReverse, CarbonUnsafe.INT_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(invertedIndexReverse, CarbonUnsafe.INT_ARRAY_OFFSET,
dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset,
invertedIndexReverse.length * CarbonCommonConstants.INT_SIZE_IN_BYTE);
@@ -129,8 +129,8 @@ public abstract class UnsafeAbstractDimensionDataChunkStore implements Dimension
* @return inverted index based on row id passed
*/
@Override public int getInvertedIndex(int rowId) {
- return CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + dataLength + (rowId
+ return CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + dataLength + ((long)rowId
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
index 4ff9c77..8c8d08f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeFixedLengthDimensionDataChunkStore.java
@@ -53,16 +53,16 @@ public class UnsafeFixedLengthDimensionDataChunkStore
@Override public byte[] getRow(int rowId) {
// if column was explicitly sorted we need to get the rowid based inverted index reverse
if (isExplicitSorted) {
- rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+ rowId = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + ((long)rowId
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
}
// creating a row
byte[] data = new byte[columnValueSize];
//copy the row from memory block based on offset
// offset position will be index * each column value length
- CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), data,
+ CarbonUnsafe.getUnsafe().copyMemory(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + ((long)rowId * columnValueSize), data,
CarbonUnsafe.BYTE_ARRAY_OFFSET, columnValueSize);
return data;
}
@@ -77,8 +77,8 @@ public class UnsafeFixedLengthDimensionDataChunkStore
@Override public int getSurrogate(int index) {
// if column was explicitly sorted we need to get the rowid based inverted index reverse
if (isExplicitSorted) {
- index = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (index
+ index = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + ((long)index
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
}
// below part is to convert the byte array to surrogate value
@@ -86,7 +86,7 @@ public class UnsafeFixedLengthDimensionDataChunkStore
int surrogate = 0;
for (int i = 0; i < columnValueSize; i++) {
surrogate <<= 8;
- surrogate ^= CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(),
+ surrogate ^= CarbonUnsafe.getUnsafe().getByte(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + startOffsetOfData) & 0xFF;
startOffsetOfData++;
}
@@ -103,14 +103,14 @@ public class UnsafeFixedLengthDimensionDataChunkStore
@Override public void fillRow(int rowId, byte[] buffer, int offset) {
// if column was explicitly sorted we need to get the rowid based inverted index reverse
if (isExplicitSorted) {
- rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+ rowId = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + ((long)rowId
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
}
//copy the row from memory block based on offset
// offset position will be index * each column value length
- CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + (rowId * columnValueSize), buffer,
+ CarbonUnsafe.getUnsafe().copyMemory(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + ((long)rowId * columnValueSize), buffer,
CarbonUnsafe.BYTE_ARRAY_OFFSET + offset, columnValueSize);
}
@@ -133,7 +133,7 @@ public class UnsafeFixedLengthDimensionDataChunkStore
index = index * columnValueSize;
int compareResult = 0;
for (int i = 0; i < compareValue.length; i++) {
- compareResult = (CarbonUnsafe.unsafe
+ compareResult = (CarbonUnsafe.getUnsafe()
.getByte(dataPageMemoryBlock.getBaseObject(), dataPageMemoryBlock.getBaseOffset() + index)
& 0xff) - (compareValue[i] & 0xff);
if (compareResult != 0) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
index c6c98f9..6193804 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/store/impl/unsafe/UnsafeVariableLengthDimesionDataChunkStore.java
@@ -68,7 +68,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore
// position from where offsets will start
this.dataPointersOffsets = this.invertedIndexReverseOffset;
if (isExplicitSorted) {
- this.dataPointersOffsets += numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE;
+ this.dataPointersOffsets += (long)numberOfRows * CarbonCommonConstants.INT_SIZE_IN_BYTE;
}
// As data is of variable length and data format is
// <length in short><data><length in short><data>
@@ -87,7 +87,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore
// as first position will be start from 2 byte as data is stored first in the memory block
// we need to skip first two bytes this is because first two bytes will be length of the data
// which we have to skip
- CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(),
+ CarbonUnsafe.getUnsafe().putInt(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + pointerOffsets,
CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
// incrementing the pointers as first value is already filled and as we are storing as int
@@ -101,7 +101,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore
// as first offset is already stored, we need to start from the 2nd row in data array
for (int i = 1; i < numberOfRows; i++) {
// first copy the length of previous row
- CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+ CarbonUnsafe.getUnsafe().copyMemory(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + startOffset, length, CarbonUnsafe.BYTE_ARRAY_OFFSET,
CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
buffer.put(length);
@@ -114,7 +114,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore
buffer.clear();
// now put the offset of current row, here we need to add 2 more bytes as current will
// also have length part so we have to skip length
- CarbonUnsafe.unsafe.putInt(dataPageMemoryBlock.getBaseObject(),
+ CarbonUnsafe.getUnsafe().putInt(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + pointerOffsets,
startOffset + CarbonCommonConstants.SHORT_SIZE_IN_BYTE);
// incrementing the pointers as first value is already filled and as we are storing as int
@@ -133,8 +133,8 @@ public class UnsafeVariableLengthDimesionDataChunkStore
@Override public byte[] getRow(int rowId) {
// if column was explicitly sorted we need to get the rowid based inverted index reverse
if (isExplicitSorted) {
- rowId = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + (rowId
+ rowId = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.invertedIndexReverseOffset + ((long)rowId
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
}
// now to get the row from memory block we need to do following thing
@@ -143,13 +143,13 @@ public class UnsafeVariableLengthDimesionDataChunkStore
// Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
// else subtract the current row offset + 2 bytes(to skip the data length)
// with complete data length
- int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ int currentDataOffset = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (rowId
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
short length = 0;
// calculating the length of data
if (rowId < numberOfRows - 1) {
- int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ int OffsetOfNextdata = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((rowId + 1)
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
length = (short) (OffsetOfNextdata - (currentDataOffset
@@ -159,7 +159,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore
length = (short) (this.dataLength - currentDataOffset);
}
byte[] data = new byte[length];
- CarbonUnsafe.unsafe.copyMemory(dataPageMemoryBlock.getBaseObject(),
+ CarbonUnsafe.getUnsafe().copyMemory(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + currentDataOffset, data,
CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
return data;
@@ -200,13 +200,13 @@ public class UnsafeVariableLengthDimesionDataChunkStore
// Subtract the current row offset + 2 bytes(to skip the data length) with next row offset
// else subtract the current row offset
// with complete data length get the offset of set of data
- int currentDataOffset = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
- dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + (index
- * CarbonCommonConstants.INT_SIZE_IN_BYTE));
+ int currentDataOffset = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
+ dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((long)index
+ * CarbonCommonConstants.INT_SIZE_IN_BYTE * 1L));
short length = 0;
// calculating the length of data
if (index < numberOfRows - 1) {
- int OffsetOfNextdata = CarbonUnsafe.unsafe.getInt(dataPageMemoryBlock.getBaseObject(),
+ int OffsetOfNextdata = CarbonUnsafe.getUnsafe().getInt(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + this.dataPointersOffsets + ((index + 1)
* CarbonCommonConstants.INT_SIZE_IN_BYTE));
length = (short) (OffsetOfNextdata - (currentDataOffset
@@ -220,7 +220,7 @@ public class UnsafeVariableLengthDimesionDataChunkStore
int compareResult;
int compareLength = Math.min(length , compareValue.length);
for (int i = 0; i < compareLength; i++) {
- compareResult = (CarbonUnsafe.unsafe.getByte(dataPageMemoryBlock.getBaseObject(),
+ compareResult = (CarbonUnsafe.getUnsafe().getByte(dataPageMemoryBlock.getBaseObject(),
dataPageMemoryBlock.getBaseOffset() + currentDataOffset) & 0xff) - (compareValue[i]
& 0xff);
// if compare result is not equal we can break
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
index 7303b66..a91d6bc 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/columnar/BlockIndexerStorageForShort.java
@@ -117,7 +117,7 @@ public class BlockIndexerStorageForShort implements IndexStorage<short[]> {
} else {
list.add(rowIds[i - 1]);
}
- double compressionPercentage = (((list.size() + map.size()) * 100) / rowIds.length);
+ int compressionPercentage = (((list.size() + map.size()) * 100) / rowIds.length);
if (compressionPercentage > 70) {
rowIdPage = rowIds;
} else {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
index 3477834..3191200 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileFactory.java
@@ -367,6 +367,9 @@ public final class FileFactory {
return path.delete();
}
File[] files = path.listFiles();
+ if (null == files) {
+ return true;
+ }
for (int i = 0; i < files.length; i++) {
deleteAllFilesOfDir(files[i]);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
index 8e6aafc..c867f75 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/impl/FileHolderImpl.java
@@ -217,7 +217,13 @@ public class FileHolderImpl implements FileHolder {
@Override public DataInputStream getDataInputStream(String filePath, long offset)
throws IOException {
FileInputStream stream = new FileInputStream(filePath);
- stream.skip(offset);
+ long skipped = stream.skip(offset);
+ long toSkip = offset - skipped;
+ while (toSkip <= 0) {
+ skipped = stream.skip(toSkip);
+ toSkip = toSkip - skipped;
+ }
+
return new DataInputStream(new BufferedInputStream(stream));
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index a0eeb44..8b44f07 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -410,6 +410,8 @@ public abstract class ColumnPage {
case DECIMAL:
putDecimal(rowId, BigDecimal.ZERO);
break;
+ default:
+ throw new IllegalArgumentException("unsupported data type: " + dataType);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
index f019ae8..06d952d 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
@@ -82,40 +82,40 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
@Override
public void putByte(int rowId, byte value) {
long offset = rowId << byteBits;
- CarbonUnsafe.unsafe.putByte(baseAddress, baseOffset + offset, value);
+ CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset, value);
}
@Override
public void putShort(int rowId, short value) {
long offset = rowId << shortBits;
- CarbonUnsafe.unsafe.putShort(baseAddress, baseOffset + offset, value);
+ CarbonUnsafe.getUnsafe().putShort(baseAddress, baseOffset + offset, value);
}
@Override
public void putShortInt(int rowId, int value) {
byte[] data = ByteUtil.to3Bytes(value);
- long offset = rowId * 3;
- CarbonUnsafe.unsafe.putByte(baseAddress, baseOffset + offset, data[0]);
- CarbonUnsafe.unsafe.putByte(baseAddress, baseOffset + offset + 1, data[1]);
- CarbonUnsafe.unsafe.putByte(baseAddress, baseOffset + offset + 2, data[2]);
+ long offset = rowId * 3L;
+ CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset, data[0]);
+ CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset + 1, data[1]);
+ CarbonUnsafe.getUnsafe().putByte(baseAddress, baseOffset + offset + 2, data[2]);
}
@Override
public void putInt(int rowId, int value) {
long offset = rowId << intBits;
- CarbonUnsafe.unsafe.putInt(baseAddress, baseOffset + offset, value);
+ CarbonUnsafe.getUnsafe().putInt(baseAddress, baseOffset + offset, value);
}
@Override
public void putLong(int rowId, long value) {
long offset = rowId << longBits;
- CarbonUnsafe.unsafe.putLong(baseAddress, baseOffset + offset, value);
+ CarbonUnsafe.getUnsafe().putLong(baseAddress, baseOffset + offset, value);
}
@Override
public void putDouble(int rowId, double value) {
long offset = rowId << doubleBits;
- CarbonUnsafe.unsafe.putDouble(baseAddress, baseOffset + offset, value);
+ CarbonUnsafe.getUnsafe().putDouble(baseAddress, baseOffset + offset, value);
}
@Override
@@ -135,47 +135,47 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
@Override
public byte getByte(int rowId) {
long offset = rowId << byteBits;
- return CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset);
+ return CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset);
}
@Override
public short getShort(int rowId) {
long offset = rowId << shortBits;
- return CarbonUnsafe.unsafe.getShort(baseAddress, baseOffset + offset);
+ return CarbonUnsafe.getUnsafe().getShort(baseAddress, baseOffset + offset);
}
@Override
public int getShortInt(int rowId) {
- long offset = rowId * 3;
+ long offset = rowId * 3L;
byte[] data = new byte[3];
- data[0] = CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset);
- data[1] = CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset + 1);
- data[2] = CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset + 2);
+ data[0] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset);
+ data[1] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset + 1);
+ data[2] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset + 2);
return ByteUtil.valueOf3Bytes(data, 0);
}
@Override
public int getInt(int rowId) {
long offset = rowId << intBits;
- return CarbonUnsafe.unsafe.getInt(baseAddress, baseOffset + offset);
+ return CarbonUnsafe.getUnsafe().getInt(baseAddress, baseOffset + offset);
}
@Override
public long getLong(int rowId) {
long offset = rowId << longBits;
- return CarbonUnsafe.unsafe.getLong(baseAddress, baseOffset + offset);
+ return CarbonUnsafe.getUnsafe().getLong(baseAddress, baseOffset + offset);
}
@Override
public float getFloat(int rowId) {
long offset = rowId << floatBits;
- return CarbonUnsafe.unsafe.getFloat(baseAddress, baseOffset + offset);
+ return CarbonUnsafe.getUnsafe().getFloat(baseAddress, baseOffset + offset);
}
@Override
public double getDouble(int rowId) {
long offset = rowId << doubleBits;
- return CarbonUnsafe.unsafe.getDouble(baseAddress, baseOffset + offset);
+ return CarbonUnsafe.getUnsafe().getDouble(baseAddress, baseOffset + offset);
}
@Override
@@ -192,7 +192,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
byte[] data = new byte[getPageSize()];
for (int i = 0; i < data.length; i++) {
long offset = i << byteBits;
- data[i] = CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset);
+ data[i] = CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset);
}
return data;
}
@@ -202,7 +202,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
short[] data = new short[getPageSize()];
for (int i = 0; i < data.length; i++) {
long offset = i << shortBits;
- data[i] = CarbonUnsafe.unsafe.getShort(baseAddress, baseOffset + offset);
+ data[i] = CarbonUnsafe.getUnsafe().getShort(baseAddress, baseOffset + offset);
}
return data;
}
@@ -210,7 +210,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
@Override
public byte[] getShortIntPage() {
byte[] data = new byte[pageSize * 3];
- CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset,
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset,
data, CarbonUnsafe.BYTE_ARRAY_OFFSET, data.length);
return data;
}
@@ -220,7 +220,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
int[] data = new int[getPageSize()];
for (int i = 0; i < data.length; i++) {
long offset = i << intBits;
- data[i] = CarbonUnsafe.unsafe.getInt(baseAddress, baseOffset + offset);
+ data[i] = CarbonUnsafe.getUnsafe().getInt(baseAddress, baseOffset + offset);
}
return data;
}
@@ -230,7 +230,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
long[] data = new long[getPageSize()];
for (int i = 0; i < data.length; i++) {
long offset = i << longBits;
- data[i] = CarbonUnsafe.unsafe.getLong(baseAddress, baseOffset + offset);
+ data[i] = CarbonUnsafe.getUnsafe().getLong(baseAddress, baseOffset + offset);
}
return data;
}
@@ -240,7 +240,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
float[] data = new float[getPageSize()];
for (int i = 0; i < data.length; i++) {
long offset = i << floatBits;
- data[i] = CarbonUnsafe.unsafe.getFloat(baseAddress, baseOffset + offset);
+ data[i] = CarbonUnsafe.getUnsafe().getFloat(baseAddress, baseOffset + offset);
}
return data;
}
@@ -250,7 +250,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
double[] data = new double[getPageSize()];
for (int i = 0; i < data.length; i++) {
long offset = i << doubleBits;
- data[i] = CarbonUnsafe.unsafe.getDouble(baseAddress, baseOffset + offset);
+ data[i] = CarbonUnsafe.getUnsafe().getDouble(baseAddress, baseOffset + offset);
}
return data;
}
@@ -267,43 +267,43 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
@Override
public void setBytePage(byte[] byteData) {
- CarbonUnsafe.unsafe.copyMemory(byteData, CarbonUnsafe.BYTE_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(byteData, CarbonUnsafe.BYTE_ARRAY_OFFSET,
baseAddress, baseOffset, byteData.length << byteBits);
}
@Override
public void setShortPage(short[] shortData) {
- CarbonUnsafe.unsafe.copyMemory(shortData, CarbonUnsafe.SHORT_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(shortData, CarbonUnsafe.SHORT_ARRAY_OFFSET,
baseAddress, baseOffset, shortData.length << shortBits);
}
@Override
public void setShortIntPage(byte[] shortIntData) {
- CarbonUnsafe.unsafe.copyMemory(shortIntData, CarbonUnsafe.BYTE_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(shortIntData, CarbonUnsafe.BYTE_ARRAY_OFFSET,
baseAddress, baseOffset, shortIntData.length);
}
@Override
public void setIntPage(int[] intData) {
- CarbonUnsafe.unsafe.copyMemory(intData, CarbonUnsafe.INT_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(intData, CarbonUnsafe.INT_ARRAY_OFFSET,
baseAddress, baseOffset, intData.length << intBits);
}
@Override
public void setLongPage(long[] longData) {
- CarbonUnsafe.unsafe.copyMemory(longData, CarbonUnsafe.LONG_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(longData, CarbonUnsafe.LONG_ARRAY_OFFSET,
baseAddress, baseOffset, longData.length << longBits);
}
@Override
public void setFloatPage(float[] floatData) {
- CarbonUnsafe.unsafe.copyMemory(floatData, CarbonUnsafe.FLOAT_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(floatData, CarbonUnsafe.FLOAT_ARRAY_OFFSET,
baseAddress, baseOffset, floatData.length << floatBits);
}
@Override
public void setDoublePage(double[] doubleData) {
- CarbonUnsafe.unsafe.copyMemory(doubleData, CarbonUnsafe.DOUBLE_ARRAY_OFFSET,
+ CarbonUnsafe.getUnsafe().copyMemory(doubleData, CarbonUnsafe.DOUBLE_ARRAY_OFFSET,
baseAddress, baseOffset, doubleData.length << doubleBits);
}
@@ -328,37 +328,37 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
case BYTE:
for (int i = 0; i < pageSize; i++) {
long offset = i << byteBits;
- codec.encode(i, CarbonUnsafe.unsafe.getByte(baseAddress, baseOffset + offset));
+ codec.encode(i, CarbonUnsafe.getUnsafe().getByte(baseAddress, baseOffset + offset));
}
break;
case SHORT:
for (int i = 0; i < pageSize; i++) {
long offset = i << shortBits;
- codec.encode(i, CarbonUnsafe.unsafe.getShort(baseAddress, baseOffset + offset));
+ codec.encode(i, CarbonUnsafe.getUnsafe().getShort(baseAddress, baseOffset + offset));
}
break;
case INT:
for (int i = 0; i < pageSize; i++) {
long offset = i << intBits;
- codec.encode(i, CarbonUnsafe.unsafe.getInt(baseAddress, baseOffset + offset));
+ codec.encode(i, CarbonUnsafe.getUnsafe().getInt(baseAddress, baseOffset + offset));
}
break;
case LONG:
for (int i = 0; i < pageSize; i++) {
long offset = i << longBits;
- codec.encode(i, CarbonUnsafe.unsafe.getLong(baseAddress, baseOffset + offset));
+ codec.encode(i, CarbonUnsafe.getUnsafe().getLong(baseAddress, baseOffset + offset));
}
break;
case FLOAT:
for (int i = 0; i < pageSize; i++) {
long offset = i << floatBits;
- codec.encode(i, CarbonUnsafe.unsafe.getFloat(baseAddress, baseOffset + offset));
+ codec.encode(i, CarbonUnsafe.getUnsafe().getFloat(baseAddress, baseOffset + offset));
}
break;
case DOUBLE:
for (int i = 0; i < pageSize; i++) {
long offset = i << doubleBits;
- codec.encode(i, CarbonUnsafe.unsafe.getDouble(baseAddress, baseOffset + offset));
+ codec.encode(i, CarbonUnsafe.getUnsafe().getDouble(baseAddress, baseOffset + offset));
}
break;
default:
@@ -366,8 +366,7 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
}
}
- @Override
- public byte[] compress(Compressor compressor) throws MemoryException, IOException {
+ @Override public byte[] compress(Compressor compressor) throws MemoryException, IOException {
if (UnsafeMemoryManager.isOffHeap()) {
// use raw compression and copy to byte[]
int inputSize = pageSize * dataType.getSizeInBytes();
@@ -377,13 +376,13 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
long outSize = compressor.rawCompress(baseOffset, inputSize, compressed.getBaseOffset());
assert outSize < Integer.MAX_VALUE;
byte[] output = new byte[(int) outSize];
- CarbonUnsafe.unsafe.copyMemory(compressed.getBaseObject(), compressed.getBaseOffset(), output,
- CarbonUnsafe.BYTE_ARRAY_OFFSET, outSize);
+ CarbonUnsafe.getUnsafe()
+ .copyMemory(compressed.getBaseObject(), compressed.getBaseOffset(), output,
+ CarbonUnsafe.BYTE_ARRAY_OFFSET, outSize);
UnsafeMemoryManager.INSTANCE.freeMemory(taskId, compressed);
return output;
} else {
return super.compress(compressor);
}
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
index c89d29d..c7d21ef 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeVarLengthColumnPage.java
@@ -95,7 +95,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
if (totalLength + requestSize > capacity) {
int newSize = 2 * capacity;
MemoryBlock newBlock = UnsafeMemoryManager.allocateMemoryWithRetry(taskId, newSize);
- CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset,
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset,
newBlock.getBaseObject(), newBlock.getBaseOffset(), capacity);
UnsafeMemoryManager.INSTANCE.freeMemory(taskId, memoryBlock);
memoryBlock = newBlock;
@@ -117,7 +117,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
} catch (MemoryException e) {
throw new RuntimeException(e);
}
- CarbonUnsafe.unsafe.copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset,
+ CarbonUnsafe.getUnsafe().copyMemory(bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET + offset,
baseAddress, baseOffset + rowOffset[rowId], length);
}
@@ -129,7 +129,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
public BigDecimal getDecimal(int rowId) {
int length = rowOffset[rowId + 1] - rowOffset[rowId];
byte[] bytes = new byte[length];
- CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset + rowOffset[rowId],
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset[rowId],
bytes, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
return decimalConverter.getDecimal(bytes);
@@ -141,7 +141,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
for (int rowId = 0; rowId < pageSize; rowId++) {
int length = rowOffset[rowId + 1] - rowOffset[rowId];
byte[] rowData = new byte[length];
- CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset + rowOffset[rowId],
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset[rowId],
rowData, CarbonUnsafe.BYTE_ARRAY_OFFSET, length);
bytes[rowId] = rowData;
}
@@ -150,7 +150,7 @@ public class UnsafeVarLengthColumnPage extends VarLengthColumnPageBase {
@Override
void copyBytes(int rowId, byte[] dest, int destOffset, int length) {
- CarbonUnsafe.unsafe.copyMemory(baseAddress, baseOffset + rowOffset[rowId],
+ CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset + rowOffset[rowId],
dest, CarbonUnsafe.BYTE_ARRAY_OFFSET + destOffset, length);
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
index ba61cd3..80545b9 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DeltaIntegralCodec.java
@@ -44,8 +44,8 @@ public class DeltaIntegralCodec extends AdaptiveCompressionCodec {
return new DeltaIntegralCodec(srcDataType, targetDataType, stats, compressor);
}
- private DeltaIntegralCodec(DataType srcDataType, DataType targetDataType,
- SimpleStatsResult stats, Compressor compressor) {
+ private DeltaIntegralCodec(DataType srcDataType, DataType targetDataType, SimpleStatsResult stats,
+ Compressor compressor) {
super(srcDataType, targetDataType, stats, compressor);
switch (srcDataType) {
case BYTE:
@@ -62,8 +62,11 @@ public class DeltaIntegralCodec extends AdaptiveCompressionCodec {
break;
case FLOAT:
case DOUBLE:
- max = (long)((double) stats.getMax());
+ max = (long) ((double) stats.getMax());
break;
+ default:
+ throw new UnsupportedOperationException(
+ "unsupported data type for Delta compress: " + srcDataType);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java
index 46c5c58..dda89e0 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/RLECodec.java
@@ -326,6 +326,8 @@ public class RLECodec implements ColumnPageCodec {
case LONG:
decodeLongPage(in, resultPage);
break;
+ default:
+ throw new RuntimeException("unsupported datatype:" + dataType);
}
return resultPage;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
index 918039a..927ab5f 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PrimitivePageStatsCollector.java
@@ -88,6 +88,9 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
instance.scale = meta.getScale();
instance.precision = meta.getPrecision();
break;
+ default:
+ throw new UnsupportedOperationException(
+ "unsupported data type for stats collection: " + meta.getSrcDataType());
}
return instance;
}
@@ -125,6 +128,9 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
instance.scale = -1;
instance.precision = -1;
break;
+ default:
+ throw new UnsupportedOperationException(
+ "unsupported data type for Stats collection: " + meta.getType());
}
return instance;
}
@@ -159,6 +165,10 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
decimal = scale;
this.scale = scale;
this.precision = precision;
+ break;
+ default:
+ throw new UnsupportedOperationException(
+ "unsupported data type for Stats collection: " + dataType);
}
}
@@ -192,6 +202,9 @@ public class PrimitivePageStatsCollector implements ColumnPageStatsCollector, Si
minDecimal = (minDecimal.compareTo(zeroDecimal) < 0) ? minDecimal : zeroDecimal;
}
break;
+ default:
+ throw new UnsupportedOperationException(
+ "unsupported data type for Stats collection: " + dataType);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java b/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java
index cbdeadb..d3c5a94 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/row/LoadStatusType.java
@@ -34,8 +34,4 @@ public enum LoadStatusType {
public String getMessage() {
return message;
}
-
- public void setMessage(String message) {
- this.message = message;
- }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
index 9922523..01ef59a 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/client/DictionaryClientHandler.java
@@ -59,7 +59,7 @@ public class DictionaryClientHandler extends ChannelInboundHandlerAdapter {
DictionaryMessage key = new DictionaryMessage();
key.readData(data);
data.release();
- responseMsgQueue.offer(key);
+ responseMsgQueue.add(key);
} catch (Exception e) {
LOGGER.error(e);
throw e;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
index 9246830..221f9b7 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/ServerDictionaryGenerator.java
@@ -16,6 +16,7 @@
*/
package org.apache.carbondata.core.dictionary.generator;
+import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -71,8 +72,14 @@ public class ServerDictionaryGenerator implements DictionaryGenerator<Integer, D
}
public void writeDictionaryData() throws Exception {
- for (String tableUniqueName: tableMap.keySet()) {
- TableDictionaryGenerator generator = tableMap.get(tableUniqueName);
+ final Iterator<Map.Entry<String, TableDictionaryGenerator>> iterator =
+ tableMap.entrySet().iterator();
+ String tableUniqueName;
+ TableDictionaryGenerator generator;
+ while (iterator.hasNext()) {
+ Map.Entry<String, TableDictionaryGenerator> entry = iterator.next();
+ tableUniqueName = entry.getKey();
+ generator = entry.getValue();
generator.writeDictionaryData(tableUniqueName);
}
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
index 56ed7b9..ee3a2fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
@@ -16,14 +16,11 @@
*/
package org.apache.carbondata.core.dictionary.generator;
-import java.util.ArrayList;
-import java.util.List;
+import java.io.IOException;
import java.util.Map;
-import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.carbondata.common.logging.LogService;
@@ -83,7 +80,6 @@ public class TableDictionaryGenerator
@Override public void writeDictionaryData(String tableUniqueName) {
int numOfCores = 1;
- final String tableName = tableUniqueName;
try {
numOfCores = Integer.parseInt(CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.NUM_CORES_LOADING,
@@ -92,16 +88,9 @@ public class TableDictionaryGenerator
numOfCores = Integer.parseInt(CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
}
long start = System.currentTimeMillis();
- List<Future<Void>> taskSubmitList =
- new ArrayList<>(columnMap.size());
ExecutorService executorService = Executors.newFixedThreadPool(numOfCores);
- for (final DictionaryGenerator generator: columnMap.values()) {
- taskSubmitList.add(executorService.submit(new Callable<Void>() {
- @Override public Void call() throws Exception {
- ((DictionaryWriter) (generator)).writeDictionaryData(tableName);
- return null;
- }
- }));
+ for (final DictionaryGenerator generator : columnMap.values()) {
+ executorService.execute(new WriteDictionaryDataRunnable(generator, tableUniqueName));
}
try {
@@ -125,4 +114,22 @@ public class TableDictionaryGenerator
}
}
}
+
+ private static class WriteDictionaryDataRunnable implements Runnable {
+ private final DictionaryGenerator generator;
+ private final String tableUniqueName;
+
+ public WriteDictionaryDataRunnable(DictionaryGenerator generator, String tableUniqueName) {
+ this.generator = generator;
+ this.tableUniqueName = tableUniqueName;
+ }
+
+ @Override public void run() {
+ try {
+ ((DictionaryWriter)generator).writeDictionaryData(tableUniqueName);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
index 84f2a0d..617038e 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/DictionaryServer.java
@@ -47,7 +47,7 @@ public class DictionaryServer {
private EventLoopGroup worker;
private int port;
private static Object lock = new Object();
- private static DictionaryServer INSTANCE = null;
+ private static volatile DictionaryServer INSTANCE = null;
private DictionaryServer(int port) {
startServer(port);
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
index 66da4d0..c3a72f0 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/Blocklet.java
@@ -31,7 +31,7 @@ import org.apache.hadoop.fs.RemoteIterator;
*/
public class Blocklet implements Serializable {
- private Path path;
+ private String path;
private String segmentId;
@@ -44,12 +44,12 @@ public class Blocklet implements Serializable {
private String[] location;
public Blocklet(String path, String blockletId) {
- this.path = new Path(path);
+ this.path = path;
this.blockletId = blockletId;
}
public Path getPath() {
- return path;
+ return new Path(path);
}
public String getBlockletId() {
@@ -65,8 +65,9 @@ public class Blocklet implements Serializable {
}
public void updateLocations() throws IOException {
- FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
- RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(path);
+ Path fspath = new Path(path);
+ FileSystem fs = fspath.getFileSystem(FileFactory.getConfiguration());
+ RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(fspath);
LocatedFileStatus fileStatus = iter.next();
location = fileStatus.getBlockLocations()[0].getHosts();
length = fileStatus.getLen();
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
index eb67294..5d17426 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/UnsafeMemoryDMStore.java
@@ -25,7 +25,7 @@ import org.apache.carbondata.core.memory.UnsafeMemoryManager;
import org.apache.carbondata.core.util.ThreadLocalTaskInfo;
import static org.apache.carbondata.core.memory.CarbonUnsafe.BYTE_ARRAY_OFFSET;
-import static org.apache.carbondata.core.memory.CarbonUnsafe.unsafe;
+import static org.apache.carbondata.core.memory.CarbonUnsafe.getUnsafe;
/**
* Store the data map row @{@link DataMapRow} data to unsafe.
@@ -67,7 +67,7 @@ public class UnsafeMemoryDMStore {
if (runningLength + rowSize >= allocatedSize) {
MemoryBlock allocate =
UnsafeMemoryManager.allocateMemoryWithRetry(taskId, allocatedSize + capacity);
- unsafe.copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(),
+ getUnsafe().copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(),
allocate.getBaseObject(), allocate.getBaseOffset(), runningLength);
UnsafeMemoryManager.INSTANCE.freeMemory(taskId, memoryBlock);
allocatedSize = allocatedSize + capacity;
@@ -104,51 +104,57 @@ public class UnsafeMemoryDMStore {
case FIXED:
switch (schema.getDataType()) {
case BYTE:
- unsafe.putByte(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
- row.getByte(index));
+ getUnsafe()
+ .putByte(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
+ row.getByte(index));
runningLength += row.getSizeInBytes(index);
break;
case SHORT:
- unsafe
+ getUnsafe()
.putShort(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
row.getShort(index));
runningLength += row.getSizeInBytes(index);
break;
case INT:
- unsafe.putInt(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
- row.getInt(index));
+ getUnsafe()
+ .putInt(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
+ row.getInt(index));
runningLength += row.getSizeInBytes(index);
break;
case LONG:
- unsafe.putLong(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
- row.getLong(index));
+ getUnsafe()
+ .putLong(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
+ row.getLong(index));
runningLength += row.getSizeInBytes(index);
break;
case FLOAT:
- unsafe
+ getUnsafe()
.putFloat(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
row.getFloat(index));
runningLength += row.getSizeInBytes(index);
break;
case DOUBLE:
- unsafe
+ getUnsafe()
.putDouble(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset() + runningLength,
row.getDouble(index));
runningLength += row.getSizeInBytes(index);
break;
case BYTE_ARRAY:
byte[] data = row.getByteArray(index);
- unsafe.copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(),
+ getUnsafe().copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(),
memoryBlock.getBaseOffset() + runningLength, data.length);
runningLength += row.getSizeInBytes(index);
break;
+ default:
+ throw new UnsupportedOperationException(
+ "unsupported data type for unsafe storage: " + schema.getDataType());
}
break;
case VARIABLE:
byte[] data = row.getByteArray(index);
- unsafe.putShort(memoryBlock.getBaseOffset() + runningLength, (short) data.length);
+ getUnsafe().putShort(memoryBlock.getBaseOffset() + runningLength, (short) data.length);
runningLength += 2;
- unsafe.copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(),
+ getUnsafe().copyMemory(data, BYTE_ARRAY_OFFSET, memoryBlock.getBaseObject(),
memoryBlock.getBaseOffset() + runningLength, data.length);
runningLength += data.length;
break;
@@ -160,6 +166,9 @@ public class UnsafeMemoryDMStore {
addToUnsafe(childSchemas[i], struct, i);
}
break;
+ default:
+ throw new UnsupportedOperationException(
+ "unsupported data type for unsafe storage: " + schema.getDataType());
}
}
@@ -172,7 +181,7 @@ public class UnsafeMemoryDMStore {
if (runningLength < allocatedSize) {
MemoryBlock allocate =
UnsafeMemoryManager.allocateMemoryWithRetry(taskId, runningLength);
- unsafe.copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(),
+ getUnsafe().copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(),
allocate.getBaseObject(), allocate.getBaseOffset(), runningLength);
UnsafeMemoryManager.INSTANCE.freeMemory(taskId, memoryBlock);
memoryBlock = allocate;
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
index 2e82c46..928c78b 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java
@@ -24,6 +24,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.BitSet;
import java.util.Comparator;
import java.util.List;
@@ -31,6 +32,7 @@ import java.util.List;
import org.apache.carbondata.common.logging.LogService;
import org.apache.carbondata.common.logging.LogServiceFactory;
import org.apache.carbondata.core.cache.Cacheable;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
import org.apache.carbondata.core.datamap.DataMapDistributable;
import org.apache.carbondata.core.datamap.dev.DataMap;
import org.apache.carbondata.core.datastore.IndexKey;
@@ -131,7 +133,8 @@ public class BlockletDataMap implements DataMap, Cacheable {
row.setInt(blockletInfo.getNumberOfRows(), ordinal++);
// add file path
- byte[] filePathBytes = filePath.getBytes();
+ byte[] filePathBytes =
+ filePath.getBytes(CarbonCommonConstants.DEFAULT_CHARSET_CLASS);
row.setByteArray(filePathBytes, ordinal++);
// add pages
@@ -244,10 +247,11 @@ public class BlockletDataMap implements DataMap, Cacheable {
}
if (LOGGER.isDebugEnabled()) {
LOGGER.debug(
- "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey
- .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey
- .getNoDictionaryKeys() + "Dictionary End Key: " + searchEndKey.getDictionaryKeys()
- + "No Dictionary End Key " + searchEndKey.getNoDictionaryKeys());
+ "Successfully retrieved the start and end key" + "Dictionary Start Key: " + Arrays
+ .toString(searchStartKey.getDictionaryKeys()) + "No Dictionary Start Key " + Arrays
+ .toString(searchStartKey.getNoDictionaryKeys()) + "Dictionary End Key: " + Arrays
+ .toString(searchEndKey.getDictionaryKeys()) + "No Dictionary End Key " + Arrays
+ .toString(searchEndKey.getNoDictionaryKeys()));
}
if (filterExp == null) {
int rowCount = unsafeMemoryDMStore.getRowCount();
@@ -284,8 +288,9 @@ public class BlockletDataMap implements DataMap, Cacheable {
}
private Blocklet createBlocklet(DataMapRow row, int blockletId) {
- Blocklet blocklet =
- new Blocklet(new String(row.getByteArray(FILE_PATH_INDEX)), blockletId + "");
+ Blocklet blocklet = new Blocklet(
+ new String(row.getByteArray(FILE_PATH_INDEX), CarbonCommonConstants.DEFAULT_CHARSET_CLASS),
+ blockletId + "");
BlockletDetailInfo detailInfo = new BlockletDetailInfo();
detailInfo.setRowCount(row.getInt(ROW_COUNT_INDEX));
detailInfo.setPagesCount(row.getShort(PAGE_COUNT_INDEX));
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
index 5509c75..8896e5d 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
@@ -52,7 +52,7 @@ public class BlockletDataRefNodeWrapper implements DataRefNode {
@Override public DataRefNode getNextDataRefNode() {
if (index + 1 < blockInfos.size()) {
- new BlockletDataRefNodeWrapper(blockInfos, index + 1, dimensionLens);
+ return new BlockletDataRefNodeWrapper(blockInfos, index + 1, dimensionLens);
}
return null;
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java b/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
index c398115..2c76990 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/row/UnsafeDataMapRow.java
@@ -21,7 +21,7 @@ import org.apache.carbondata.core.indexstore.schema.DataMapSchema;
import org.apache.carbondata.core.memory.MemoryBlock;
import static org.apache.carbondata.core.memory.CarbonUnsafe.BYTE_ARRAY_OFFSET;
-import static org.apache.carbondata.core.memory.CarbonUnsafe.unsafe;
+import static org.apache.carbondata.core.memory.CarbonUnsafe.getUnsafe;
/**
* Unsafe implementation of data map row.
@@ -43,14 +43,15 @@ public class UnsafeDataMapRow extends DataMapRow {
int position = getPosition(ordinal);
switch (schemas[ordinal].getSchemaType()) {
case VARIABLE:
- length = unsafe.getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
+ length =
+ getUnsafe().getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
position += 2;
break;
default:
length = schemas[ordinal].getLength();
}
byte[] data = new byte[length];
- unsafe.copyMemory(block.getBaseObject(), block.getBaseOffset() + pointer + position, data,
+ getUnsafe().copyMemory(block.getBaseObject(), block.getBaseOffset() + pointer + position, data,
BYTE_ARRAY_OFFSET, data.length);
return data;
}
@@ -60,7 +61,8 @@ public class UnsafeDataMapRow extends DataMapRow {
int position = getPosition(ordinal);
switch (schemas[ordinal].getSchemaType()) {
case VARIABLE:
- length = unsafe.getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
+ length =
+ getUnsafe().getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
break;
default:
length = schemas[ordinal].getLength();
@@ -72,7 +74,8 @@ public class UnsafeDataMapRow extends DataMapRow {
int length;
switch (schemas[ordinal].getSchemaType()) {
case VARIABLE:
- length = unsafe.getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
+ length =
+ getUnsafe().getShort(block.getBaseObject(), block.getBaseOffset() + pointer + position);
break;
default:
length = schemas[ordinal].getLength();
@@ -91,7 +94,7 @@ public class UnsafeDataMapRow extends DataMapRow {
}
@Override public int getInt(int ordinal) {
- return unsafe
+ return getUnsafe()
.getInt(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
}
@@ -104,7 +107,7 @@ public class UnsafeDataMapRow extends DataMapRow {
}
@Override public byte getByte(int ordinal) {
- return unsafe
+ return getUnsafe()
.getByte(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
}
@@ -113,7 +116,7 @@ public class UnsafeDataMapRow extends DataMapRow {
}
@Override public short getShort(int ordinal) {
- return unsafe
+ return getUnsafe()
.getShort(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
}
@@ -122,7 +125,7 @@ public class UnsafeDataMapRow extends DataMapRow {
}
@Override public long getLong(int ordinal) {
- return unsafe
+ return getUnsafe()
.getLong(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
}
@@ -131,7 +134,7 @@ public class UnsafeDataMapRow extends DataMapRow {
}
@Override public float getFloat(int ordinal) {
- return unsafe
+ return getUnsafe()
.getFloat(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
}
@@ -140,7 +143,7 @@ public class UnsafeDataMapRow extends DataMapRow {
}
@Override public double getDouble(int ordinal) {
- return unsafe
+ return getUnsafe()
.getDouble(block.getBaseObject(), block.getBaseOffset() + pointer + getPosition(ordinal));
}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/500654e6/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java
index f628bca..72b250b 100644
--- a/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/keygenerator/columnar/impl/MultiDimKeyVarLengthEquiSplitGenerator.java
@@ -109,29 +109,6 @@ public class MultiDimKeyVarLengthEquiSplitGenerator extends MultiDimKeyVarLength
dimBlockArray[j] = convertToArray(dimBlockSet[j]);
}
- int[][] splitDimArrayLocalIndexes = new int[splitDimArray.length][];
- for (int j = 0; j < splitDimArrayLocalIndexes.length; j++) {
- splitDimArrayLocalIndexes[j] = splitDimArray[j].length > 0 ?
- new int[] { 0, splitDimArray[j][1] - splitDimArray[j][0] } :
- new int[0];
- }
-
- int[][][] byteRangesForDims = new int[byteRangesForKeys.length][][];
- for (int j = 0; j < byteRangesForKeys.length; j++) {
- if (dimBlockArray[j].length > 1) {
- int[] bArray1 = splitDimArrayLocalIndexes[dimBlockArray[j][0]];
- byteRangesForDims[j] = new int[2][2];
- byteRangesForDims[j][0] =
- new int[] { bArray1[bArray1.length - 1], bArray1[bArray1.length - 1] };
- byteRangesForDims[j][1] = new int[] { 0,
- (byteRangesForKeys[j][byteRangesForKeys[j].length - 1] - byteRangesForKeys[j][0]) - 1 };
- } else {
- byteRangesForDims[j] = new int[1][1];
- int[] bArray1 = splitDimArray[dimBlockArray[j][0]];
- byteRangesForDims[j][0] = new int[] { byteRangesForKeys[j][0] - bArray1[0],
- byteRangesForKeys[j][1] - bArray1[0] };
- }
- }
blockKeySize = new int[splitDimArray.length];
for (int j = 0; j < blockKeySize.length; j++) {