You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@carbondata.apache.org by ra...@apache.org on 2018/05/22 16:34:01 UTC
carbondata git commit: [HOTFIX] Implementing getMemorySize in
BlockletDataMapIndexWrapper
Repository: carbondata
Updated Branches:
refs/heads/master 9aa3a8c22 -> 16ed99a11
[HOTFIX] Implementing getMemorySize in BlockletDataMapIndexWrapper
This closes #2330
Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/16ed99a1
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/16ed99a1
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/16ed99a1
Branch: refs/heads/master
Commit: 16ed99a119f95b70a01c95120b20e632405d0a54
Parents: 9aa3a8c
Author: dhatchayani <dh...@gmail.com>
Authored: Tue May 22 12:33:24 2018 +0530
Committer: ravipesala <ra...@gmail.com>
Committed: Tue May 22 22:03:31 2018 +0530
----------------------------------------------------------------------
.../core/indexstore/BlockletDataMapIndexStore.java | 9 ++-------
.../core/indexstore/BlockletDataMapIndexWrapper.java | 10 +++++++++-
2 files changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/carbondata/blob/16ed99a1/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
index ba4193e..db49976 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java
@@ -78,7 +78,6 @@ public class BlockletDataMapIndexStore
try {
SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore();
Set<String> filesRead = new HashSet<>();
- long memorySize = 0L;
String segmentFilePath = identifier.getIndexFilePath();
Map<String, BlockMetaInfo> carbonDataFileBlockMetaInfoMapping = BlockletDataMapUtil
.createCarbonDataFileBlockMetaInfoMapping(segmentFilePath);
@@ -89,7 +88,6 @@ public class BlockletDataMapIndexStore
carbonDataFileBlockMetaInfoMapping);
BlockletDataMap blockletDataMap =
loadAndGetDataMap(identifier, indexFileStore, blockMetaInfoMap);
- memorySize += blockletDataMap.getMemorySize();
dataMaps.add(blockletDataMap);
blockletDataMapIndexWrapper = new BlockletDataMapIndexWrapper(dataMaps);
} else {
@@ -103,13 +101,12 @@ public class BlockletDataMapIndexStore
carbonDataFileBlockMetaInfoMapping);
BlockletDataMap blockletDataMap =
loadAndGetDataMap(blockIndexUniqueIdentifier, indexFileStore, blockMetaInfoMap);
- memorySize += blockletDataMap.getMemorySize();
dataMaps.add(blockletDataMap);
}
blockletDataMapIndexWrapper = new BlockletDataMapIndexWrapper(dataMaps);
}
lruCache.put(identifier.getUniqueTableSegmentIdentifier(), blockletDataMapIndexWrapper,
- memorySize);
+ blockletDataMapIndexWrapper.getMemorySize());
} catch (Throwable e) {
// clear all the memory used by datamaps loaded
for (DataMap dataMap : dataMaps) {
@@ -189,7 +186,6 @@ public class BlockletDataMapIndexStore
if (lock == null) {
lock = addAndGetSegmentLock(uniqueTableSegmentIdentifier);
}
- long memorySize = 0L;
// As dataMap will use unsafe memory, it is not recommended to overwrite an existing entry
// as in that case clearing unsafe memory need to be taken card. If at all datamap entry
// in the cache need to be overwritten then use the invalidate interface
@@ -201,10 +197,9 @@ public class BlockletDataMapIndexStore
try {
for (BlockletDataMap blockletDataMap: dataMaps) {
blockletDataMap.convertToUnsafeDMStore();
- memorySize += blockletDataMap.getMemorySize();
}
lruCache.put(tableBlockIndexUniqueIdentifier.getUniqueTableSegmentIdentifier(), wrapper,
- memorySize);
+ wrapper.getMemorySize());
} catch (Throwable e) {
// clear all the memory acquired by data map in case of any failure
for (DataMap blockletDataMap : dataMaps) {
http://git-wip-us.apache.org/repos/asf/carbondata/blob/16ed99a1/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexWrapper.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexWrapper.java
index d674cb4..ea85930 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexWrapper.java
@@ -30,8 +30,16 @@ public class BlockletDataMapIndexWrapper implements Cacheable, Serializable {
private List<BlockletDataMap> dataMaps;
+ // size of the wrapper. basically the total size of the datamaps this wrapper is holding
+ private long wrapperSize;
+
public BlockletDataMapIndexWrapper(List<BlockletDataMap> dataMaps) {
this.dataMaps = dataMaps;
+ this.wrapperSize = 0L;
+ // add the size of each and every datamap in this wrapper
+ for (BlockletDataMap dataMap : dataMaps) {
+ this.wrapperSize += dataMap.getMemorySize();
+ }
}
@Override public long getFileTimeStamp() {
@@ -43,7 +51,7 @@ public class BlockletDataMapIndexWrapper implements Cacheable, Serializable {
}
@Override public long getMemorySize() {
- return 0;
+ return wrapperSize;
}
public List<BlockletDataMap> getDataMaps() {