You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by op...@apache.org on 2019/01/21 09:50:44 UTC
[hbase] branch branch-2.1 updated: HBASE-21738 Remove all the
CLSM#size operation in our memstore because it's an quite time consuming.
This is an automated email from the ASF dual-hosted git repository.
openinx pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 51b04f6 HBASE-21738 Remove all the CLSM#size operation in our memstore because it's an quite time consuming.
51b04f6 is described below
commit 51b04f6f74e26215c92b57657fb41c474a0c5f02
Author: huzheng <op...@gmail.com>
AuthorDate: Sat Jan 19 16:54:17 2019 +0800
HBASE-21738 Remove all the CLSM#size operation in our memstore because it's an quite time consuming.
---
.../hbase/regionserver/AbstractMemStore.java | 4 +-
.../hbase/regionserver/CSLMImmutableSegment.java | 4 +-
.../regionserver/CellArrayImmutableSegment.java | 14 +++---
.../regionserver/CellChunkImmutableSegment.java | 26 +++++------
.../apache/hadoop/hbase/regionserver/CellSet.java | 3 ++
.../hbase/regionserver/CompactionPipeline.java | 50 +++++++++++++---------
.../regionserver/CompositeImmutableSegment.java | 2 +-
.../hadoop/hbase/regionserver/DefaultMemStore.java | 4 +-
.../apache/hadoop/hbase/regionserver/HRegion.java | 24 ++++++-----
.../hadoop/hbase/regionserver/MemStoreSize.java | 26 +++++++----
.../hadoop/hbase/regionserver/MemStoreSizing.java | 27 ++++++++----
.../hadoop/hbase/regionserver/MutableSegment.java | 8 ++--
.../regionserver/NonThreadSafeMemStoreSizing.java | 21 ++++++---
.../regionserver/RegionServicesForStores.java | 5 ++-
.../apache/hadoop/hbase/regionserver/Segment.java | 29 +++++++------
.../regionserver/ThreadSafeMemStoreSizing.java | 21 ++++++---
.../hbase/regionserver/TestCellSkipListSet.java | 8 ++--
.../hbase/regionserver/TestCompactingMemStore.java | 13 ++++--
.../TestCompactingToCellFlatMapMemStore.java | 6 +--
.../hadoop/hbase/regionserver/TestHStore.java | 8 ++--
.../regionserver/TestRegionServerAccounting.java | 31 +++++---------
21 files changed, 194 insertions(+), 140 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
index de3f114..b0b141c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
@@ -93,8 +93,8 @@ public abstract class AbstractMemStore implements MemStore {
// regionServices can be null when testing
if (regionServices != null) {
regionServices.addMemStoreSize(memstoreAccounting.getDataSize(),
- memstoreAccounting.getHeapSize(),
- memstoreAccounting.getOffHeapSize());
+ memstoreAccounting.getHeapSize(), memstoreAccounting.getOffHeapSize(),
+ memstoreAccounting.getCellsCount());
}
this.timeOfOldestEdit = Long.MAX_VALUE;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CSLMImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CSLMImmutableSegment.java
index 855fd08..52ff44a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CSLMImmutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CSLMImmutableSegment.java
@@ -40,9 +40,9 @@ public class CSLMImmutableSegment extends ImmutableSegment {
super(segment);
// update the segment metadata heap size
long indexOverhead = -MutableSegment.DEEP_OVERHEAD + DEEP_OVERHEAD_CSLM;
- incMemStoreSize(0, indexOverhead, 0); // CSLM is always on-heap
+ incMemStoreSize(0, indexOverhead, 0, 0); // CSLM is always on-heap
if (memstoreSizing != null) {
- memstoreSizing.incMemStoreSize(0, indexOverhead, 0);
+ memstoreSizing.incMemStoreSize(0, indexOverhead, 0, 0);
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java
index dadfc48..2608976 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java
@@ -45,7 +45,7 @@ public class CellArrayImmutableSegment extends ImmutableSegment {
protected CellArrayImmutableSegment(CellComparator comparator, MemStoreSegmentsIterator iterator,
MemStoreLAB memStoreLAB, int numOfCells, MemStoreCompactionStrategy.Action action) {
super(null, comparator, memStoreLAB); // initiailize the CellSet with NULL
- incMemStoreSize(0, DEEP_OVERHEAD_CAM, 0); // CAM is always on-heap
+ incMemStoreSize(0, DEEP_OVERHEAD_CAM, 0, 0); // CAM is always on-heap
// build the new CellSet based on CellArrayMap and update the CellSet of the new Segment
initializeCellSet(numOfCells, iterator, action);
}
@@ -59,16 +59,18 @@ public class CellArrayImmutableSegment extends ImmutableSegment {
MemStoreCompactionStrategy.Action action) {
super(segment); // initiailize the upper class
long indexOverhead = DEEP_OVERHEAD_CAM - CSLMImmutableSegment.DEEP_OVERHEAD_CSLM;
- incMemStoreSize(0, indexOverhead, 0); // CAM is always on-heap
+ incMemStoreSize(0, indexOverhead, 0, 0); // CAM is always on-heap
+ memstoreSizing.incMemStoreSize(0, indexOverhead, 0, 0);
int numOfCells = segment.getCellsCount();
// build the new CellSet based on CellChunkMap and update the CellSet of this Segment
reinitializeCellSet(numOfCells, segment.getScanner(Long.MAX_VALUE), segment.getCellSet(),
- action);
+ action);
// arrange the meta-data size, decrease all meta-data sizes related to SkipList;
// add sizes of CellArrayMap entry (reinitializeCellSet doesn't take the care for the sizes)
- long newSegmentSizeDelta = numOfCells*(indexEntrySize()-ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY);
- incMemStoreSize(0, newSegmentSizeDelta, 0);
- memstoreSizing.incMemStoreSize(0, newSegmentSizeDelta, 0);
+ long newSegmentSizeDelta =
+ numOfCells * (indexEntrySize() - ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY);
+ incMemStoreSize(0, newSegmentSizeDelta, 0, 0);
+ memstoreSizing.incMemStoreSize(0, newSegmentSizeDelta, 0, 0);
}
@Override
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
index e2f8205..9653e75 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java
@@ -57,10 +57,10 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
// memStoreLAB cannot be null in this class
boolean onHeap = getMemStoreLAB().isOnHeap();
// initiate the heapSize with the size of the segment metadata
- if(onHeap) {
- incMemStoreSize(0, indexOverhead, 0);
+ if (onHeap) {
+ incMemStoreSize(0, indexOverhead, 0, 0);
} else {
- incMemStoreSize(0, 0, indexOverhead);
+ incMemStoreSize(0, 0, indexOverhead, 0);
}
// build the new CellSet based on CellArrayMap and update the CellSet of the new Segment
initializeCellSet(numOfCells, iterator, action);
@@ -79,9 +79,12 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
boolean onHeap = getMemStoreLAB().isOnHeap();
// initiate the heapSize with the size of the segment metadata
if(onHeap) {
- incMemStoreSize(0, indexOverhead, 0);
+ incMemStoreSize(0, indexOverhead, 0, 0);
+ memstoreSizing.incMemStoreSize(0, indexOverhead, 0, 0);
} else {
- incMemStoreSize(0, -CSLMImmutableSegment.DEEP_OVERHEAD_CSLM, DEEP_OVERHEAD_CCM);
+ incMemStoreSize(0, -CSLMImmutableSegment.DEEP_OVERHEAD_CSLM, DEEP_OVERHEAD_CCM, 0);
+ memstoreSizing.incMemStoreSize(0, -CSLMImmutableSegment.DEEP_OVERHEAD_CSLM, DEEP_OVERHEAD_CCM,
+ 0);
}
int numOfCells = segment.getCellsCount();
// build the new CellSet based on CellChunkMap
@@ -92,12 +95,11 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
// (reinitializeCellSet doesn't take the care for the sizes)
long newSegmentSizeDelta = numOfCells*(indexEntrySize()-ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY);
if(onHeap) {
- incMemStoreSize(0, newSegmentSizeDelta, 0);
- memstoreSizing.incMemStoreSize(0, newSegmentSizeDelta, 0);
+ incMemStoreSize(0, newSegmentSizeDelta, 0, 0);
+ memstoreSizing.incMemStoreSize(0, newSegmentSizeDelta, 0, 0);
} else {
- incMemStoreSize(0, 0, newSegmentSizeDelta);
- memstoreSizing.incMemStoreSize(0, 0, newSegmentSizeDelta);
-
+ incMemStoreSize(0, 0, newSegmentSizeDelta, 0);
+ memstoreSizing.incMemStoreSize(0, 0, newSegmentSizeDelta, 0);
}
}
@@ -332,8 +334,8 @@ public class CellChunkImmutableSegment extends ImmutableSegment {
long newCellSize = getCellLength(cell);
long heapOverhead = newHeapSize - oldHeapSize;
long offHeapOverhead = newOffHeapSize - oldOffHeapSize;
- //TODO: maybe need to update the dataSize of the region
- incMemStoreSize(newCellSize - oldCellSize, heapOverhead, offHeapOverhead);
+ // TODO: maybe need to update the dataSize of the region
+ incMemStoreSize(newCellSize - oldCellSize, heapOverhead, offHeapOverhead, 0);
return cell;
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSet.java
index 94a256d..5190d7b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSet.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellSet.java
@@ -217,6 +217,9 @@ public class CellSet implements NavigableSet<Cell> {
@Override
public int size() {
+ if (delegatee instanceof ConcurrentSkipListMap) {
+ throw new UnsupportedOperationException("ConcurrentSkipListMap.size() is time-consuming");
+ }
return this.delegatee.size();
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
index 5afaec1..cdec051 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java
@@ -78,9 +78,8 @@ public class CompactionPipeline {
ImmutableSegment immutableSegment = SegmentFactory.instance().
createImmutableSegment(segment, memstoreAccounting);
if (region != null) {
- region.addMemStoreSize(memstoreAccounting.getDataSize(),
- memstoreAccounting.getHeapSize(),
- memstoreAccounting.getOffHeapSize());
+ region.addMemStoreSize(memstoreAccounting.getDataSize(), memstoreAccounting.getHeapSize(),
+ memstoreAccounting.getOffHeapSize(), memstoreAccounting.getCellsCount());
}
synchronized (pipeline){
boolean res = addFirst(immutableSegment);
@@ -142,31 +141,31 @@ public class CompactionPipeline {
if (updateRegionSize && region != null) {
// update the global memstore size counter
long suffixDataSize = getSegmentsKeySize(suffix);
- long newDataSize = 0;
- if(segment != null) {
- newDataSize = segment.getDataSize();
- }
- long dataSizeDelta = suffixDataSize - newDataSize;
long suffixHeapSize = getSegmentsHeapSize(suffix);
long suffixOffHeapSize = getSegmentsOffHeapSize(suffix);
+ int suffixCellsCount = getSegmentsCellsCount(suffix);
+ long newDataSize = 0;
long newHeapSize = 0;
long newOffHeapSize = 0;
- if(segment != null) {
+ int newCellsCount = 0;
+ if (segment != null) {
+ newDataSize = segment.getDataSize();
newHeapSize = segment.getHeapSize();
newOffHeapSize = segment.getOffHeapSize();
+ newCellsCount = segment.getCellsCount();
}
+ long dataSizeDelta = suffixDataSize - newDataSize;
long offHeapSizeDelta = suffixOffHeapSize - newOffHeapSize;
long heapSizeDelta = suffixHeapSize - newHeapSize;
- region.addMemStoreSize(-dataSizeDelta, -heapSizeDelta, -offHeapSizeDelta);
- LOG.debug("Suffix data size={}, new segment data size={}, "
- + "suffix heap size={}," + "new segment heap size={}"
- + "suffix off heap size={}," + "new segment off heap size={}"
- , suffixDataSize
- , newDataSize
- , suffixHeapSize
- , newHeapSize
- , suffixOffHeapSize
- , newOffHeapSize);
+ int cellsCountDelta = suffixCellsCount - newCellsCount;
+ region.addMemStoreSize(-dataSizeDelta, -heapSizeDelta, -offHeapSizeDelta, -cellsCountDelta);
+ LOG.debug(
+ "Suffix data size={}, new segment data size={}, "
+ + "suffix heap size={}, new segment heap size={}"
+ + "suffix off heap size={}, new segment off heap size={}"
+ + "suffix cells count={}, new cells count={}",
+ suffixDataSize, newDataSize, suffixHeapSize, newHeapSize, suffixOffHeapSize, newOffHeapSize,
+ suffixCellsCount, newCellsCount);
}
return true;
}
@@ -195,6 +194,14 @@ public class CompactionPipeline {
return res;
}
+ private static int getSegmentsCellsCount(List<? extends Segment> list) {
+ int res = 0;
+ for (Segment segment : list) {
+ res += segment.getCellsCount();
+ }
+ return res;
+ }
+
/**
* If the caller holds the current version, go over the the pipeline and try to flatten each
* segment. Flattening is replacing the ConcurrentSkipListMap based CellSet to CellArrayMap based.
@@ -226,12 +233,13 @@ public class CompactionPipeline {
ImmutableSegment newS = SegmentFactory.instance().createImmutableSegmentByFlattening(
(CSLMImmutableSegment)s,idxType,newMemstoreAccounting,action);
replaceAtIndex(i,newS);
- if(region != null) {
+ if (region != null) {
// Update the global memstore size counter upon flattening there is no change in the
// data size
MemStoreSize mss = newMemstoreAccounting.getMemStoreSize();
Preconditions.checkArgument(mss.getDataSize() == 0, "Not zero!");
- region.addMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize());
+ region.addMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(),
+ mss.getCellsCount());
}
LOG.debug("Compaction pipeline segment {} flattened", s);
return true;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java
index dcfaf81..3dcfa55 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java
@@ -190,7 +190,7 @@ public class CompositeImmutableSegment extends ImmutableSegment {
* Updates the heap size counter of the segment by the given delta
*/
@Override
- public long incMemStoreSize(long delta, long heapOverhead, long offHeapOverhead) {
+ public long incMemStoreSize(long delta, long heapOverhead, long offHeapOverhead, int cellsCount) {
throw new IllegalStateException("Not supported by CompositeImmutableScanner");
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
index bd6c8ac..a4f14d8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
@@ -104,8 +104,8 @@ public class DefaultMemStore extends AbstractMemStore {
// regionServices can be null when testing
if (regionServices != null) {
regionServices.addMemStoreSize(memstoreAccounting.getDataSize(),
- memstoreAccounting.getHeapSize(),
- memstoreAccounting.getOffHeapSize());
+ memstoreAccounting.getHeapSize(), memstoreAccounting.getOffHeapSize(),
+ memstoreAccounting.getCellsCount());
}
this.snapshot = immutableSegment;
resetActive();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index afa7b84..3da0bcd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -1238,28 +1238,32 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
* store
*/
void incMemStoreSize(MemStoreSize mss) {
- incMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize());
+ incMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(),
+ mss.getCellsCount());
}
- void incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+ void incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta,
+ int cellsCountDelta) {
if (this.rsAccounting != null) {
rsAccounting.incGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta);
}
- long dataSize =
- this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta);
+ long dataSize = this.memStoreSizing.incMemStoreSize(dataSizeDelta, heapSizeDelta,
+ offHeapSizeDelta, cellsCountDelta);
checkNegativeMemStoreDataSize(dataSize, dataSizeDelta);
}
void decrMemStoreSize(MemStoreSize mss) {
- decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize());
+ decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(),
+ mss.getCellsCount());
}
- void decrMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+ void decrMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta,
+ int cellsCountDelta) {
if (this.rsAccounting != null) {
rsAccounting.decGlobalMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta);
}
- long dataSize =
- this.memStoreSizing.decMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta);
+ long dataSize = this.memStoreSizing.decMemStoreSize(dataSizeDelta, heapSizeDelta,
+ offHeapSizeDelta, cellsCountDelta);
checkNegativeMemStoreDataSize(dataSize, -dataSizeDelta);
}
@@ -2737,7 +2741,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// Set down the memstore size by amount of flush.
MemStoreSize mss = prepareResult.totalFlushableSize.getMemStoreSize();
- this.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize());
+ this.decrMemStoreSize(mss);
if (wal != null) {
// write flush marker to WAL. If fail, we should throw DroppedSnapshotException
@@ -3153,7 +3157,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
});
// update memStore size
region.incMemStoreSize(memStoreAccounting.getDataSize(), memStoreAccounting.getHeapSize(),
- memStoreAccounting.getOffHeapSize());
+ memStoreAccounting.getOffHeapSize(), memStoreAccounting.getCellsCount());
}
public boolean isDone() {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java
index 97a416e..86cf810 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSize.java
@@ -44,20 +44,23 @@ public class MemStoreSize {
*/
private final long offHeapSize;
+ private final int cellsCount;
+
/**
* Package private constructor.
*/
MemStoreSize() {
- this(0L, 0L, 0L);
+ this(0L, 0L, 0L, 0);
}
/**
* Package private constructor.
*/
- MemStoreSize(long dataSize, long heapSize, long offHeapSize) {
+ MemStoreSize(long dataSize, long heapSize, long offHeapSize, int cellsCount) {
this.dataSize = dataSize;
this.heapSize = heapSize;
this.offHeapSize = offHeapSize;
+ this.cellsCount = cellsCount;
}
/**
@@ -67,10 +70,12 @@ public class MemStoreSize {
this.dataSize = memStoreSize.getDataSize();
this.heapSize = memStoreSize.getHeapSize();
this.offHeapSize = memStoreSize.getOffHeapSize();
+ this.cellsCount = memStoreSize.getCellsCount();
}
public boolean isEmpty() {
- return this.dataSize == 0 && this.heapSize == 0 && this.offHeapSize == 0;
+ return this.dataSize == 0 && this.heapSize == 0 && this.offHeapSize == 0
+ && this.cellsCount == 0;
}
public long getDataSize() {
@@ -85,6 +90,10 @@ public class MemStoreSize {
return this.offHeapSize;
}
+ public int getCellsCount() {
+ return this.cellsCount;
+ }
+
@Override
public boolean equals(Object obj) {
if (obj == null) {
@@ -93,9 +102,9 @@ public class MemStoreSize {
if (!(obj instanceof MemStoreSize)) {
return false;
}
- MemStoreSize other = (MemStoreSize)obj;
- return this.dataSize == other.dataSize && this.heapSize == other.heapSize &&
- this.offHeapSize == other.offHeapSize;
+ MemStoreSize other = (MemStoreSize) obj;
+ return this.dataSize == other.dataSize && this.heapSize == other.heapSize
+ && this.offHeapSize == other.offHeapSize && this.cellsCount == other.cellsCount;
}
@Override
@@ -103,12 +112,13 @@ public class MemStoreSize {
long h = 31 * this.dataSize;
h = h + 31 * this.heapSize;
h = h + 31 * this.offHeapSize;
+ h = h + 31 * this.cellsCount;
return (int) h;
}
@Override
public String toString() {
- return "dataSize=" + this.dataSize + ", getHeapSize=" + this.heapSize +
- ", getOffHeapSize=" + this.offHeapSize;
+ return "dataSize=" + this.dataSize + ", getHeapSize=" + this.heapSize + ", getOffHeapSize="
+ + this.offHeapSize + ", getCellsCount=" + this.cellsCount;
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java
index 8430ac6..87297f6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSizing.java
@@ -53,7 +53,7 @@ import org.apache.yetus.audience.InterfaceAudience;
*/
@InterfaceAudience.Private
public interface MemStoreSizing {
- static final MemStoreSizing DUD = new MemStoreSizing() {
+ MemStoreSizing DUD = new MemStoreSizing() {
private final MemStoreSize mss = new MemStoreSize();
@Override
@@ -77,8 +77,13 @@ public interface MemStoreSizing {
}
@Override
- public long incMemStoreSize(long dataSizeDelta, long heapSizeDelta,
- long offHeapSizeDelta) {
+ public int getCellsCount() {
+ return this.mss.getCellsCount();
+ }
+
+ @Override
+ public long incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta,
+ int cellsCountDelta) {
throw new RuntimeException("I'm a DUD, you can't use me!");
}
};
@@ -86,31 +91,35 @@ public interface MemStoreSizing {
/**
* @return The new dataSize ONLY as a convenience
*/
- long incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta);
+ long incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta,
+ int cellsCount);
default long incMemStoreSize(MemStoreSize delta) {
- return incMemStoreSize(delta.getDataSize(), delta.getHeapSize(), delta.getOffHeapSize());
+ return incMemStoreSize(delta.getDataSize(), delta.getHeapSize(), delta.getOffHeapSize(),
+ delta.getCellsCount());
}
/**
* @return The new dataSize ONLY as a convenience
*/
default long decMemStoreSize(long dataSizeDelta, long heapSizeDelta,
- long offHeapSizeDelta) {
- return incMemStoreSize(-dataSizeDelta, -heapSizeDelta, -offHeapSizeDelta);
+ long offHeapSizeDelta, int cellsCountDelta) {
+ return incMemStoreSize(-dataSizeDelta, -heapSizeDelta, -offHeapSizeDelta, -cellsCountDelta);
}
default long decMemStoreSize(MemStoreSize delta) {
- return incMemStoreSize(-delta.getDataSize(), -delta.getHeapSize(), -delta.getOffHeapSize());
+ return incMemStoreSize(-delta.getDataSize(), -delta.getHeapSize(), -delta.getOffHeapSize(),
+ -delta.getCellsCount());
}
long getDataSize();
long getHeapSize();
long getOffHeapSize();
+ int getCellsCount();
/**
* @return Use this datastructure to return all three settings, {@link #getDataSize()},
* {@link #getHeapSize()}, and {@link #getOffHeapSize()}, in the one go.
*/
MemStoreSize getMemStoreSize();
-}
\ No newline at end of file
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
index 714e9bc..a5cdda9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
@@ -45,9 +45,9 @@ public class MutableSegment extends Segment {
protected MutableSegment(CellSet cellSet, CellComparator comparator,
MemStoreLAB memStoreLAB, MemStoreSizing memstoreSizing) {
super(cellSet, comparator, memStoreLAB, TimeRangeTracker.create(TimeRangeTracker.Type.SYNC));
- incMemStoreSize(0, DEEP_OVERHEAD, 0); // update the mutable segment metadata
+ incMemStoreSize(0, DEEP_OVERHEAD, 0, 0); // update the mutable segment metadata
if (memstoreSizing != null) {
- memstoreSizing.incMemStoreSize(0, DEEP_OVERHEAD, 0);
+ memstoreSizing.incMemStoreSize(0, DEEP_OVERHEAD, 0, 0);
}
}
@@ -93,9 +93,9 @@ public class MutableSegment extends Segment {
int cellLen = getCellLength(cur);
long heapSize = heapSizeChange(cur, true);
long offHeapSize = offHeapSizeChange(cur, true);
- incMemStoreSize(-cellLen, -heapSize, -offHeapSize);
+ incMemStoreSize(-cellLen, -heapSize, -offHeapSize, -1);
if (memStoreSizing != null) {
- memStoreSizing.decMemStoreSize(cellLen, heapSize, offHeapSize);
+ memStoreSizing.decMemStoreSize(cellLen, heapSize, offHeapSize, 1);
}
it.remove();
} else {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonThreadSafeMemStoreSizing.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonThreadSafeMemStoreSizing.java
index 601ff33..ba5db0a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonThreadSafeMemStoreSizing.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/NonThreadSafeMemStoreSizing.java
@@ -32,30 +32,32 @@ class NonThreadSafeMemStoreSizing implements MemStoreSizing {
private long dataSize = 0;
private long heapSize = 0;
private long offHeapSize = 0;
+ private int cellsCount = 0;
NonThreadSafeMemStoreSizing() {
- this(0, 0, 0);
+ this(0, 0, 0, 0);
}
NonThreadSafeMemStoreSizing(MemStoreSize mss) {
- this(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize());
+ this(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount());
}
- NonThreadSafeMemStoreSizing(long dataSize, long heapSize, long offHeapSize) {
- incMemStoreSize(dataSize, heapSize, offHeapSize);
+ NonThreadSafeMemStoreSizing(long dataSize, long heapSize, long offHeapSize, int cellsCount) {
+ incMemStoreSize(dataSize, heapSize, offHeapSize, cellsCount);
}
@Override
public MemStoreSize getMemStoreSize() {
- return new MemStoreSize(this.dataSize, this.heapSize, this.offHeapSize);
+ return new MemStoreSize(this.dataSize, this.heapSize, this.offHeapSize, this.cellsCount);
}
@Override
public long incMemStoreSize(long dataSizeDelta, long heapSizeDelta,
- long offHeapSizeDelta) {
+ long offHeapSizeDelta, int cellsCountDelta) {
this.offHeapSize += offHeapSizeDelta;
this.heapSize += heapSizeDelta;
this.dataSize += dataSizeDelta;
+ this.cellsCount += cellsCountDelta;
return this.dataSize;
}
@@ -75,7 +77,12 @@ class NonThreadSafeMemStoreSizing implements MemStoreSizing {
}
@Override
+ public int getCellsCount() {
+ return cellsCount;
+ }
+
+ @Override
public String toString() {
return getMemStoreSize().toString();
}
-}
\ No newline at end of file
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServicesForStores.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServicesForStores.java
index b088856..c1af9db 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServicesForStores.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServicesForStores.java
@@ -65,8 +65,9 @@ public class RegionServicesForStores {
region.unblockUpdates();
}
- public void addMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
- region.incMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta);
+ public void addMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta,
+ int cellsCountDelta) {
+ region.incMemStoreSize(dataSizeDelta, heapSizeDelta, offHeapSizeDelta, cellsCountDelta);
}
public RegionInfo getRegionInfo() {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
index 1f414a7..bf2f9de 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
@@ -80,21 +80,24 @@ public abstract class Segment implements MemStoreSizing {
long dataSize = 0;
long heapSize = 0;
long OffHeapSize = 0;
+ int cellsCount = 0;
for (Segment segment : segments) {
MemStoreSize memStoreSize = segment.getMemStoreSize();
dataSize += memStoreSize.getDataSize();
heapSize += memStoreSize.getHeapSize();
OffHeapSize += memStoreSize.getOffHeapSize();
+ cellsCount += memStoreSize.getCellsCount();
}
this.comparator = comparator;
// Do we need to be thread safe always? What if ImmutableSegment?
// DITTO for the TimeRangeTracker below.
- this.memStoreSizing = new ThreadSafeMemStoreSizing(dataSize, heapSize, OffHeapSize);
+ this.memStoreSizing = new ThreadSafeMemStoreSizing(dataSize, heapSize, OffHeapSize, cellsCount);
this.timeRangeTracker = trt;
}
// This constructor is used to create empty Segments.
- protected Segment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB, TimeRangeTracker trt) {
+ protected Segment(CellSet cellSet, CellComparator comparator, MemStoreLAB memStoreLAB,
+ TimeRangeTracker trt) {
this.cellSet.set(cellSet);
this.comparator = comparator;
this.minSequenceId = Long.MAX_VALUE;
@@ -135,12 +138,6 @@ public abstract class Segment implements MemStoreSizing {
return getCellSet().isEmpty();
}
- /**
- * @return number of cells in segment
- */
- public int getCellsCount() {
- return getCellSet().size();
- }
/**
* Closing a segment before it is being discarded
@@ -169,7 +166,7 @@ public abstract class Segment implements MemStoreSizing {
return cell;
}
- Cell cellFromMslab = null;
+ Cell cellFromMslab;
if (forceCloneOfBigCell) {
cellFromMslab = this.memStoreLAB.forceCopyOfBigCellInto(cell);
} else {
@@ -240,8 +237,13 @@ public abstract class Segment implements MemStoreSizing {
}
@Override
- public long incMemStoreSize(long delta, long heapOverhead, long offHeapOverhead) {
- return this.memStoreSizing.incMemStoreSize(delta, heapOverhead, offHeapOverhead);
+ public int getCellsCount() {
+ return this.memStoreSizing.getCellsCount();
+ }
+
+ @Override
+ public long incMemStoreSize(long delta, long heapOverhead, long offHeapOverhead, int cellsCount) {
+ return this.memStoreSizing.incMemStoreSize(delta, heapOverhead, offHeapOverhead, cellsCount);
}
public long getMinSequenceId() {
@@ -296,6 +298,7 @@ public abstract class Segment implements MemStoreSizing {
protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed,
MemStoreSizing memstoreSizing) {
long cellSize = 0;
+ int cellsCount = succ ? 1 : 0;
// If there's already a same cell in the CellSet and we are using MSLAB, we must count in the
// MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger
// than the counted number)
@@ -308,9 +311,9 @@ public abstract class Segment implements MemStoreSizing {
// is needed.
long heapSize = heapSizeChange(cellToAdd, sizeChanged);
long offHeapSize = offHeapSizeChange(cellToAdd, sizeChanged);
- incMemStoreSize(cellSize, heapSize, offHeapSize);
+ incMemStoreSize(cellSize, heapSize, offHeapSize, cellsCount);
if (memstoreSizing != null) {
- memstoreSizing.incMemStoreSize(cellSize, heapSize, offHeapSize);
+ memstoreSizing.incMemStoreSize(cellSize, heapSize, offHeapSize, cellsCount);
}
getTimeRangeTracker().includeTimestamp(cellToAdd);
minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId());
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ThreadSafeMemStoreSizing.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ThreadSafeMemStoreSizing.java
index de05493..3dec1d6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ThreadSafeMemStoreSizing.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ThreadSafeMemStoreSizing.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.yetus.audience.InterfaceAudience;
@@ -35,26 +36,29 @@ class ThreadSafeMemStoreSizing implements MemStoreSizing {
private final AtomicLong dataSize = new AtomicLong();
private final AtomicLong heapSize = new AtomicLong();
private final AtomicLong offHeapSize = new AtomicLong();
+ private final AtomicInteger cellsCount = new AtomicInteger();
ThreadSafeMemStoreSizing() {
- this(0, 0, 0);
+ this(0, 0, 0, 0);
}
ThreadSafeMemStoreSizing(MemStoreSize mss) {
- this(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize());
+ this(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(), mss.getCellsCount());
}
- ThreadSafeMemStoreSizing(long dataSize, long heapSize, long offHeapSize) {
- incMemStoreSize(dataSize, heapSize, offHeapSize);
+ ThreadSafeMemStoreSizing(long dataSize, long heapSize, long offHeapSize, int cellsCount) {
+ incMemStoreSize(dataSize, heapSize, offHeapSize, cellsCount);
}
public MemStoreSize getMemStoreSize() {
- return new MemStoreSize(getDataSize(), getHeapSize(), getOffHeapSize());
+ return new MemStoreSize(getDataSize(), getHeapSize(), getOffHeapSize(), getCellsCount());
}
- public long incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta) {
+ public long incMemStoreSize(long dataSizeDelta, long heapSizeDelta, long offHeapSizeDelta,
+ int cellsCount) {
this.offHeapSize.addAndGet(offHeapSizeDelta);
this.heapSize.addAndGet(heapSizeDelta);
+ this.cellsCount.addAndGet(cellsCount);
return this.dataSize.addAndGet(dataSizeDelta);
}
@@ -74,6 +78,11 @@ class ThreadSafeMemStoreSizing implements MemStoreSizing {
}
@Override
+ public int getCellsCount() {
+ return cellsCount.get();
+ }
+
+ @Override
public String toString() {
return getMemStoreSize().toString();
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java
index 4f46bcf..c2e2ca1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCellSkipListSet.java
@@ -48,20 +48,20 @@ public class TestCellSkipListSet extends TestCase {
}
public void testAdd() throws Exception {
- byte [] bytes = Bytes.toBytes(getName());
+ byte[] bytes = Bytes.toBytes(getName());
KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
this.csls.add(kv);
assertTrue(this.csls.contains(kv));
- assertEquals(1, this.csls.size());
+ assertEquals(1, this.csls.getDelegatee().size());
Cell first = this.csls.first();
assertTrue(kv.equals(first));
assertTrue(Bytes.equals(kv.getValueArray(), kv.getValueOffset(), kv.getValueLength(),
first.getValueArray(), first.getValueOffset(), first.getValueLength()));
// Now try overwritting
- byte [] overwriteValue = Bytes.toBytes("overwrite");
+ byte[] overwriteValue = Bytes.toBytes("overwrite");
KeyValue overwrite = new KeyValue(bytes, bytes, bytes, overwriteValue);
this.csls.add(overwrite);
- assertEquals(1, this.csls.size());
+ assertEquals(1, this.csls.getDelegatee().size());
first = this.csls.first();
assertTrue(Bytes.equals(overwrite.getValueArray(), overwrite.getValueOffset(),
overwrite.getValueLength(), first.getValueArray(), first.getValueOffset(),
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
index 8dbddb9..eef91d2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
@@ -679,7 +679,8 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
mss = memstore.getFlushableSize();
MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot
// simulate flusher
- region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize());
+ region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(),
+ mss.getCellsCount());
ImmutableSegment s = memstore.getSnapshot();
assertEquals(7, s.getCellsCount());
assertEquals(0, regionServicesForStores.getMemStoreSize());
@@ -756,7 +757,8 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
mss = memstore.getFlushableSize();
MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot
// simulate flusher
- region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize());
+ region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(),
+ mss.getCellsCount());
ImmutableSegment s = memstore.getSnapshot();
assertEquals(4, s.getCellsCount());
assertEquals(0, regionServicesForStores.getMemStoreSize());
@@ -831,6 +833,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
byte[] qf = Bytes.toBytes("testqualifier");
long size = hmc.getActive().getDataSize();
long heapOverhead = hmc.getActive().getHeapSize();
+ int cellsCount = hmc.getActive().getCellsCount();
int totalLen = 0;
for (int i = 0; i < keys.length; i++) {
long timestamp = System.currentTimeMillis();
@@ -843,7 +846,8 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp());
}
regionServicesForStores.addMemStoreSize(hmc.getActive().getDataSize() - size,
- hmc.getActive().getHeapSize() - heapOverhead, 0);
+ hmc.getActive().getHeapSize() - heapOverhead, 0,
+ hmc.getActive().getCellsCount() - cellsCount);
return totalLen;
}
@@ -853,6 +857,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
byte[] qf = Bytes.toBytes("testqualifier");
long size = hmc.getActive().getDataSize();
long heapOverhead = hmc.getActive().getHeapSize();
+ int cellsCount = hmc.getActive().getCellsCount();
int totalLen = 0;
for (int i = 0; i < keys.length; i++) {
long timestamp = System.currentTimeMillis();
@@ -864,7 +869,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp());
}
regionServicesForStores.addMemStoreSize(hmc.getActive().getDataSize() - size,
- hmc.getActive().getHeapSize() - heapOverhead, 0);
+ hmc.getActive().getHeapSize() - heapOverhead, 0, cellsCount);
return totalLen;
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
index 70d6970..9ba3613 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
@@ -283,13 +283,13 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
mss = memstore.getFlushableSize();
MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot
// simulate flusher
- region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize());
+ region.decrMemStoreSize(mss.getDataSize(), mss.getHeapSize(), mss.getOffHeapSize(),
+ mss.getCellsCount());
ImmutableSegment s = memstore.getSnapshot();
assertEquals(4, s.getCellsCount());
assertEquals(0, regionServicesForStores.getMemStoreSize());
memstore.clearSnapshot(snapshot.getId());
-
}
//////////////////////////////////////////////////////////////////////////////
@@ -914,7 +914,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
}
MemStoreSize mss = memstoreSizing.getMemStoreSize();
regionServicesForStores.addMemStoreSize(mss.getDataSize(), mss.getHeapSize(),
- mss.getOffHeapSize());
+ mss.getOffHeapSize(), mss.getCellsCount());
return mss.getDataSize();
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
index 1b35b95..46ba464 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHStore.java
@@ -264,7 +264,7 @@ public class TestHStore {
MemStoreSizing kvSize = new NonThreadSafeMemStoreSizing();
store.add(new KeyValue(row, family, qf1, 1, (byte[]) null), kvSize);
// add the heap size of active (mutable) segment
- kvSize.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0);
+ kvSize.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0, 0);
mss = store.memstore.getFlushableSize();
assertEquals(kvSize.getMemStoreSize(), mss);
// Flush. Bug #1 from HBASE-10466. Make sure size calculation on failed flush is right.
@@ -277,12 +277,12 @@ public class TestHStore {
}
// due to snapshot, change mutable to immutable segment
kvSize.incMemStoreSize(0,
- CSLMImmutableSegment.DEEP_OVERHEAD_CSLM-MutableSegment.DEEP_OVERHEAD, 0);
+ CSLMImmutableSegment.DEEP_OVERHEAD_CSLM - MutableSegment.DEEP_OVERHEAD, 0, 0);
mss = store.memstore.getFlushableSize();
assertEquals(kvSize.getMemStoreSize(), mss);
MemStoreSizing kvSize2 = new NonThreadSafeMemStoreSizing();
- store.add(new KeyValue(row, family, qf2, 2, (byte[])null), kvSize2);
- kvSize2.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0);
+ store.add(new KeyValue(row, family, qf2, 2, (byte[]) null), kvSize2);
+ kvSize2.incMemStoreSize(0, MutableSegment.DEEP_OVERHEAD, 0, 0);
// Even though we add a new kv, we expect the flushable size to be 'same' since we have
// not yet cleared the snapshot -- the above flush failed.
assertEquals(kvSize.getMemStoreSize(), mss);
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAccounting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAccounting.java
index fb246d5..bf4bca0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAccounting.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerAccounting.java
@@ -50,22 +50,18 @@ public class TestRegionServerAccounting {
public void testOnheapMemstoreHigherWaterMarkLimits() {
RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf);
long dataSize = regionServerAccounting.getGlobalMemStoreLimit();
- MemStoreSize memstoreSize =
- new MemStoreSize(dataSize, dataSize, 0);
+ MemStoreSize memstoreSize = new MemStoreSize(dataSize, dataSize, 0, 0);
regionServerAccounting.incGlobalMemStoreSize(memstoreSize);
- assertEquals(FlushType.ABOVE_ONHEAP_HIGHER_MARK,
- regionServerAccounting.isAboveHighWaterMark());
+ assertEquals(FlushType.ABOVE_ONHEAP_HIGHER_MARK, regionServerAccounting.isAboveHighWaterMark());
}
@Test
public void testOnheapMemstoreLowerWaterMarkLimits() {
RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf);
long dataSize = regionServerAccounting.getGlobalMemStoreLimit();
- MemStoreSize memstoreSize =
- new MemStoreSize(dataSize, dataSize, 0);
+ MemStoreSize memstoreSize = new MemStoreSize(dataSize, dataSize, 0, 0);
regionServerAccounting.incGlobalMemStoreSize(memstoreSize);
- assertEquals(FlushType.ABOVE_ONHEAP_LOWER_MARK,
- regionServerAccounting.isAboveLowWaterMark());
+ assertEquals(FlushType.ABOVE_ONHEAP_LOWER_MARK, regionServerAccounting.isAboveLowWaterMark());
}
@Test
@@ -76,7 +72,7 @@ public class TestRegionServerAccounting {
RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf);
// this will breach offheap limit as data size is higher and not due to heap size
MemStoreSize memstoreSize =
- new MemStoreSize((3L * 1024L * 1024L * 1024L), 0, (1L * 1024L * 1024L * 1024L));
+ new MemStoreSize((3L * 1024L * 1024L * 1024L), 0, (1L * 1024L * 1024L * 1024L), 100);
regionServerAccounting.incGlobalMemStoreSize(memstoreSize);
assertEquals(FlushType.ABOVE_OFFHEAP_HIGHER_MARK,
regionServerAccounting.isAboveHighWaterMark());
@@ -90,11 +86,9 @@ public class TestRegionServerAccounting {
RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf);
// this will breach higher limit as heap size is higher and not due to offheap size
long dataSize = regionServerAccounting.getGlobalOnHeapMemStoreLimit();
- MemStoreSize memstoreSize =
- new MemStoreSize(dataSize, dataSize, 0);
+ MemStoreSize memstoreSize = new MemStoreSize(dataSize, dataSize, 0, 100);
regionServerAccounting.incGlobalMemStoreSize(memstoreSize);
- assertEquals(FlushType.ABOVE_ONHEAP_HIGHER_MARK,
- regionServerAccounting.isAboveHighWaterMark());
+ assertEquals(FlushType.ABOVE_ONHEAP_HIGHER_MARK, regionServerAccounting.isAboveHighWaterMark());
}
@Test
@@ -105,10 +99,9 @@ public class TestRegionServerAccounting {
RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf);
// this will breach offheap limit as data size is higher and not due to heap size
MemStoreSize memstoreSize =
- new MemStoreSize((3L * 1024L * 1024L * 1024L), 0, (1L * 1024L * 1024L * 1024L));
+ new MemStoreSize((3L * 1024L * 1024L * 1024L), 0, (1L * 1024L * 1024L * 1024L), 100);
regionServerAccounting.incGlobalMemStoreSize(memstoreSize);
- assertEquals(FlushType.ABOVE_OFFHEAP_LOWER_MARK,
- regionServerAccounting.isAboveLowWaterMark());
+ assertEquals(FlushType.ABOVE_OFFHEAP_LOWER_MARK, regionServerAccounting.isAboveLowWaterMark());
}
@Test
@@ -119,10 +112,8 @@ public class TestRegionServerAccounting {
RegionServerAccounting regionServerAccounting = new RegionServerAccounting(conf);
// this will breach higher limit as heap size is higher and not due to offheap size
long dataSize = regionServerAccounting.getGlobalOnHeapMemStoreLimit();
- MemStoreSize memstoreSize =
- new MemStoreSize(dataSize, dataSize, 0);
+ MemStoreSize memstoreSize = new MemStoreSize(dataSize, dataSize, 0, 100);
regionServerAccounting.incGlobalMemStoreSize(memstoreSize);
- assertEquals(FlushType.ABOVE_ONHEAP_LOWER_MARK,
- regionServerAccounting.isAboveLowWaterMark());
+ assertEquals(FlushType.ABOVE_ONHEAP_LOWER_MARK, regionServerAccounting.isAboveLowWaterMark());
}
}