You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by al...@apache.org on 2018/08/13 12:28:29 UTC
hbase git commit: HBASE-21029 Miscount of memstore's heap/offheap
size if same cell was put
Repository: hbase
Updated Branches:
refs/heads/branch-2.0 c1982f44d -> ce8904034
HBASE-21029 Miscount of memstore's heap/offheap size if same cell was put
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ce890403
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ce890403
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ce890403
Branch: refs/heads/branch-2.0
Commit: ce8904034ed38ff3c7e0e7e313d5fc9ed677a542
Parents: c1982f4
Author: Allan Yang <al...@apache.org>
Authored: Mon Aug 13 20:27:51 2018 +0800
Committer: Allan Yang <al...@apache.org>
Committed: Mon Aug 13 20:27:51 2018 +0800
----------------------------------------------------------------------
.../apache/hadoop/hbase/regionserver/Segment.java | 10 +++++++---
.../hbase/regionserver/TestDefaultMemStore.java | 15 +++++++++++++++
2 files changed, 22 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ce890403/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
index 0c837f3..81efae6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
@@ -307,11 +307,15 @@ public abstract class Segment implements MemStoreSizing {
// If there's already a same cell in the CellSet and we are using MSLAB, we must count in the
// MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger
// than the counted number)
- if (succ || mslabUsed) {
+ boolean sizeChanged = succ || mslabUsed;
+ if (sizeChanged) {
cellSize = getCellLength(cellToAdd);
}
- long heapSize = heapSizeChange(cellToAdd, succ);
- long offHeapSize = offHeapSizeChange(cellToAdd, succ);
+ // same as above, if MSLAB is used, we need to inc the heap/offheap size, otherwise there will
+ // be a memory miscount. Since we are now use heapSize + offHeapSize to decide whether a flush
+ // is needed.
+ long heapSize = heapSizeChange(cellToAdd, sizeChanged);
+ long offHeapSize = offHeapSizeChange(cellToAdd, sizeChanged);
incMemStoreSize(cellSize, heapSize, offHeapSize);
if (memstoreSizing != null) {
memstoreSizing.incMemStoreSize(cellSize, heapSize, offHeapSize);
http://git-wip-us.apache.org/repos/asf/hbase/blob/ce890403/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
index 77f796f..ad0d2ff 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
@@ -138,6 +138,21 @@ public class TestDefaultMemStore {
Segment segment = this.memstore.getActive();
MemStoreLAB msLab = segment.getMemStoreLAB();
if (msLab != null) {
+ if (msLab.isOnHeap()) {
+ assertTrue("HeapSize should always bigger or equal than data size",
+ sizeChangeForFirstCell.getHeapSize() >= sizeChangeForFirstCell
+ .getDataSize());
+ assertTrue("HeapSize should always bigger or equal than data size",
+ sizeChangeForSecondCell.getHeapSize() >= sizeChangeForSecondCell
+ .getDataSize());
+ } else {
+ assertTrue("OffHeapSize should always bigger or equal than data size",
+ sizeChangeForFirstCell.getOffHeapSize() >= sizeChangeForFirstCell
+ .getDataSize());
+ assertTrue("OffHeapSize should always bigger or equal than data size",
+ sizeChangeForSecondCell.getOffHeapSize() >= sizeChangeForSecondCell
+ .getDataSize());
+ }
// make sure memstore size increased even when writing the same cell, if using MSLAB
assertEquals(Segment.getCellLength(kv),
sizeChangeForSecondCell.getMemStoreSize().getDataSize());