You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by li...@apache.org on 2014/02/11 19:28:37 UTC
svn commit: r1567269 - in
/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver:
HRegion.java MemStore.java Store.java
Author: liyin
Date: Tue Feb 11 18:28:36 2014
New Revision: 1567269
URL: http://svn.apache.org/r1567269
Log:
[master] Let HRegion get correct flushable size from each Store
Author: fan
Summary:
Current calculation of flushable size in HRegion is wrong when previous flush attempt fails. The bug messes up HRegion's total memstore size, which leads to a very serious data loss for all data in memstore(s) when region is closing.
This fix corrects the size change after flush succeeds.
Test Plan: TBD
Reviewers: aaiyer, liyintang, jiqingt
Reviewed By: jiqingt
CC: HBase Eng@, hbase-dev@, manukranthk
Differential Revision: https://phabricator.fb.com/D1158429
Task ID: 3621385
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1567269&r1=1567268&r2=1567269&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue Feb 11 18:28:36 2014
@@ -1506,7 +1506,7 @@ public class HRegion implements HeapSize
for (Store store : storesToFlush) {
LOG.debug("Flushing Column Family: " + store.getColumnFamilyName() +
" which was occupying " +
- StringUtils.humanReadableInt(store.getMemStoreSize()) +
+ StringUtils.humanReadableInt(store.getFlushableMemstoreSize()) +
" of memstore.");
}
}
@@ -1561,7 +1561,7 @@ public class HRegion implements HeapSize
sequenceId = myseqid;
}
for (Store s : storesToFlush) {
- totalMemstoreSizeOfFlushableStores += s.getMemStoreSize();
+ totalMemstoreSizeOfFlushableStores += s.getFlushableMemstoreSize();
storeFlushers.add(s.getStoreFlusher(sequenceId));
}
@@ -2621,8 +2621,8 @@ public class HRegion implements HeapSize
* @return true if the size of the store is above the flush threshold for column families
*/
private boolean shouldFlushStore(Store store) {
- return (store.getMemStoreSize() > this.columnfamilyMemstoreFlushSize) ?
- true : false;
+ return store.getSnapshotSize() > 0 ||
+ store.getMemStoreSize() > this.columnfamilyMemstoreFlushSize;
}
/**
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java?rev=1567269&r1=1567268&r2=1567269&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java Tue Feb 11 18:28:36 2014
@@ -84,6 +84,7 @@ public class MemStore implements HeapSiz
// Used to track own heapSize
final AtomicLong size;
+ private long snapshotSize;
private AtomicLong numDeletesInKvSet;
private AtomicLong numDeletesInSnapshot;
@@ -122,6 +123,7 @@ public class MemStore implements HeapSiz
timeRangeTracker = new TimeRangeTracker();
snapshotTimeRangeTracker = new TimeRangeTracker();
this.size = new AtomicLong(DEEP_OVERHEAD);
+ this.snapshotSize = 0;
this.numDeletesInKvSet = new AtomicLong(0);
this.numDeletesInSnapshot = new AtomicLong(0);
@@ -201,6 +203,7 @@ public class MemStore implements HeapSiz
}
// Reset heap to not include any keys
+ this.snapshotSize = keySize();
this.size.set(DEEP_OVERHEAD);
this.numDeletesInSnapshot = numDeletesInKvSet;
this.numDeletesInKvSet.set(0);
@@ -248,6 +251,7 @@ public class MemStore implements HeapSiz
tmpAllocator = this.snapshotAllocator;
this.snapshotAllocator = null;
}
+ this.snapshotSize = 0;
} finally {
this.lock.writeLock().unlock();
}
@@ -883,6 +887,20 @@ public class MemStore implements HeapSiz
return heapSize() - DEEP_OVERHEAD;
}
+ public long getSnapshotSize() {
+ return snapshotSize;
+ }
+
+ /**
+ * Flush will first clear out the data in snapshot if any. If snapshot is
+ * empty, current keyvalue set will be flushed.
+ *
+ * @return size of data that is going to be flushed
+ */
+ public long getFlushableSize() {
+ return snapshotSize > 0 ? snapshotSize : keySize();
+ }
+
/**
* Code to help figure if our approximation of object heap sizes is close
* enough. See hbase-900. Fills memstores then waits so user can heap
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1567269&r1=1567268&r2=1567269&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Tue Feb 11 18:28:36 2014
@@ -1980,6 +1980,14 @@ public class Store extends SchemaConfigu
return this.memstore.keySize();
}
+ public long getSnapshotSize() {
+ return this.memstore.getSnapshotSize();
+ }
+
+ public long getFlushableMemstoreSize() {
+ return this.memstore.getFlushableSize();
+ }
+
/**
* A helper function to get the smallest LSN in the mestore.
* @return