You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by sy...@apache.org on 2015/10/30 17:54:15 UTC
[16/17] hbase git commit: HBASE-14721 Memstore add cells - Avoid many
garbage.
HBASE-14721 Memstore add cells - Avoid many garbage.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/23fa1818
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/23fa1818
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/23fa1818
Branch: refs/heads/hbase-12439
Commit: 23fa18184cb68ca05246beb2189f8801200bdd7c
Parents: 84f4cce
Author: anoopsjohn <an...@gmail.com>
Authored: Fri Oct 30 18:39:42 2015 +0530
Committer: anoopsjohn <an...@gmail.com>
Committed: Fri Oct 30 18:39:42 2015 +0530
----------------------------------------------------------------------
.../hbase/regionserver/DefaultMemStore.java | 18 ++++++------------
.../hadoop/hbase/regionserver/HRegion.java | 20 ++++++--------------
.../hadoop/hbase/regionserver/HStore.java | 3 +--
.../hadoop/hbase/regionserver/MemStore.java | 6 ++----
.../apache/hadoop/hbase/regionserver/Store.java | 5 ++---
.../hadoop/hbase/regionserver/TestStore.java | 17 ++++++-----------
6 files changed, 23 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/23fa1818/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
index 9bc6a9c..d760e9a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.CollectionBackedScanner;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.htrace.Trace;
@@ -221,13 +220,12 @@ public class DefaultMemStore implements MemStore {
/**
* Write an update
* @param cell
- * @return approximate size of the passed KV & newly added KV which maybe different than the
- * passed-in KV
+ * @return approximate size of the passed Cell.
*/
@Override
- public Pair<Long, Cell> add(Cell cell) {
+ public long add(Cell cell) {
Cell toAdd = maybeCloneWithAllocator(cell);
- return new Pair<Long, Cell>(internalAdd(toAdd), toAdd);
+ return internalAdd(toAdd);
}
@Override
@@ -964,21 +962,17 @@ public class DefaultMemStore implements MemStore {
byte [] empty = new byte[0];
for (int i = 0; i < count; i++) {
// Give each its own ts
- Pair<Long, Cell> ret = memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty));
- size += ret.getFirst();
+ size += memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty));
}
LOG.info("memstore1 estimated size=" + size);
for (int i = 0; i < count; i++) {
- Pair<Long, Cell> ret = memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty));
- size += ret.getFirst();
+ size += memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty));
}
LOG.info("memstore1 estimated size (2nd loading of same data)=" + size);
// Make a variably sized memstore.
DefaultMemStore memstore2 = new DefaultMemStore();
for (int i = 0; i < count; i++) {
- Pair<Long, Cell> ret = memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i,
- new byte[i]));
- size += ret.getFirst();
+ size += memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, new byte[i]));
}
LOG.info("memstore2 estimated size=" + size);
final int seconds = 30;
http://git-wip-us.apache.org/repos/asf/hbase/blob/23fa1818/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 19bcd33..e987bc6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -3686,15 +3686,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
int listSize = cells.size();
for (int i=0; i < listSize; i++) {
Cell cell = cells.get(i);
- if (cell.getSequenceId() == 0) {
+ if (cell.getSequenceId() == 0 || isInReplay) {
CellUtil.setSequenceId(cell, mvccNum);
}
- Pair<Long, Cell> ret = store.add(cell);
- size += ret.getFirst();
- if(isInReplay) {
- // set memstore newly added cells with replay mvcc number
- CellUtil.setSequenceId(ret.getSecond(), mvccNum);
- }
+ size += store.add(cell);
}
}
@@ -4937,7 +4932,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
* @return True if we should flush.
*/
protected boolean restoreEdit(final Store s, final Cell cell) {
- long kvSize = s.add(cell).getFirst();
+ long kvSize = s.add(cell);
if (this.rsAccounting != null) {
rsAccounting.addAndGetRegionReplayEditsSize(getRegionInfo().getRegionName(), kvSize);
}
@@ -6829,8 +6824,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
checkFamily(CellUtil.cloneFamily(cell));
// unreachable
}
- Pair<Long, Cell> ret = store.add(cell);
- addedSize += ret.getFirst();
+ addedSize += store.add(cell);
}
}
@@ -7168,8 +7162,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// otherwise keep older versions around
for (Cell cell: entry.getValue()) {
CellUtil.setSequenceId(cell, writeEntry.getWriteNumber());
- Pair<Long, Cell> ret = store.add(cell);
- size += ret.getFirst();
+ size += store.add(cell);
doRollBackMemstore = true;
}
}
@@ -7399,8 +7392,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
// otherwise keep older versions around
for (Cell cell : entry.getValue()) {
CellUtil.setSequenceId(cell, writeEntry.getWriteNumber());
- Pair<Long, Cell> ret = store.add(cell);
- size += ret.getFirst();
+ size += store.add(cell);
doRollBackMemstore = true;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/23fa1818/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index c0f1d55..1d996b2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -89,7 +89,6 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.util.ClassSize;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.ReflectionUtils;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.util.StringUtils;
@@ -675,7 +674,7 @@ public class HStore implements Store {
}
@Override
- public Pair<Long, Cell> add(final Cell cell) {
+ public long add(final Cell cell) {
lock.readLock().lock();
try {
return this.memstore.add(cell);
http://git-wip-us.apache.org/repos/asf/hbase/blob/23fa1818/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
index d24299d..e9f8103 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
@@ -22,7 +22,6 @@ import java.util.List;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.io.HeapSize;
-import org.apache.hadoop.hbase.util.Pair;
/**
* The MemStore holds in-memory modifications to the Store. Modifications are {@link Cell}s.
@@ -68,10 +67,9 @@ public interface MemStore extends HeapSize {
/**
* Write an update
* @param cell
- * @return approximate size of the passed KV and the newly added KV which maybe different from the
- * passed in KV.
+ * @return approximate size of the passed cell.
*/
- Pair<Long, Cell> add(final Cell cell);
+ long add(final Cell cell);
/**
* @return Oldest timestamp of all the Cells in the MemStore
http://git-wip-us.apache.org/repos/asf/hbase/blob/23fa1818/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 33e4416..83a24a5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController;
import org.apache.hadoop.hbase.security.User;
-import org.apache.hadoop.hbase.util.Pair;
/**
* Interface for objects that hold a column family in a Region. Its a memstore and a set of zero or
@@ -127,9 +126,9 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf
/**
* Adds a value to the memstore
* @param cell
- * @return memstore size delta & newly added KV which maybe different than the passed in KV
+ * @return memstore size delta
*/
- Pair<Long, Cell> add(Cell cell);
+ long add(Cell cell);
/**
* When was the last edit done in the memstore
http://git-wip-us.apache.org/repos/asf/hbase/blob/23fa1818/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
index 6b669a0..85c75b1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestStore.java
@@ -223,7 +223,7 @@ public class TestStore {
long size = store.memstore.getFlushableSize();
Assert.assertEquals(0, size);
LOG.info("Adding some data");
- long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null)).getFirst();
+ long kvSize = store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
size = store.memstore.getFlushableSize();
Assert.assertEquals(kvSize, size);
// Flush. Bug #1 from HBASE-10466. Make sure size calculation on failed flush is right.
@@ -638,20 +638,15 @@ public class TestStore {
size += this.store.add(new KeyValue(Bytes.toBytes("200909091000"), family, qf1,
- System.currentTimeMillis(),
- Bytes.toBytes(newValue))).getFirst();
+ System.currentTimeMillis(), Bytes.toBytes(newValue)));
size += this.store.add(new KeyValue(Bytes.toBytes("200909091200"), family, qf1,
- System.currentTimeMillis(),
- Bytes.toBytes(newValue))).getFirst();
+ System.currentTimeMillis(), Bytes.toBytes(newValue)));
size += this.store.add(new KeyValue(Bytes.toBytes("200909091300"), family, qf1,
- System.currentTimeMillis(),
- Bytes.toBytes(newValue))).getFirst();
+ System.currentTimeMillis(), Bytes.toBytes(newValue)));
size += this.store.add(new KeyValue(Bytes.toBytes("200909091400"), family, qf1,
- System.currentTimeMillis(),
- Bytes.toBytes(newValue))).getFirst();
+ System.currentTimeMillis(), Bytes.toBytes(newValue)));
size += this.store.add(new KeyValue(Bytes.toBytes("200909091500"), family, qf1,
- System.currentTimeMillis(),
- Bytes.toBytes(newValue))).getFirst();
+ System.currentTimeMillis(), Bytes.toBytes(newValue)));
for ( int i = 0 ; i < 10000 ; ++i) {