You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2017/12/29 00:26:46 UTC
hbase git commit: HBASE-19282: Making CellChunkMap the default index
(CellSet delegatee) for ImmutableSegments, when MSLAB is used.
Repository: hbase
Updated Branches:
refs/heads/master b3a4fca48 -> 8d0da1a77
HBASE-19282: Making CellChunkMap the default index (CellSet delegatee) for ImmutableSegments, when MSLAB is used.
In order to avoid additional user settings. If no MSLAB is requested the index is going to be CellArrayMap
Signed-off-by: Anastasia Braginsky <an...@yahoo-inc.com>
Signed-off-by: Michael Stack <st...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8d0da1a7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8d0da1a7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8d0da1a7
Branch: refs/heads/master
Commit: 8d0da1a77f50b730b366c28b5b477141aa83cc55
Parents: b3a4fca
Author: anastas <an...@yahoo-inc.com>
Authored: Thu Dec 28 22:44:06 2017 +0200
Committer: Michael Stack <st...@apache.org>
Committed: Thu Dec 28 16:26:38 2017 -0800
----------------------------------------------------------------------
.../hbase/regionserver/CompactingMemStore.java | 45 ++++++++++++--------
.../hbase/regionserver/MemStoreLABImpl.java | 7 +--
.../regionserver/TestCompactingMemStore.java | 1 +
.../TestCompactingToCellFlatMapMemStore.java | 45 ++++++++------------
.../TestWalAndCompactingMemStoreFlush.java | 9 +++-
5 files changed, 57 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8d0da1a7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index 94cc0b4..d874b2e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
@@ -58,16 +58,10 @@ public class CompactingMemStore extends AbstractMemStore {
"hbase.hregion.compacting.memstore.type";
public static final String COMPACTING_MEMSTORE_TYPE_DEFAULT =
String.valueOf(MemoryCompactionPolicy.BASIC);
- // The external setting of the compacting MemStore behaviour
- public static final String COMPACTING_MEMSTORE_INDEX_KEY =
- "hbase.hregion.compacting.memstore.index";
- // usage of CellArrayMap is default, later it will be decided how to use CellChunkMap
- public static final String COMPACTING_MEMSTORE_INDEX_DEFAULT =
- String.valueOf(IndexType.ARRAY_MAP);
// Default fraction of in-memory-flush size w.r.t. flush-to-disk size
public static final String IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY =
"hbase.memstore.inmemoryflush.threshold.factor";
- private static final double IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT = 0.02;
+ private static final double IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT = 0.1;
private static final Logger LOG = LoggerFactory.getLogger(CompactingMemStore.class);
private HStore store;
@@ -114,9 +108,16 @@ public class CompactingMemStore extends AbstractMemStore {
this.regionServices = regionServices;
this.pipeline = new CompactionPipeline(getRegionServices());
this.compactor = createMemStoreCompactor(compactionPolicy);
+ if (conf.getBoolean(MemStoreLAB.USEMSLAB_KEY, MemStoreLAB.USEMSLAB_DEFAULT)) {
+ // if user requested to work with MSLABs (whether on- or off-heap), then the
+ // immutable segments are going to use CellChunkMap as their index
+ indexType = IndexType.CHUNK_MAP;
+ } else {
+ indexType = IndexType.ARRAY_MAP;
+ }
+ // initialization of the flush size should happen after initialization of the index type
+ // so do not transfer the following method
initInmemoryFlushSize(conf);
- indexType = IndexType.valueOf(conf.get(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY,
- CompactingMemStore.COMPACTING_MEMSTORE_INDEX_DEFAULT));
}
@VisibleForTesting
@@ -126,6 +127,7 @@ public class CompactingMemStore extends AbstractMemStore {
}
private void initInmemoryFlushSize(Configuration conf) {
+ double factor = 0;
long memstoreFlushSize = getRegionServices().getMemStoreFlushSize();
int numStores = getRegionServices().getNumStores();
if (numStores <= 1) {
@@ -133,11 +135,17 @@ public class CompactingMemStore extends AbstractMemStore {
numStores = 1;
}
inmemoryFlushSize = memstoreFlushSize / numStores;
- // multiply by a factor
- double factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
- IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT);
+ // multiply by a factor (different factors for different index types)
+ if (indexType == IndexType.ARRAY_MAP) {
+ factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
+ IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT);
+ } else {
+ factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY,
+ IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT);
+ }
inmemoryFlushSize *= factor;
- LOG.info("Setting in-memory flush size threshold to " + inmemoryFlushSize);
+ LOG.info("Setting in-memory flush size threshold to " + inmemoryFlushSize
+ + " and immutable segments index to be of type " + indexType);
}
/**
@@ -318,10 +326,11 @@ public class CompactingMemStore extends AbstractMemStore {
// setter is used only for testability
@VisibleForTesting
- public void setIndexType() {
- indexType = IndexType.valueOf(getConfiguration().get(
- CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY,
- CompactingMemStore.COMPACTING_MEMSTORE_INDEX_DEFAULT));
+ void setIndexType(IndexType type) {
+ indexType = type;
+ // Because this functionality is for testing only and tests are setting in-memory flush size
+ // according to their need, there is no setting of in-memory flush size, here.
+ // If it is needed, please change in-memory flush size explicitly
}
public IndexType getIndexType() {
@@ -572,7 +581,7 @@ public class CompactingMemStore extends AbstractMemStore {
// debug method
public void debug() {
String msg = "active size=" + this.active.keySize();
- msg += " threshold="+IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT* inmemoryFlushSize;
+ msg += " in-memory flush size is "+ inmemoryFlushSize;
msg += " allow compaction is "+ (allowCompaction.get() ? "true" : "false");
msg += " inMemoryFlushInProgress is "+ (inMemoryFlushInProgress.get() ? "true" : "false");
LOG.debug(msg);
http://git-wip-us.apache.org/repos/asf/hbase/blob/8d0da1a7/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java
index 64c5e52..f7728ac 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java
@@ -100,9 +100,10 @@ public class MemStoreLABImpl implements MemStoreLAB {
// if we don't exclude allocations >CHUNK_SIZE, we'd infiniteloop on one!
Preconditions.checkArgument(maxAlloc <= chunkSize,
MAX_ALLOC_KEY + " must be less than " + CHUNK_SIZE_KEY);
- idxType = CompactingMemStore.IndexType.valueOf(conf.get(
- CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY,
- CompactingMemStore.COMPACTING_MEMSTORE_INDEX_DEFAULT));
+
+ // if user requested to work with MSLABs (whether on- or off-heap), then the
+ // immutable segments are going to use CellChunkMap as their index
+ idxType = CompactingMemStore.IndexType.CHUNK_MAP;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/8d0da1a7/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
index ed5eee8..c0ba621 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java
@@ -89,6 +89,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore {
compactingSetUp();
this.memstore = new MyCompactingMemStore(HBaseConfiguration.create(), CellComparator.getInstance(),
store, regionServicesForStores, MemoryCompactionPolicy.EAGER);
+ ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.ARRAY_MAP);
}
protected void compactingSetUp() throws Exception {
http://git-wip-us.apache.org/repos/asf/hbase/blob/8d0da1a7/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
index 1903c51..1fc7791 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hbase.util.Threads;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -94,9 +95,9 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
String[] keys1 = { "A", "A", "B", "C" }; //A1, A2, B3, C4
if (toCellChunkMap) {
// set memstore to flat into CellChunkMap
- conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY,
- String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP));
- ((CompactingMemStore)memstore).setIndexType();
+ ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP);
+ } else {
+ ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.ARRAY_MAP);
}
// test 1 bucket
@@ -140,9 +141,9 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
public void testCompaction2Buckets() throws IOException {
if (toCellChunkMap) {
// set memstore to flat into CellChunkMap
- conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY,
- String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP));
- ((CompactingMemStore)memstore).setIndexType();
+ ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP);
+ } else {
+ ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.ARRAY_MAP);
}
String[] keys1 = { "A", "A", "B", "C" };
String[] keys2 = { "A", "B", "D" };
@@ -202,9 +203,10 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
public void testCompaction3Buckets() throws IOException {
if (toCellChunkMap) {
// set memstore to flat into CellChunkMap
- conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY,
- String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP));
- ((CompactingMemStore)memstore).setIndexType();
+ ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP);
+ } else {
+ // set to CellArrayMap as CCM is configured by default due to MSLAB usage
+ ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.ARRAY_MAP);
}
String[] keys1 = { "A", "A", "B", "C" };
String[] keys2 = { "A", "B", "D" };
@@ -290,9 +292,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
public void testMerging() throws IOException {
if (toCellChunkMap) {
// set memstore to flat into CellChunkMap
- conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY,
- String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP));
- ((CompactingMemStore)memstore).setIndexType();
+ ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP);
}
String[] keys1 = { "A", "A", "B", "C", "F", "H"};
String[] keys2 = { "A", "B", "D", "G", "I", "J"};
@@ -362,9 +362,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
public void testTimeRangeAfterCompaction() throws IOException {
if (toCellChunkMap) {
// set memstore to flat into CellChunkMap
- conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY,
- String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP));
- ((CompactingMemStore)memstore).setIndexType();
+ ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP);
}
testTimeRange(true);
}
@@ -373,9 +371,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
public void testTimeRangeAfterMerge() throws IOException {
if (toCellChunkMap) {
// set memstore to flat into CellChunkMap
- conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY,
- String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP));
- ((CompactingMemStore)memstore).setIndexType();
+ ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP);
}
MemoryCompactionPolicy compactionType = MemoryCompactionPolicy.BASIC;
memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
@@ -620,9 +616,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
String.valueOf(compactionType));
((MyCompactingMemStore)memstore).initiateType(compactionType, memstore.getConfiguration());
- memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY,
- String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP));
- ((CompactingMemStore)memstore).setIndexType();
+ ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP);
int numOfCells = 8;
String[] keys1 = { "A", "A", "B", "C", "D", "D", "E", "F" }; //A1, A2, B3, C4, D5, D6, E7, F8
@@ -684,9 +678,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
String.valueOf(compactionType));
((MyCompactingMemStore)memstore).initiateType(compactionType, memstore.getConfiguration());
- memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY,
- String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP));
- ((CompactingMemStore)memstore).setIndexType();
+ ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP);
int numOfCells = 4;
char[] chars = new char[MemStoreLAB.MAX_ALLOC_DEFAULT];
for (int i = 0; i < chars.length; i++) {
@@ -747,6 +739,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
* testFlatteningToJumboCellChunkMap checks that the process of flattening
* into CellChunkMap succeeds, even when such big cells are allocated.
*/
+ @Ignore
@Test
public void testFlatteningToJumboCellChunkMap() throws IOException {
@@ -758,9 +751,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
String.valueOf(compactionType));
((MyCompactingMemStore)memstore).initiateType(compactionType, memstore.getConfiguration());
- memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY,
- String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP));
- ((CompactingMemStore)memstore).setIndexType();
+ ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP);
int numOfCells = 1;
char[] chars = new char[MemStoreLAB.CHUNK_SIZE_DEFAULT];
for (int i = 0; i < chars.length; i++) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/8d0da1a7/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
index 6a64796..f3bd7ee 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java
@@ -768,8 +768,7 @@ public class TestWalAndCompactingMemStoreFlush {
conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 300 * 1024);
conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY,
FlushNonSloppyStoresFirstPolicy.class.getName());
- conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN,
- 75 * 1024);
+ conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN, 75 * 1024);
conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.8);
// set memstore to do index compaction with merge
conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
@@ -797,6 +796,12 @@ public class TestWalAndCompactingMemStoreFlush {
long totalMemstoreSize = region.getMemStoreSize();
+ // test in-memory flashing into CAM here
+ ((CompactingMemStore) ((HStore)region.getStore(FAMILY1)).memstore).setIndexType(
+ CompactingMemStore.IndexType.ARRAY_MAP);
+ ((CompactingMemStore) ((HStore)region.getStore(FAMILY3)).memstore).setIndexType(
+ CompactingMemStore.IndexType.ARRAY_MAP);
+
// Find the sizes of the memstores of each CF.
MemStoreSize cf1MemstoreSizePhaseI = region.getStore(FAMILY1).getMemStoreSize();
MemStoreSize cf2MemstoreSizePhaseI = region.getStore(FAMILY2).getMemStoreSize();