You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2018/05/28 07:40:59 UTC
[05/34] hbase git commit: HBASE-20390: IMC Default Parameters for
2.0.0
HBASE-20390: IMC Default Parameters for 2.0.0
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1eabbb42
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1eabbb42
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1eabbb42
Branch: refs/heads/HBASE-19064
Commit: 1eabbb42954b94308f85434ed2a9a80ba3e03c5e
Parents: 874f1e8
Author: eshcar <es...@oath.com>
Authored: Sat May 26 22:57:28 2018 +0300
Committer: eshcar <es...@oath.com>
Committed: Sat May 26 22:57:28 2018 +0300
----------------------------------------------------------------------
.../hadoop/hbase/regionserver/CompactingMemStore.java | 10 +++++-----
.../hbase/regionserver/MemStoreCompactionStrategy.java | 2 +-
.../regionserver/TestCompactingToCellFlatMapMemStore.java | 4 +++-
src/main/asciidoc/_chapters/inmemory_compaction.adoc | 7 ++++---
4 files changed, 13 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/1eabbb42/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
index fc3e5ba..3886e7d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java
@@ -43,12 +43,12 @@ import org.apache.hadoop.hbase.wal.WAL;
/**
* A memstore implementation which supports in-memory compaction.
* A compaction pipeline is added between the active set and the snapshot data structures;
- * it consists of a list of kv-sets that are subject to compaction.
- * Like the snapshot, all pipeline components are read-only; updates only affect the active set.
+ * it consists of a list of segments that are subject to compaction.
+ * Like the snapshot, all pipeline segments are read-only; updates only affect the active set.
* To ensure this property we take advantage of the existing blocking mechanism -- the active set
* is pushed to the pipeline while holding the region's updatesLock in exclusive mode.
- * Periodically, a compaction is applied in the background to all pipeline components resulting
- * in a single read-only component. The ``old'' components are discarded when no scanner is reading
+ * Periodically, a compaction is applied in the background to all pipeline segments resulting
+ * in a single read-only component. The ``old'' segments are discarded when no scanner is reading
* them.
*/
@InterfaceAudience.Private
@@ -62,7 +62,7 @@ public class CompactingMemStore extends AbstractMemStore {
// Default fraction of in-memory-flush size w.r.t. flush-to-disk size
public static final String IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY =
"hbase.memstore.inmemoryflush.threshold.factor";
- private static final double IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT = 0.1;
+ private static final double IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT = 0.014;
private static final Logger LOG = LoggerFactory.getLogger(CompactingMemStore.class);
private HStore store;
http://git-wip-us.apache.org/repos/asf/hbase/blob/1eabbb42/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.java
index 2f16055..6d28e37 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactionStrategy.java
@@ -42,7 +42,7 @@ public abstract class MemStoreCompactionStrategy {
// The upper bound for the number of segments we store in the pipeline prior to merging.
public static final String COMPACTING_MEMSTORE_THRESHOLD_KEY =
"hbase.hregion.compacting.pipeline.segments.limit";
- public static final int COMPACTING_MEMSTORE_THRESHOLD_DEFAULT = 4;
+ public static final int COMPACTING_MEMSTORE_THRESHOLD_DEFAULT = 2;
/**
* Types of actions to be done on the pipeline upon MemStoreCompaction invocation.
http://git-wip-us.apache.org/repos/asf/hbase/blob/1eabbb42/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
index e824120..943757b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java
@@ -846,8 +846,10 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore
// set memstore to flat into CellChunkMap
MemoryCompactionPolicy compactionType = MemoryCompactionPolicy.BASIC;
+ memstore.getConfiguration().setInt(MemStoreCompactionStrategy
+ .COMPACTING_MEMSTORE_THRESHOLD_KEY, 4);
memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
- String.valueOf(compactionType));
+ String.valueOf(compactionType));
((MyCompactingMemStore) memstore).initiateType(compactionType, memstore.getConfiguration());
((CompactingMemStore) memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP);
http://git-wip-us.apache.org/repos/asf/hbase/blob/1eabbb42/src/main/asciidoc/_chapters/inmemory_compaction.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/inmemory_compaction.adoc b/src/main/asciidoc/_chapters/inmemory_compaction.adoc
index da3abb1..21513f3 100644
--- a/src/main/asciidoc/_chapters/inmemory_compaction.adoc
+++ b/src/main/asciidoc/_chapters/inmemory_compaction.adoc
@@ -52,7 +52,7 @@ In this section we describe how to enable Accordion and the available configurat
To enable in-memory compactions, set the _IN_MEMORY_COMPACTION_ attribute
on per column family where you want the behavior. The _IN_MEMORY_COMPACTION_
-attribute can have one of three values.
+attribute can have one of four values.
* _NONE_: No in-memory compaction.
* _BASIC_: Basic policy enables flushing and keeps a pipeline of flushes until we trip the pipeline maximum threshold and then we flush to disk. No in-memory compaction but can help throughput as data is moved from the profligate, native ConcurrentSkipListMap data-type to more compact (and efficient) data types.
@@ -94,11 +94,12 @@ to set the in-memory attribute to _NONE_).
When an in-memory flush happens is calculated by dividing the configured region flush size (Set in the table descriptor
or read from _hbase.hregion.memstore.flush.size_) by the number of column families and then multiplying by
-_hbase.memstore.inmemoryflush.threshold.factor_ (default is 0.1).
+_hbase.memstore.inmemoryflush.threshold.factor_ (default is 0.014 - a little less than the size
+of a chunk).
The number of flushes carried by the pipeline is monitored so as to fit within the bounds of memstore sizing
but you can also set a maximum on the number of flushes total by setting
-_hbase.hregion.compacting.pipeline.segments.limit_. Default is 4.
+_hbase.hregion.compacting.pipeline.segments.limit_. Default is 2.
When a column family Store is created, it says what memstore type is in effect. As of this writing
there is the old-school _DefaultMemStore_ which fills a _ConcurrentSkipListMap_ and then flushes