You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ga...@apache.org on 2017/02/24 22:56:00 UTC
hbase git commit: HBASE-17057 Minor compactions should also drop page
cache (Ashu Pachauri)
Repository: hbase
Updated Branches:
refs/heads/master a8fd1119c -> e7d16db2a
HBASE-17057 Minor compactions should also drop page cache (Ashu Pachauri)
Signed-off-by: Gary Helmling <ga...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e7d16db2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e7d16db2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e7d16db2
Branch: refs/heads/master
Commit: e7d16db2ac06127498d0a056e6b07cbc1cba36bc
Parents: a8fd111
Author: Ashu Pachauri <as...@gmail.com>
Authored: Wed Nov 30 15:47:59 2016 -0800
Committer: Gary Helmling <ga...@apache.org>
Committed: Fri Feb 24 14:42:10 2017 -0800
----------------------------------------------------------------------
.../src/main/resources/hbase-default.xml | 18 +++++++++++++
.../regionserver/compactions/Compactor.java | 28 ++++++++++++++++----
2 files changed, 41 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7d16db2/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index 0324bd5..40cf675 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -803,6 +803,24 @@ possible configurations would overwhelm and obscure the important.
the default.</description>
</property>
<property>
+ <name>hbase.regionserver.majorcompaction.pagecache.drop</name>
+ <value>true</value>
+ <description>Specifies whether to drop pages read/written into the system page cache by
+ major compactions. Setting it to true helps prevent major compactions from
+ polluting the page cache, which is almost always required, especially for clusters
+ with low/moderate memory to storage ratio.</description>
+ </property>
+ <property>
+ <name>hbase.regionserver.minorcompaction.pagecache.drop</name>
+ <value>true</value>
+ <description>Specifies whether to drop pages read/written into the system page cache by
+ minor compactions. Setting it to true helps prevent minor compactions from
+ polluting the page cache, which is most beneficial on clusters with low
+ memory to storage ratio or very write heavy clusters. You may want to set it to
+ false under moderate to low write workload when bulk of the reads are
+ on the most recently written data.</description>
+ </property>
+ <property>
<name>hbase.hstore.compaction.kv.max</name>
<value>10</value>
<description>The maximum number of KeyValues to read and then write in a batch when flushing or
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7d16db2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
index e1c08bd..84e4314 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
@@ -79,6 +79,15 @@ public abstract class Compactor<T extends CellSink> {
/** specify how many days to keep MVCC values during major compaction **/
protected int keepSeqIdPeriod;
+ // Configs that drive whether we drop page cache behind compactions
+ protected static final String MAJOR_COMPACTION_DROP_CACHE =
+ "hbase.regionserver.majorcompaction.pagecache.drop";
+ protected static final String MINOR_COMPACTION_DROP_CACHE =
+ "hbase.regionserver.minorcompaction.pagecache.drop";
+
+ private boolean dropCacheMajor;
+ private boolean dropCacheMinor;
+
//TODO: depending on Store is not good but, realistically, all compactors currently do.
Compactor(final Configuration conf, final Store store) {
this.conf = conf;
@@ -89,8 +98,12 @@ public abstract class Compactor<T extends CellSink> {
Compression.Algorithm.NONE : this.store.getFamily().getCompactionCompressionType();
this.keepSeqIdPeriod = Math.max(this.conf.getInt(HConstants.KEEP_SEQID_PERIOD,
HConstants.MIN_KEEP_SEQID_PERIOD), HConstants.MIN_KEEP_SEQID_PERIOD);
+ this.dropCacheMajor = conf.getBoolean(MAJOR_COMPACTION_DROP_CACHE, true);
+ this.dropCacheMinor = conf.getBoolean(MINOR_COMPACTION_DROP_CACHE, true);
}
+
+
protected interface CellSinkFactory<S> {
S createWriter(InternalScanner scanner, FileDetails fd, boolean shouldDropBehind)
throws IOException;
@@ -271,6 +284,13 @@ public abstract class Compactor<T extends CellSink> {
List<StoreFileScanner> scanners;
Collection<StoreFile> readersToClose;
T writer = null;
+ boolean dropCache;
+ if (request.isMajor() || request.isAllFiles()) {
+ dropCache = this.dropCacheMajor;
+ } else {
+ dropCache = this.dropCacheMinor;
+ }
+
if (this.conf.getBoolean("hbase.regionserver.compaction.private.readers", true)) {
// clone all StoreFiles, so we'll do the compaction on a independent copy of StoreFiles,
// HFiles, and their readers
@@ -282,12 +302,10 @@ public abstract class Compactor<T extends CellSink> {
clonedStoreFile.createReader();
readersToClose.add(clonedStoreFile);
}
- scanners = createFileScanners(readersToClose, smallestReadPoint,
- store.throttleCompaction(request.getSize()));
+ scanners = createFileScanners(readersToClose, smallestReadPoint, dropCache);
} else {
readersToClose = Collections.emptyList();
- scanners = createFileScanners(request.getFiles(), smallestReadPoint,
- store.throttleCompaction(request.getSize()));
+ scanners = createFileScanners(request.getFiles(), smallestReadPoint, dropCache);
}
InternalScanner scanner = null;
boolean finished = false;
@@ -309,7 +327,7 @@ public abstract class Compactor<T extends CellSink> {
smallestReadPoint = Math.min(fd.minSeqIdToKeep, smallestReadPoint);
cleanSeqId = true;
}
- writer = sinkFactory.createWriter(scanner, fd, store.throttleCompaction(request.getSize()));
+ writer = sinkFactory.createWriter(scanner, fd, dropCache);
finished = performCompaction(fd, scanner, writer, smallestReadPoint, cleanSeqId,
throughputController, request.isAllFiles(), request.getFiles().size());
if (!finished) {