You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ra...@apache.org on 2016/08/23 05:36:32 UTC
hbase git commit: HBASE-16454 Compactor's shipping logic decision
should be based on the current store's block size (Ram)
Repository: hbase
Updated Branches:
refs/heads/master 2aae923c3 -> ff6182b02
HBASE-16454 Compactor's shipping logic decision should be based on the
current store's block size (Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ff6182b0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ff6182b0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ff6182b0
Branch: refs/heads/master
Commit: ff6182b020c0d75f8d077eaeef7ae84124e1d4ce
Parents: 2aae923
Author: Ramkrishna <ra...@intel.com>
Authored: Tue Aug 23 11:06:02 2016 +0530
Committer: Ramkrishna <ra...@intel.com>
Committed: Tue Aug 23 11:06:02 2016 +0530
----------------------------------------------------------------------
.../hadoop/hbase/mob/DefaultMobStoreCompactor.java | 5 +++--
.../hbase/regionserver/compactions/Compactor.java | 12 +++++-------
2 files changed, 8 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/ff6182b0/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
index 711f31d..55aea00 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
@@ -160,12 +160,13 @@ public class DefaultMobStoreCompactor extends DefaultCompactor {
* @param cleanSeqId When true, remove seqId(used to be mvcc) value which is <= smallestReadPoint
* @param throughputController The compaction throughput controller.
* @param major Is a major compaction.
+ * @param numofFilesToCompact the number of files to compact
* @return Whether compaction ended; false if it was interrupted for any reason.
*/
@Override
protected boolean performCompaction(FileDetails fd, InternalScanner scanner, CellSink writer,
- long smallestReadPoint, boolean cleanSeqId,
- ThroughputController throughputController, boolean major) throws IOException {
+ long smallestReadPoint, boolean cleanSeqId, ThroughputController throughputController,
+ boolean major, int numofFilesToCompact) throws IOException {
if (!(scanner instanceof MobCompactionStoreScanner)) {
throw new IllegalArgumentException(
"The scanner should be an instance of MobCompactionStoreScanner");
http://git-wip-us.apache.org/repos/asf/hbase/blob/ff6182b0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
index 1796bca..c695788 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
@@ -310,7 +310,7 @@ public abstract class Compactor<T extends CellSink> {
}
writer = sinkFactory.createWriter(scanner, fd, store.throttleCompaction(request.getSize()));
finished = performCompaction(fd, scanner, writer, smallestReadPoint, cleanSeqId,
- throughputController, request.isAllFiles());
+ throughputController, request.isAllFiles(), request.getFiles().size());
if (!finished) {
throw new InterruptedIOException("Aborting compaction of store " + store + " in region "
+ store.getRegionInfo().getRegionNameAsString() + " because it was interrupted.");
@@ -385,11 +385,12 @@ public abstract class Compactor<T extends CellSink> {
* @param cleanSeqId When true, remove seqId(used to be mvcc) value which is <=
* smallestReadPoint
* @param major Is a major compaction.
+ * @param numofFilesToCompact the number of files to compact
* @return Whether compaction ended; false if it was interrupted for some reason.
*/
protected boolean performCompaction(FileDetails fd, InternalScanner scanner, CellSink writer,
- long smallestReadPoint, boolean cleanSeqId,
- ThroughputController throughputController, boolean major) throws IOException {
+ long smallestReadPoint, boolean cleanSeqId, ThroughputController throughputController,
+ boolean major, int numofFilesToCompact) throws IOException {
long bytesWrittenProgressForCloseCheck = 0;
long bytesWrittenProgressForLog = 0;
long bytesWrittenProgressForShippedCall = 0;
@@ -409,10 +410,7 @@ public abstract class Compactor<T extends CellSink> {
throughputController.start(compactionName);
KeyValueScanner kvs = (scanner instanceof KeyValueScanner)? (KeyValueScanner)scanner : null;
- long minFilesToCompact = Math.max(2L,
- conf.getInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY,
- /* old name */ conf.getInt("hbase.hstore.compactionThreshold", 3)));
- long shippedCallSizeLimit = (long) minFilesToCompact * HConstants.DEFAULT_BLOCKSIZE;
+ long shippedCallSizeLimit = (long) numofFilesToCompact * this.store.getFamily().getBlocksize();
try {
do {
hasMore = scanner.next(cells, scannerContext);