You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by se...@apache.org on 2013/04/30 00:37:10 UTC
svn commit: r1477392 - in /hbase/branches/0.95/hbase-server/src:
main/java/org/apache/hadoop/hbase/regionserver/compactions/
test/java/org/apache/hadoop/hbase/regionserver/compactions/
Author: sershe
Date: Mon Apr 29 22:37:10 2013
New Revision: 1477392
URL: http://svn.apache.org/r1477392
Log:
HBASE-8448 RatioBasedCompactionPolicy (and derived ones) can select already-compacting files for compaction
Modified:
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java?rev=1477392&r1=1477391&r2=1477392&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java Mon Apr 29 22:37:10 2013
@@ -37,8 +37,6 @@ import org.apache.hadoop.hbase.regionser
@InterfaceAudience.Private
public class ExploringCompactionPolicy extends RatioBasedCompactionPolicy {
- /** Computed number of files that are needed to assume compactions are stuck. */
- private final long filesNeededToForce;
/**
* Constructor for ExploringCompactionPolicy.
@@ -48,20 +46,17 @@ public class ExploringCompactionPolicy e
public ExploringCompactionPolicy(final Configuration conf,
final StoreConfigInformation storeConfigInfo) {
super(conf, storeConfigInfo);
- filesNeededToForce = storeConfigInfo.getBlockingFileCount();
}
@Override
final ArrayList<StoreFile> applyCompactionPolicy(final ArrayList<StoreFile> candidates,
- final boolean mayUseOffPeak) throws IOException {
+ final boolean mayUseOffPeak, final boolean mightBeStuck) throws IOException {
// Start off choosing nothing.
List<StoreFile> bestSelection = new ArrayList<StoreFile>(0);
List<StoreFile> smallest = new ArrayList<StoreFile>(0);
long bestSize = 0;
long smallestSize = Long.MAX_VALUE;
- boolean mightBeStuck = candidates.size() >= filesNeededToForce;
-
// Consider every starting place.
for (int start = 0; start < candidates.size(); start++) {
// Consider every different sub list permutation in between start and end with min files.
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java?rev=1477392&r1=1477391&r2=1477392&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java Mon Apr 29 22:37:10 2013
@@ -82,6 +82,12 @@ public class RatioBasedCompactionPolicy
final boolean mayUseOffPeak, final boolean forceMajor) throws IOException {
// Preliminary compaction subject to filters
ArrayList<StoreFile> candidateSelection = new ArrayList<StoreFile>(candidateFiles);
+ // Stuck and not compacting enough (estimate). It is not guaranteed that we will be
+ // able to compact more if stuck and compacting, because ratio policy excludes some
+ // non-compacting files from consideration during compaction (see getCurrentEligibleFiles).
+ int futureFiles = filesCompacting.isEmpty() ? 0 : 1;
+ boolean mayBeStuck = (candidateFiles.size() - filesCompacting.size() + futureFiles)
+ >= storeConfigInfo.getBlockingFileCount();
candidateSelection = getCurrentEligibleFiles(candidateSelection, filesCompacting);
long cfTtl = this.storeConfigInfo.getStoreFileTtl();
if (!forceMajor) {
@@ -110,18 +116,10 @@ public class RatioBasedCompactionPolicy
if (!majorCompaction) {
// we're doing a minor compaction, let's see what files are applicable
candidateSelection = filterBulk(candidateSelection);
- candidateSelection = applyCompactionPolicy(candidateSelection, mayUseOffPeak);
+ candidateSelection = applyCompactionPolicy(candidateSelection, mayUseOffPeak, mayBeStuck);
candidateSelection = checkMinFilesCriteria(candidateSelection);
}
candidateSelection = removeExcessFiles(candidateSelection, isUserCompaction, majorCompaction);
-
- if (candidateSelection.size() == 0
- && candidateFiles.size() >= storeConfigInfo.getBlockingFileCount()) {
- candidateSelection = new ArrayList<StoreFile>(candidateFiles);
- candidateSelection
- .subList(0, Math.max(0,candidateSelection.size() - comConf.getMinFilesToCompact()))
- .clear();
- }
CompactionRequest result = new CompactionRequest(candidateSelection);
result.setOffPeak(!candidateSelection.isEmpty() && !majorCompaction && mayUseOffPeak);
return result;
@@ -261,8 +259,8 @@ public class RatioBasedCompactionPolicy
* | | | | | | | | | | | |
* | | | | | | | | | | | |
*/
- ArrayList<StoreFile> applyCompactionPolicy(
- ArrayList<StoreFile> candidates, boolean mayUseOffPeak) throws IOException {
+ ArrayList<StoreFile> applyCompactionPolicy(ArrayList<StoreFile> candidates,
+ boolean mayUseOffPeak, boolean mayBeStuck) throws IOException {
if (candidates.isEmpty()) {
return candidates;
}
@@ -298,11 +296,14 @@ public class RatioBasedCompactionPolicy
if (start < countOfFiles) {
LOG.info("Default compaction algorithm has selected " + (countOfFiles - start)
+ " files from " + countOfFiles + " candidates");
+ } else if (mayBeStuck) {
+ // We may be stuck. Compact the latest files if we can.
+ int filesToLeave = candidates.size() - comConf.getMinFilesToCompact();
+ if (filesToLeave >= 0) {
+ start = filesToLeave;
+ }
}
-
- if (start > 0) {
- candidates.subList(0, start).clear();
- }
+ candidates.subList(0, start).clear();
return candidates;
}
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java?rev=1477392&r1=1477391&r2=1477392&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/EverythingPolicy.java Mon Apr 29 22:37:10 2013
@@ -43,7 +43,7 @@ public class EverythingPolicy extends Ra
@Override
final ArrayList<StoreFile> applyCompactionPolicy(final ArrayList<StoreFile> candidates,
- final boolean mayUseOffPeak) throws IOException {
+ final boolean mayUseOffPeak, final boolean mayBeStuck) throws IOException {
if (candidates.size() < comConf.getMinFilesToCompact()) {
return new ArrayList<StoreFile>(0);