You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ec...@apache.org on 2013/04/05 23:27:01 UTC
svn commit: r1465135 - in /hbase/branches/0.95/hbase-server/src:
main/java/org/apache/hadoop/hbase/regionserver/
main/java/org/apache/hadoop/hbase/regionserver/compactions/
test/java/org/apache/hadoop/hbase/regionserver/
test/java/org/apache/hadoop/hba...
Author: eclark
Date: Fri Apr 5 21:27:00 2013
New Revision: 1465135
URL: http://svn.apache.org/r1465135
Log:
HBASE-7842 Add compaction policy that explores more storefile groups
Added:
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
Removed:
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactionPolicy.java
Modified:
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java
hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java
hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java?rev=1465135&r1=1465134&r2=1465135&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreEngine.java Fri Apr 5 21:27:00 2013
@@ -19,7 +19,6 @@
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -27,11 +26,10 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionPolicy;
-import org.apache.hadoop.hbase.regionserver.compactions.Compactor;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactionPolicy;
+import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
import org.apache.hadoop.hbase.util.ReflectionUtils;
+import org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy;
/**
* Default StoreEngine creates the default compactor, policy, and store file manager, or
@@ -39,7 +37,7 @@ import org.apache.hadoop.hbase.util.Refl
*/
@InterfaceAudience.Private
public class DefaultStoreEngine extends StoreEngine<
- DefaultCompactionPolicy, DefaultCompactor, DefaultStoreFileManager> {
+ RatioBasedCompactionPolicy, DefaultCompactor, DefaultStoreFileManager> {
public static final String DEFAULT_COMPACTOR_CLASS_KEY =
"hbase.hstore.defaultengine.compactor.class";
@@ -48,8 +46,8 @@ public class DefaultStoreEngine extends
private static final Class<? extends DefaultCompactor>
DEFAULT_COMPACTOR_CLASS = DefaultCompactor.class;
- private static final Class<? extends DefaultCompactionPolicy>
- DEFAULT_COMPACTION_POLICY_CLASS = DefaultCompactionPolicy.class;
+ private static final Class<? extends RatioBasedCompactionPolicy>
+ DEFAULT_COMPACTION_POLICY_CLASS = ExploringCompactionPolicy.class;
@Override
protected void createComponents(
@@ -98,4 +96,5 @@ public class DefaultStoreEngine extends
storeFileManager.getStorefiles(), filesCompacting);
}
}
+
}
Modified: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java?rev=1465135&r1=1465134&r2=1465135&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java (original)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreEngine.java Fri Apr 5 21:27:00 2013
@@ -27,8 +27,6 @@ import org.apache.hadoop.hbase.KeyValue.
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionPolicy;
import org.apache.hadoop.hbase.regionserver.compactions.Compactor;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactionPolicy;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
import org.apache.hadoop.hbase.util.ReflectionUtils;
/**
Added: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java?rev=1465135&view=auto
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java (added)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java Fri Apr 5 21:27:00 2013
@@ -0,0 +1,89 @@
+package org.apache.hadoop.hbase.regionserver.compactions;
+
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+
+@InterfaceAudience.Private
+public class ExploringCompactionPolicy extends RatioBasedCompactionPolicy {
+
+ public ExploringCompactionPolicy(Configuration conf,
+ StoreConfigInformation storeConfigInfo) {
+ super(conf, storeConfigInfo);
+ }
+
+ @Override
+ ArrayList<StoreFile> applyCompactionPolicy(ArrayList<StoreFile> candidates,
+ boolean mayUseOffPeak) throws IOException {
+ // Start off choosing nothing.
+ List<StoreFile> bestSelection = new ArrayList<StoreFile>(0);
+ long bestSize = 0;
+
+ // Consider every starting place.
+ for (int start = 0; start < candidates.size(); start++) {
+ // Consider every different sub list permutation in between start and end with min files.
+ for(int currentEnd = start + comConf.getMinFilesToCompact() - 1;
+ currentEnd < candidates.size(); currentEnd++) {
+ List<StoreFile> potentialMatchFiles = candidates.subList(start, currentEnd+1);
+
+ // Sanity checks
+ if (potentialMatchFiles.size() < comConf.getMinFilesToCompact()) continue;
+ if (potentialMatchFiles.size() > comConf.getMaxFilesToCompact()) continue;
+ if (!filesInRatio(potentialMatchFiles, mayUseOffPeak)) continue;
+
+ // Compute the total size of files that will
+ // have to be read if this set of files is compacted.
+ long size = 0;
+
+ for (StoreFile s:potentialMatchFiles) {
+ size += s.getReader().length();
+ }
+
+ // Keep if this gets rid of more files. Or the same number of files for less io.
+ if (potentialMatchFiles.size() > bestSelection.size() ||
+ (potentialMatchFiles.size() == bestSelection.size() && size < bestSize)) {
+ bestSelection = potentialMatchFiles;
+ bestSize = size;
+ }
+ }
+ }
+
+ return new ArrayList<StoreFile>(bestSelection);
+ }
+
+ /**
+ * Check that all files satisfy the r
+ * @param files
+ * @return
+ */
+ private boolean filesInRatio(List<StoreFile> files, boolean isOffPeak) {
+ if (files.size() < 2) {
+ return true;
+ }
+ double currentRatio = isOffPeak ?
+ comConf.getCompactionRatioOffPeak() : comConf.getCompactionRatio();
+
+ long totalFileSize = 0;
+ for (int i = 0; i < files.size(); i++) {
+ totalFileSize += files.get(i).getReader().length();
+ }
+ for (int i = 0; i < files.size(); i++) {
+ long singleFileSize = files.get(i).getReader().length();
+ long sumAllOtherFilesize = totalFileSize - singleFileSize;
+
+ if (( singleFileSize > sumAllOtherFilesize * currentRatio)
+ && (sumAllOtherFilesize >= comConf.getMinCompactSize())){
+ return false;
+ }
+ }
+
+ return true;
+
+ }
+}
Added: hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java?rev=1465135&view=auto
==============================================================================
--- hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java (added)
+++ hbase/branches/0.95/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java Fri Apr 5 21:27:00 2013
@@ -0,0 +1,384 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.regionserver.compactions;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.StoreUtils;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.collect.Collections2;
+
+/**
+ * The default algorithm for selecting files for compaction.
+ * Combines the compaction configuration and the provisional file selection that
+ * it's given to produce the list of suitable candidates for compaction.
+ */
+@InterfaceAudience.Private
+public class RatioBasedCompactionPolicy extends CompactionPolicy {
+ private static final Log LOG = LogFactory.getLog(RatioBasedCompactionPolicy.class);
+
+ public RatioBasedCompactionPolicy(Configuration conf,
+ StoreConfigInformation storeConfigInfo) {
+ super(conf, storeConfigInfo);
+ }
+
+ private ArrayList<StoreFile> getCurrentEligibleFiles(
+ ArrayList<StoreFile> candidateFiles, final List<StoreFile> filesCompacting) {
+ // candidates = all storefiles not already in compaction queue
+ if (!filesCompacting.isEmpty()) {
+ // exclude all files older than the newest file we're currently
+ // compacting. this allows us to preserve contiguity (HBASE-2856)
+ StoreFile last = filesCompacting.get(filesCompacting.size() - 1);
+ int idx = candidateFiles.indexOf(last);
+ Preconditions.checkArgument(idx != -1);
+ candidateFiles.subList(0, idx + 1).clear();
+ }
+ return candidateFiles;
+ }
+
+ public List<StoreFile> preSelectCompactionForCoprocessor(
+ final Collection<StoreFile> candidates, final List<StoreFile> filesCompacting) {
+ return getCurrentEligibleFiles(new ArrayList<StoreFile>(candidates), filesCompacting);
+ }
+
+ /**
+ * @param candidateFiles candidate files, ordered from oldest to newest
+ * @return subset copy of candidate list that meets compaction criteria
+ * @throws java.io.IOException
+ */
+ public CompactionRequest selectCompaction(Collection<StoreFile> candidateFiles,
+ final List<StoreFile> filesCompacting, final boolean isUserCompaction,
+ final boolean mayUseOffPeak, final boolean forceMajor) throws IOException {
+ // Preliminary compaction subject to filters
+ ArrayList<StoreFile> candidateSelection = new ArrayList<StoreFile>(candidateFiles);
+ candidateSelection = getCurrentEligibleFiles(candidateSelection, filesCompacting);
+ long cfTtl = this.storeConfigInfo.getStoreFileTtl();
+ if (!forceMajor) {
+ // If there are expired files, only select them so that compaction deletes them
+ if (comConf.shouldDeleteExpired() && (cfTtl != Long.MAX_VALUE)) {
+ ArrayList<StoreFile> expiredSelection = selectExpiredStoreFiles(
+ candidateSelection, EnvironmentEdgeManager.currentTimeMillis() - cfTtl);
+ if (expiredSelection != null) {
+ return new CompactionRequest(expiredSelection);
+ }
+ }
+ candidateSelection = skipLargeFiles(candidateSelection);
+ }
+
+ // Force a major compaction if this is a user-requested major compaction,
+ // or if we do not have too many files to compact and this was requested
+ // as a major compaction.
+ // Or, if there are any references among the candidates.
+ boolean majorCompaction = (
+ (forceMajor && isUserCompaction)
+ || ((forceMajor || isMajorCompaction(candidateSelection))
+ && (candidateSelection.size() < comConf.getMaxFilesToCompact()))
+ || StoreUtils.hasReferences(candidateSelection)
+ );
+
+ if (!majorCompaction) {
+ // we're doing a minor compaction, let's see what files are applicable
+ candidateSelection = filterBulk(candidateSelection);
+ candidateSelection = applyCompactionPolicy(candidateSelection, mayUseOffPeak);
+ candidateSelection = checkMinFilesCriteria(candidateSelection);
+ }
+ candidateSelection = removeExcessFiles(candidateSelection, isUserCompaction, majorCompaction);
+ CompactionRequest result = new CompactionRequest(candidateSelection);
+ result.setOffPeak(!candidateSelection.isEmpty() && !majorCompaction && mayUseOffPeak);
+ return result;
+ }
+
+ /**
+ * Select the expired store files to compact
+ *
+ * @param candidates the initial set of storeFiles
+ * @param maxExpiredTimeStamp
+ * The store file will be marked as expired if its max time stamp is
+ * less than this maxExpiredTimeStamp.
+ * @return A CompactSelection contains the expired store files as
+ * filesToCompact
+ */
+ private ArrayList<StoreFile> selectExpiredStoreFiles(
+ ArrayList<StoreFile> candidates, long maxExpiredTimeStamp) {
+ if (candidates == null || candidates.size() == 0) return null;
+ ArrayList<StoreFile> expiredStoreFiles = null;
+
+ for (StoreFile storeFile : candidates) {
+ if (storeFile.getReader().getMaxTimestamp() < maxExpiredTimeStamp) {
+ LOG.info("Deleting the expired store file by compaction: "
+ + storeFile.getPath() + " whose maxTimeStamp is "
+ + storeFile.getReader().getMaxTimestamp()
+ + " while the max expired timestamp is " + maxExpiredTimeStamp);
+ if (expiredStoreFiles == null) {
+ expiredStoreFiles = new ArrayList<StoreFile>();
+ }
+ expiredStoreFiles.add(storeFile);
+ }
+ }
+
+ return expiredStoreFiles;
+ }
+
+ /**
+ * @param candidates pre-filtrate
+ * @return filtered subset
+ * exclude all files above maxCompactSize
+ * Also save all references. We MUST compact them
+ */
+ private ArrayList<StoreFile> skipLargeFiles(ArrayList<StoreFile> candidates) {
+ int pos = 0;
+ while (pos < candidates.size() && !candidates.get(pos).isReference()
+ && (candidates.get(pos).getReader().length() > comConf.getMaxCompactSize())) {
+ ++pos;
+ }
+ if (pos > 0) {
+ LOG.debug("Some files are too large. Excluding " + pos
+ + " files from compaction candidates");
+ candidates.subList(0, pos).clear();
+ }
+ return candidates;
+ }
+
+ /**
+ * @param candidates pre-filtrate
+ * @return filtered subset
+ * exclude all bulk load files if configured
+ */
+ private ArrayList<StoreFile> filterBulk(ArrayList<StoreFile> candidates) {
+ candidates.removeAll(Collections2.filter(candidates,
+ new Predicate<StoreFile>() {
+ @Override
+ public boolean apply(StoreFile input) {
+ return input.excludeFromMinorCompaction();
+ }
+ }));
+ return candidates;
+ }
+
+ /**
+ * @param candidates pre-filtrate
+ * @return filtered subset
+ * take upto maxFilesToCompact from the start
+ */
+ private ArrayList<StoreFile> removeExcessFiles(ArrayList<StoreFile> candidates,
+ boolean isUserCompaction, boolean isMajorCompaction) {
+ int excess = candidates.size() - comConf.getMaxFilesToCompact();
+ if (excess > 0) {
+ if (isMajorCompaction && isUserCompaction) {
+ LOG.debug("Warning, compacting more than " + comConf.getMaxFilesToCompact() +
+ " files because of a user-requested major compaction");
+ } else {
+ LOG.debug("Too many admissible files. Excluding " + excess
+ + " files from compaction candidates");
+ candidates.subList(comConf.getMaxFilesToCompact(), candidates.size()).clear();
+ }
+ }
+ return candidates;
+ }
+ /**
+ * @param candidates pre-filtrate
+ * @return filtered subset
+ * forget the compactionSelection if we don't have enough files
+ */
+ private ArrayList<StoreFile> checkMinFilesCriteria(ArrayList<StoreFile> candidates) {
+ int minFiles = comConf.getMinFilesToCompact();
+ if (candidates.size() < minFiles) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Not compacting files because we only have " + candidates.size() +
+ " files ready for compaction. Need " + minFiles + " to initiate.");
+ }
+ candidates.clear();
+ }
+ return candidates;
+ }
+
+ /**
+ * @param candidates pre-filtrate
+ * @return filtered subset
+ * -- Default minor compaction selection algorithm:
+ * choose CompactSelection from candidates --
+ * First exclude bulk-load files if indicated in configuration.
+ * Start at the oldest file and stop when you find the first file that
+ * meets compaction criteria:
+ * (1) a recently-flushed, small file (i.e. <= minCompactSize)
+ * OR
+ * (2) within the compactRatio of sum(newer_files)
+ * Given normal skew, any newer files will also meet this criteria
+ * <p/>
+ * Additional Note:
+ * If fileSizes.size() >> maxFilesToCompact, we will recurse on
+ * compact(). Consider the oldest files first to avoid a
+ * situation where we always compact [end-threshold,end). Then, the
+ * last file becomes an aggregate of the previous compactions.
+ *
+ * normal skew:
+ *
+ * older ----> newer (increasing seqID)
+ * _
+ * | | _
+ * | | | | _
+ * --|-|- |-|- |-|---_-------_------- minCompactSize
+ * | | | | | | | | _ | |
+ * | | | | | | | | | | | |
+ * | | | | | | | | | | | |
+ */
+ ArrayList<StoreFile> applyCompactionPolicy(
+ ArrayList<StoreFile> candidates, boolean mayUseOffPeak) throws IOException {
+ if (candidates.isEmpty()) {
+ return candidates;
+ }
+
+ // we're doing a minor compaction, let's see what files are applicable
+ int start = 0;
+ double ratio = comConf.getCompactionRatio();
+ if (mayUseOffPeak) {
+ ratio = comConf.getCompactionRatioOffPeak();
+ LOG.info("Running an off-peak compaction, selection ratio = " + ratio);
+ }
+
+ // get store file sizes for incremental compacting selection.
+ final int countOfFiles = candidates.size();
+ long[] fileSizes = new long[countOfFiles];
+ long[] sumSize = new long[countOfFiles];
+ for (int i = countOfFiles - 1; i >= 0; --i) {
+ StoreFile file = candidates.get(i);
+ fileSizes[i] = file.getReader().length();
+ // calculate the sum of fileSizes[i,i+maxFilesToCompact-1) for algo
+ int tooFar = i + comConf.getMaxFilesToCompact() - 1;
+ sumSize[i] = fileSizes[i]
+ + ((i + 1 < countOfFiles) ? sumSize[i + 1] : 0)
+ - ((tooFar < countOfFiles) ? fileSizes[tooFar] : 0);
+ }
+
+
+ while (countOfFiles - start >= comConf.getMinFilesToCompact() &&
+ fileSizes[start] > Math.max(comConf.getMinCompactSize(),
+ (long) (sumSize[start + 1] * ratio))) {
+ ++start;
+ }
+ if (start < countOfFiles) {
+ LOG.info("Default compaction algorithm has selected " + (countOfFiles - start)
+ + " files from " + countOfFiles + " candidates");
+ }
+
+ if (start > 0) {
+ candidates.subList(0, start).clear();
+ }
+ return candidates;
+ }
+
+ /*
+ * @param filesToCompact Files to compact. Can be null.
+ * @return True if we should run a major compaction.
+ */
+ public boolean isMajorCompaction(final Collection<StoreFile> filesToCompact)
+ throws IOException {
+ boolean result = false;
+ long mcTime = getNextMajorCompactTime(filesToCompact);
+ if (filesToCompact == null || filesToCompact.isEmpty() || mcTime == 0) {
+ return result;
+ }
+ // TODO: Use better method for determining stamp of last major (HBASE-2990)
+ long lowTimestamp = StoreUtils.getLowestTimestamp(filesToCompact);
+ long now = System.currentTimeMillis();
+ if (lowTimestamp > 0l && lowTimestamp < (now - mcTime)) {
+ // Major compaction time has elapsed.
+ long cfTtl = this.storeConfigInfo.getStoreFileTtl();
+ if (filesToCompact.size() == 1) {
+ // Single file
+ StoreFile sf = filesToCompact.iterator().next();
+ Long minTimestamp = sf.getMinimumTimestamp();
+ long oldest = (minTimestamp == null)
+ ? Long.MIN_VALUE
+ : now - minTimestamp.longValue();
+ if (sf.isMajorCompaction() &&
+ (cfTtl == HConstants.FOREVER || oldest < cfTtl)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping major compaction of " + this +
+ " because one (major) compacted file only and oldestTime " +
+ oldest + "ms is < ttl=" + cfTtl);
+ }
+ } else if (cfTtl != HConstants.FOREVER && oldest > cfTtl) {
+ LOG.debug("Major compaction triggered on store " + this +
+ ", because keyvalues outdated; time since last major compaction " +
+ (now - lowTimestamp) + "ms");
+ result = true;
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Major compaction triggered on store " + this +
+ "; time since last major compaction " + (now - lowTimestamp) + "ms");
+ }
+ result = true;
+ }
+ }
+ return result;
+ }
+
+ public long getNextMajorCompactTime(final Collection<StoreFile> filesToCompact) {
+ // default = 24hrs
+ long ret = comConf.getMajorCompactionPeriod();
+ if (ret > 0) {
+ // default = 20% = +/- 4.8 hrs
+ double jitterPct = comConf.getMajorCompactionJitter();
+ if (jitterPct > 0) {
+ long jitter = Math.round(ret * jitterPct);
+ // deterministic jitter avoids a major compaction storm on restart
+ Integer seed = StoreUtils.getDeterministicRandomSeed(filesToCompact);
+ if (seed != null) {
+ double rnd = (new Random(seed)).nextDouble();
+ ret += jitter - Math.round(2L * jitter * rnd);
+ } else {
+ ret = 0; // no storefiles == no major compaction
+ }
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * @param compactionSize Total size of some compaction
+ * @return whether this should be a large or small compaction
+ */
+ public boolean throttleCompaction(long compactionSize) {
+ return compactionSize > comConf.getThrottlePoint();
+ }
+
+ @Override
+ public boolean needsCompaction(final Collection<StoreFile> storeFiles,
+ final List<StoreFile> filesCompacting) {
+ int numCandidates = storeFiles.size() - filesCompacting.size();
+ return numCandidates > comConf.getMinFilesToCompact();
+ }
+}
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java?rev=1465135&r1=1465134&r2=1465135&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java Fri Apr 5 21:27:00 2013
@@ -55,7 +55,7 @@ import org.apache.hadoop.hbase.io.hfile.
import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.regionserver.compactions.Compactor;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactionPolicy;
+import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@@ -321,7 +321,8 @@ public class TestCompaction extends HBas
assertEquals(2, s.getStorefilesCount());
// ensure that major compaction time is deterministic
- DefaultCompactionPolicy c = (DefaultCompactionPolicy)s.storeEngine.getCompactionPolicy();
+ RatioBasedCompactionPolicy
+ c = (RatioBasedCompactionPolicy)s.storeEngine.getCompactionPolicy();
Collection<StoreFile> storeFiles = s.getStorefiles();
long mcTime = c.getNextMajorCompactTime(storeFiles);
for (int i = 0; i < 10; ++i) {
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java?rev=1465135&r1=1465134&r2=1465135&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultCompactSelection.java Fri Apr 5 21:27:00 2013
@@ -20,8 +20,6 @@ package org.apache.hadoop.hbase.regionse
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Calendar;
-import java.util.GregorianCalendar;
import java.util.List;
import junit.framework.TestCase;
@@ -40,7 +38,7 @@ import org.apache.hadoop.hbase.SmallTest
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactionPolicy;
+import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
import org.apache.hadoop.hbase.util.Bytes;
@@ -171,6 +169,16 @@ public class TestDefaultCompactSelection
}
};
}
+
+ @Override
+ public String toString() {
+ return "MockStoreFile{" +
+ "length=" + length +
+ ", isRef=" + isRef +
+ ", ageInDisk=" + ageInDisk +
+ ", sequenceid=" + sequenceid +
+ '}';
+ }
}
ArrayList<Long> toArrayList(long... numbers) {
@@ -234,7 +242,7 @@ public class TestDefaultCompactSelection
throws IOException {
store.forceMajor = forcemajor;
//Test Default compactions
- CompactionRequest result = ((DefaultCompactionPolicy)store.storeEngine.getCompactionPolicy())
+ CompactionRequest result = ((RatioBasedCompactionPolicy)store.storeEngine.getCompactionPolicy())
.selectCompaction(candidates, new ArrayList<StoreFile>(), false, isOffPeak, forcemajor);
List<StoreFile> actual = new ArrayList<StoreFile>(result.getFiles());
if (isOffPeak && !forcemajor) {
@@ -269,7 +277,13 @@ public class TestDefaultCompactSelection
*/
// don't exceed max file compact threshold
// note: file selection starts with largest to smallest.
- compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 7, 6, 5, 4, 3);
+ compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
+
+ compactEquals(sfCreate(50, 10, 10 ,10, 10), 10, 10, 10, 10);
+
+ compactEquals(sfCreate(10, 10, 10, 10, 50), 10, 10, 10, 10);
+
+ compactEquals(sfCreate(251, 253, 251, maxSize -1), 251, 253, 251);
/* MAJOR COMPACTION */
// if a major compaction has been forced, then compact everything
@@ -280,7 +294,7 @@ public class TestDefaultCompactSelection
compactEquals(sfCreate(tooBig, 12,12), true, tooBig, 12, 12);
// don't exceed max file compact threshold, even with major compaction
store.forceMajor = true;
- compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 7, 6, 5, 4, 3);
+ compactEquals(sfCreate(7, 6, 5, 4, 3, 2, 1), 5, 4, 3, 2, 1);
store.forceMajor = false;
// if we exceed maxCompactSize, downgrade to minor
// if not, it creates a 'snowball effect' when files >> maxCompactSize:
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java?rev=1465135&r1=1465134&r2=1465135&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultStoreEngine.java Fri Apr 5 21:27:00 2013
@@ -23,7 +23,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
import org.apache.hadoop.hbase.SmallTests;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactionPolicy;
+import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
import org.junit.Assert;
import org.junit.Test;
@@ -38,7 +38,7 @@ public class TestDefaultStoreEngine {
}
}
- public static class DummyCompactionPolicy extends DefaultCompactionPolicy {
+ public static class DummyCompactionPolicy extends RatioBasedCompactionPolicy {
public DummyCompactionPolicy(Configuration conf, StoreConfigInformation storeConfigInfo) {
super(conf, storeConfigInfo);
}
Modified: hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java?rev=1465135&r1=1465134&r2=1465135&view=diff
==============================================================================
--- hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java (original)
+++ hbase/branches/0.95/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/PerfTestCompactionPolicies.java Fri Apr 5 21:27:00 2013
@@ -26,7 +26,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.SmallTests;
-import org.apache.hadoop.hbase.KeyValue.KVComparator;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -53,7 +52,7 @@ public class PerfTestCompactionPolicies
static final Log LOG = LogFactory.getLog(PerfTestCompactionPolicies.class);
- private final DefaultCompactionPolicy cp;
+ private final RatioBasedCompactionPolicy cp;
private final int max;
private final int min;
private final float ratio;
@@ -64,11 +63,16 @@ public class PerfTestCompactionPolicies
@Parameterized.Parameters
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
- {DefaultCompactionPolicy.class, 3, 2, 1.2f},
- {DefaultCompactionPolicy.class, 4, 2, 1.2f},
- {DefaultCompactionPolicy.class, 5, 2, 1.2f},
- {DefaultCompactionPolicy.class, 4, 2, 1.3f},
- {DefaultCompactionPolicy.class, 4, 2, 1.4f},
+ {RatioBasedCompactionPolicy.class, 3, 2, 1.2f},
+ {ExploringCompactionPolicy.class, 3, 2, 1.2f},
+ {RatioBasedCompactionPolicy.class, 4, 2, 1.2f},
+ {ExploringCompactionPolicy.class, 4, 2, 1.2f},
+ {RatioBasedCompactionPolicy.class, 5, 2, 1.2f},
+ {ExploringCompactionPolicy.class, 5, 2, 1.2f},
+ {RatioBasedCompactionPolicy.class, 4, 2, 1.3f},
+ {ExploringCompactionPolicy.class, 4, 2, 1.3f},
+ {RatioBasedCompactionPolicy.class, 4, 2, 1.4f},
+ {ExploringCompactionPolicy.class, 4, 2, 1.4f},
});
}