You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by se...@apache.org on 2013/05/30 19:55:04 UTC
svn commit: r1487948 - in
/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver:
CompactSplitThread.java compactions/ExploringCompactionPolicy.java
compactions/RatioBasedCompactionPolicy.java
Author: sershe
Date: Thu May 30 17:55:04 2013
New Revision: 1487948
URL: http://svn.apache.org/r1487948
Log:
HBASE-8638 add logging to compaction policy
Modified:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=1487948&r1=1487947&r2=1487948&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java Thu May 30 17:55:04 2013
@@ -19,6 +19,8 @@
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -395,7 +397,11 @@ public class CompactSplitThread implemen
}
}
} catch (IOException ex) {
- LOG.error("Compaction failed " + this, RemoteExceptionHandler.checkIOException(ex));
+ IOException remoteEx = RemoteExceptionHandler.checkIOException(ex);
+ LOG.error("Compaction failed " + this, remoteEx);
+ if (remoteEx != ex) {
+ LOG.info("Compaction failed at original callstack: " + formatStackTrace(ex));
+ }
server.checkFileSystem();
} catch (Exception ex) {
LOG.error("Compaction failed " + this, ex);
@@ -406,6 +412,14 @@ public class CompactSplitThread implemen
this.compaction.getRequest().afterExecute();
}
+ private String formatStackTrace(Exception ex) {
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ ex.printStackTrace(pw);
+ pw.flush();
+ return sw.toString();
+ }
+
@Override
public int compareTo(CompactionRunner o) {
// Only compare the underlying request, for queue sorting purposes.
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java?rev=1487948&r1=1487947&r2=1487948&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/ExploringCompactionPolicy.java Thu May 30 17:55:04 2013
@@ -23,6 +23,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.regionserver.StoreConfigInformation;
@@ -36,7 +38,7 @@ import org.apache.hadoop.hbase.regionser
*/
@InterfaceAudience.Private
public class ExploringCompactionPolicy extends RatioBasedCompactionPolicy {
-
+ private static final Log LOG = LogFactory.getLog(ExploringCompactionPolicy.class);
/**
* Constructor for ExploringCompactionPolicy.
@@ -57,6 +59,7 @@ public class ExploringCompactionPolicy e
long bestSize = 0;
long smallestSize = Long.MAX_VALUE;
+ int opts = 0, optsInRatio = 0, bestStart = -1; // for debug logging
// Consider every starting place.
for (int start = 0; start < candidates.size(); start++) {
// Consider every different sub list permutation in between start and end with min files.
@@ -83,26 +86,34 @@ public class ExploringCompactionPolicy e
smallestSize = size;
}
- if (size >= comConf.getMinCompactSize()
- && !filesInRatio(potentialMatchFiles, mayUseOffPeak)) {
+ if (size > comConf.getMaxCompactSize()) {
continue;
}
- if (size > comConf.getMaxCompactSize()) {
+ ++opts;
+ if (size >= comConf.getMinCompactSize()
+ && !filesInRatio(potentialMatchFiles, mayUseOffPeak)) {
continue;
}
+ ++optsInRatio;
// Keep if this gets rid of more files. Or the same number of files for less io.
if (potentialMatchFiles.size() > bestSelection.size()
|| (potentialMatchFiles.size() == bestSelection.size() && size < bestSize)) {
bestSelection = potentialMatchFiles;
bestSize = size;
+ bestStart = start;
}
}
}
if (bestSelection.size() == 0 && mightBeStuck) {
+ LOG.debug("Exploring compaction algorithm has selected " + smallest.size()
+ + " files of size "+ smallestSize + " because the store might be stuck");
return new ArrayList<StoreFile>(smallest);
}
+ LOG.debug("Exploring compaction algorithm has selected " + bestSelection.size()
+ + " files of size " + bestSize + " starting at candidate #" + bestStart +
+ " after considering " + opts + " permutations with " + optsInRatio + " in ratio");
return new ArrayList<StoreFile>(bestSelection);
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java?rev=1487948&r1=1487947&r2=1487948&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java Thu May 30 17:55:04 2013
@@ -89,6 +89,10 @@ public class RatioBasedCompactionPolicy
boolean mayBeStuck = (candidateFiles.size() - filesCompacting.size() + futureFiles)
>= storeConfigInfo.getBlockingFileCount();
candidateSelection = getCurrentEligibleFiles(candidateSelection, filesCompacting);
+ LOG.debug("Selecting compaction from " + candidateFiles.size() + " store files, " +
+ filesCompacting.size() + " compacting, " + candidateSelection.size() +
+ " eligible, " + storeConfigInfo.getBlockingFileCount() + " blocking");
+
long cfTtl = this.storeConfigInfo.getStoreFileTtl();
if (!forceMajor) {
// If there are expired files, only select them so that compaction deletes them