You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jy...@apache.org on 2013/02/28 01:33:06 UTC
svn commit: r1451057 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/coprocessor/
main/java/org/apache/hadoop/hbase/regionserver/
main/java/org/apache/hadoop/hbase/regionserver/compactions/
test/java/org/apache/hadoop/hbase/regionserver/
Author: jyates
Date: Thu Feb 28 00:33:06 2013
New Revision: 1451057
URL: http://svn.apache.org/r1451057
Log:
HBASE-7725: Add ability to create custom compaction request
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java Thu Feb 28 00:33:06 2013
@@ -41,6 +41,8 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactSelection;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.util.Pair;
@@ -110,10 +112,23 @@ public abstract class BaseRegionObserver
@Override
public void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Store store, final List<StoreFile> candidates, final CompactionRequest request)
+ throws IOException {
+ preCompactSelection(c, store, candidates);
+ }
+
+ @Override
+ public void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
final Store store, final List<StoreFile> candidates) throws IOException { }
@Override
public void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Store store, final ImmutableList<StoreFile> selected, CompactionRequest request) {
+ postCompactSelection(c, store, selected);
+ }
+
+ @Override
+ public void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
final Store store, final ImmutableList<StoreFile> selected) { }
@Override
@@ -123,6 +138,13 @@ public abstract class BaseRegionObserver
}
@Override
+ public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
+ final Store store, final InternalScanner scanner, CompactionRequest request)
+ throws IOException {
+ return preCompact(e, store, scanner);
+ }
+
+ @Override
public InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
final Store store, List<? extends KeyValueScanner> scanners, final ScanType scanType,
final long earliestPutTs, final InternalScanner s) throws IOException {
@@ -130,11 +152,25 @@ public abstract class BaseRegionObserver
}
@Override
+ public InternalScanner preCompactScannerOpen(
+ final ObserverContext<RegionCoprocessorEnvironment> c, final Store store,
+ List<? extends KeyValueScanner> scanners, final ScanType scanType, final long earliestPutTs,
+ final InternalScanner s, CompactionRequest request) throws IOException {
+ return preCompactScannerOpen(c, store, scanners, scanType, earliestPutTs, s);
+ }
+
+ @Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, final Store store,
final StoreFile resultFile) throws IOException {
}
@Override
+ public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, final Store store,
+ final StoreFile resultFile, CompactionRequest request) throws IOException {
+ postCompact(e, store, resultFile);
+ }
+
+ @Override
public void preGetClosestRowBefore(final ObserverContext<RegionCoprocessorEnvironment> e,
final byte [] row, final byte [] family, final Result result)
throws IOException {
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java Thu Feb 28 00:33:06 2013
@@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
@@ -125,9 +126,23 @@ public interface RegionObserver extends
final StoreFile resultFile) throws IOException;
/**
- * Called prior to selecting the {@link StoreFile}s to compact from the list
- * of available candidates. To alter the files used for compaction, you may
- * mutate the passed in list of candidates.
+ * Called prior to selecting the {@link StoreFile StoreFiles} to compact from the list of
+ * available candidates. To alter the files used for compaction, you may mutate the passed in list
+ * of candidates.
+ * @param c the environment provided by the region server
+ * @param store the store where compaction is being requested
+ * @param candidates the store files currently available for compaction
+ * @param request custom compaction request
+ * @throws IOException if an error occurred on the coprocessor
+ */
+ void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Store store, final List<StoreFile> candidates, final CompactionRequest request)
+ throws IOException;
+
+ /**
+ * Called prior to selecting the {@link StoreFile}s to compact from the list of available
+ * candidates. To alter the files used for compaction, you may mutate the passed in list of
+ * candidates.
* @param c the environment provided by the region server
* @param store the store where compaction is being requested
* @param candidates the store files currently available for compaction
@@ -147,9 +162,20 @@ public interface RegionObserver extends
final Store store, final ImmutableList<StoreFile> selected);
/**
- * Called prior to writing the {@link StoreFile}s selected for compaction into
- * a new {@code StoreFile}. To override or modify the compaction process,
- * implementing classes have two options:
+ * Called after the {@link StoreFile}s to compact have been selected from the available
+ * candidates.
+ * @param c the environment provided by the region server
+ * @param store the store being compacted
+ * @param selected the store files selected to compact
+ * @param request custom compaction request
+ */
+ void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Store store, final ImmutableList<StoreFile> selected, CompactionRequest request);
+
+ /**
+ * Called prior to writing the {@link StoreFile}s selected for compaction into a new
+ * {@code StoreFile}. To override or modify the compaction process, implementing classes have two
+ * options:
* <ul>
* <li>Wrap the provided {@link InternalScanner} with a custom
* implementation that is returned from this method. The custom scanner
@@ -164,32 +190,55 @@ public interface RegionObserver extends
* </ul>
* @param c the environment provided by the region server
* @param store the store being compacted
- * @param scanner the scanner over existing data used in the store file
- * rewriting
- * @return the scanner to use during compaction. Should not be {@code null}
- * unless the implementation is writing new store files on its own.
+ * @param scanner the scanner over existing data used in the store file rewriting
+ * @return the scanner to use during compaction. Should not be {@code null} unless the
+ * implementation is writing new store files on its own.
* @throws IOException if an error occurred on the coprocessor
*/
InternalScanner preCompact(final ObserverContext<RegionCoprocessorEnvironment> c,
final Store store, final InternalScanner scanner) throws IOException;
/**
- * Called prior to writing the {@link StoreFile}s selected for compaction into
- * a new {@code StoreFile} and prior to creating the scanner used to read the
- * input files. To override or modify the compaction process,
- * implementing classes can return a new scanner to provide the KeyValues to be
- * stored into the new {@code StoreFile} or null to perform the default processing.
- * Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
+ * Called prior to writing the {@link StoreFile}s selected for compaction into a new
+ * {@code StoreFile}. To override or modify the compaction process, implementing classes have two
+ * options:
+ * <ul>
+ * <li>Wrap the provided {@link InternalScanner} with a custom implementation that is returned
+ * from this method. The custom scanner can then inspect {@link KeyValue}s from the wrapped
+ * scanner, applying its own policy to what gets written.</li>
+ * <li>Call {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} and provide a
+ * custom implementation for writing of new {@link StoreFile}s. <strong>Note: any implementations
+ * bypassing core compaction using this approach must write out new store files themselves or the
+ * existing data will no longer be available after compaction.</strong></li>
+ * </ul>
+ * @param c the environment provided by the region server
+ * @param store the store being compacted
+ * @param scanner the scanner over existing data used in the store file rewriting
+ * @param request the requested compaction
+ * @return the scanner to use during compaction. Should not be {@code null} unless the
+ * implementation is writing new store files on its own.
+ * @throws IOException if an error occurred on the coprocessor
+ */
+ InternalScanner preCompact(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Store store, final InternalScanner scanner, CompactionRequest request)
+ throws IOException;
+
+ /**
+ * Called prior to writing the {@link StoreFile}s selected for compaction into a new
+ * {@code StoreFile} and prior to creating the scanner used to read the input files. To override
+ * or modify the compaction process, implementing classes can return a new scanner to provide the
+ * KeyValues to be stored into the new {@code StoreFile} or null to perform the default
+ * processing. Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
* effect in this hook.
* @param c the environment provided by the region server
* @param store the store being compacted
* @param scanners the list {@link StoreFileScanner}s to be read from
* @param scantype the {@link ScanType} indicating whether this is a major or minor compaction
- * @param earliestPutTs timestamp of the earliest put that was found in any of the involved
- * store files
+ * @param earliestPutTs timestamp of the earliest put that was found in any of the involved store
+ * files
* @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain
- * @return the scanner to use during compaction. {@code null} if the default implementation
- * is to be used.
+ * @return the scanner to use during compaction. {@code null} if the default implementation is to
+ * be used.
* @throws IOException if an error occurred on the coprocessor
*/
InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
@@ -197,6 +246,30 @@ public interface RegionObserver extends
final long earliestPutTs, final InternalScanner s) throws IOException;
/**
+ * Called prior to writing the {@link StoreFile}s selected for compaction into a new
+ * {@code StoreFile} and prior to creating the scanner used to read the input files. To override
+ * or modify the compaction process, implementing classes can return a new scanner to provide the
+ * KeyValues to be stored into the new {@code StoreFile} or null to perform the default
+ * processing. Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
+ * effect in this hook.
+ * @param c the environment provided by the region server
+ * @param store the store being compacted
+ * @param scanners the list {@link StoreFileScanner}s to be read from
+ * @param scanType the {@link ScanType} indicating whether this is a major or minor compaction
+ * @param earliestPutTs timestamp of the earliest put that was found in any of the involved store
+ * files
+ * @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain
+ * @param request the requested compaction
+ * @return the scanner to use during compaction. {@code null} if the default implementation is to
+ * be used.
+ * @throws IOException if an error occurred on the coprocessor
+ */
+ InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+ final Store store, List<? extends KeyValueScanner> scanners, final ScanType scanType,
+ final long earliestPutTs, final InternalScanner s, CompactionRequest request)
+ throws IOException;
+
+ /**
* Called after compaction has completed and the new store file has been
* moved in to place.
* @param c the environment provided by the region server
@@ -208,9 +281,20 @@ public interface RegionObserver extends
StoreFile resultFile) throws IOException;
/**
- * Called before the region is split.
+ * Called after compaction has completed and the new store file has been moved in to place.
* @param c the environment provided by the region server
- * (e.getRegion() returns the parent region)
+ * @param store the store being compacted
+ * @param resultFile the new store file written out during compaction
+ * @param request the requested compaction
+ * @throws IOException if an error occurred on the coprocessor
+ */
+ void postCompact(final ObserverContext<RegionCoprocessorEnvironment> c, final Store store,
+ StoreFile resultFile, CompactionRequest request) throws IOException;
+
+ /**
+ * Called before the region is split.
+ * @param c the environment provided by the region server (e.getRegion() returns the parent
+ * region)
* @throws IOException if an error occurred on the coprocessor
*/
void preSplit(final ObserverContext<RegionCoprocessorEnvironment> c) throws IOException;
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java Thu Feb 28 00:33:06 2013
@@ -21,7 +21,9 @@ package org.apache.hadoop.hbase.regionse
import java.io.IOException;
import java.util.concurrent.Executors;
+import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.PriorityBlockingQueue;
import java.util.concurrent.RejectedExecutionException;
@@ -184,37 +186,51 @@ public class CompactSplitThread implemen
}
}
- public synchronized void requestCompaction(final HRegion r,
- final String why) throws IOException {
- for(Store s : r.getStores().values()) {
- requestCompaction(r, s, why, Store.NO_PRIORITY);
- }
+ @Override
+ public synchronized List<CompactionRequest> requestCompaction(final HRegion r, final String why)
+ throws IOException {
+ return requestCompaction(r, why, null);
}
- public synchronized void requestCompaction(final HRegion r, final Store s,
- final String why) throws IOException {
- requestCompaction(r, s, why, Store.NO_PRIORITY);
+ @Override
+ public synchronized List<CompactionRequest> requestCompaction(final HRegion r, final String why,
+ List<CompactionRequest> requests) throws IOException {
+ return requestCompaction(r, why, Store.NO_PRIORITY, requests);
}
- public synchronized void requestCompaction(final HRegion r, final String why,
- int p) throws IOException {
- for(Store s : r.getStores().values()) {
- requestCompaction(r, s, why, p);
+ @Override
+ public synchronized CompactionRequest requestCompaction(final HRegion r, final Store s,
+ final String why,
+ CompactionRequest request) throws IOException {
+ return requestCompaction(r, s, why, Store.NO_PRIORITY, request);
+ }
+
+ @Override
+ public synchronized List<CompactionRequest> requestCompaction(final HRegion r, final String why,
+ int pri, final List<CompactionRequest> requests) throws IOException {
+ List<CompactionRequest> ret;
+ // not a special compaction request, so make out own list
+ if (requests == null) {
+ ret = new ArrayList<CompactionRequest>(r.getStores().size());
+ for (Store s : r.getStores().values()) {
+ ret.add(requestCompaction(r, s, why, pri, null));
+ }
+ } else {
+ ret = new ArrayList<CompactionRequest>(requests.size());
+ for (CompactionRequest request : requests) {
+ ret.add(requestCompaction(r, request.getStore(), why, pri, request));
+ }
}
+ return ret;
}
- /**
- * @param r HRegion store belongs to
- * @param s Store to request compaction on
- * @param why Why compaction requested -- used in debug messages
- * @param priority override the default priority (NO_PRIORITY == decide)
- */
- public synchronized void requestCompaction(final HRegion r, final Store s,
- final String why, int priority) throws IOException {
+ @Override
+ public synchronized CompactionRequest requestCompaction(final HRegion r, final Store s,
+ final String why, int priority, CompactionRequest request) throws IOException {
if (this.server.isStopped()) {
- return;
+ return null;
}
- CompactionRequest cr = s.requestCompaction(priority);
+ CompactionRequest cr = s.requestCompaction(priority, request);
if (cr != null) {
cr.setServer(server);
if (priority != Store.NO_PRIORITY) {
@@ -235,6 +251,7 @@ public class CompactSplitThread implemen
" because compaction request was cancelled");
}
}
+ return cr;
}
/**
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java Thu Feb 28 00:33:06 2013
@@ -20,39 +20,71 @@
package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
public interface CompactionRequestor {
/**
* @param r Region to compact
* @param why Why compaction was requested -- used in debug messages
+ * @return the created {@link CompactionRequest CompactionRequets} or, if no compactions were
+ * started, an empty list
+ * @throws IOException
+ */
+ public List<CompactionRequest> requestCompaction(final HRegion r, final String why)
+ throws IOException;
+
+ /**
+ * @param r Region to compact
+ * @param why Why compaction was requested -- used in debug messages
+ * @param requests custom compaction requests. Each compaction must specify the store on which it
+ * is acting. Can be <tt>null</tt> in which case a compaction will be attempted on all
+ * stores for the region.
+ * @return The created {@link CompactionRequest CompactionRequests} or an empty list if no
+ * compactions were started
* @throws IOException
*/
- public void requestCompaction(final HRegion r, final String why) throws IOException;
+ public List<CompactionRequest> requestCompaction(final HRegion r, final String why,
+ List<CompactionRequest> requests) throws IOException;
/**
* @param r Region to compact
* @param s Store within region to compact
* @param why Why compaction was requested -- used in debug messages
+ * @param request custom compaction request for the {@link HRegion} and {@link Store}. Custom
+ * request must be <tt>null</tt> or be constructed with matching region and store.
+ * @return The created {@link CompactionRequest} or <tt>null</tt> if no compaction was started.
* @throws IOException
*/
- public void requestCompaction(final HRegion r, final Store s, final String why) throws IOException;
+ public CompactionRequest requestCompaction(final HRegion r, final Store s, final String why,
+ CompactionRequest request) throws IOException;
/**
* @param r Region to compact
* @param why Why compaction was requested -- used in debug messages
* @param pri Priority of this compaction. minHeap. <=0 is critical
+ * @param requests custom compaction requests. Each compaction must specify the store on which it
+ * is acting. Can be <tt>null</tt> in which case a compaction will be attempted on all
+ * stores for the region.
+ * @return The created {@link CompactionRequest CompactionRequests} or an empty list if no
+ * compactions were started.
* @throws IOException
*/
- public void requestCompaction(final HRegion r, final String why, int pri) throws IOException;
+ public List<CompactionRequest> requestCompaction(final HRegion r, final String why, int pri,
+ List<CompactionRequest> requests) throws IOException;
/**
* @param r Region to compact
* @param s Store within region to compact
* @param why Why compaction was requested -- used in debug messages
* @param pri Priority of this compaction. minHeap. <=0 is critical
+ * @param request request custom compaction request to run. {@link Store} and {@link HRegion} for
+ * the request must match the region and store specified here.
+ * @return The created {@link CompactionRequest} or <tt>null</tt> if no compaction was started
* @throws IOException
*/
- public void requestCompaction(final HRegion r, final Store s,
- final String why, int pri) throws IOException;
+ public CompactionRequest requestCompaction(final HRegion r, final Store s, final String why,
+ int pri, CompactionRequest request) throws IOException;
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java Thu Feb 28 00:33:06 2013
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.util.StringUtils;
@@ -50,24 +51,45 @@ class Compactor extends Configured {
}
/**
- * Do a minor/major compaction on an explicit set of storefiles from a Store.
- *
- * @param store Store the files belong to
- * @param filesToCompact which files to compact
- * @param majorCompaction true to major compact (prune all deletes, max versions, etc)
- * @param maxId Readers maximum sequence id.
- * @return Product of compaction or null if all cells expired or deleted and
- * nothing made it through the compaction.
+ * Compact a list of files for testing. Creates a fake {@link CompactionRequest} to pass to the
+ * actual compaction method
+ * @param store store which should be compacted
+ * @param conf configuration to use when generating the compaction selection
+ * @param filesToCompact the files to compact. They are used a the compaction selection for the
+ * generated {@link CompactionRequest}
+ * @param isMajor <tt>true</tt> to initiate a major compaction (prune all deletes, max versions,
+ * etc)
+ * @param maxId maximum sequenceID == the last key of all files in the compaction
+ * @return product of the compaction or null if all cells expired or deleted and nothing made it
+ * through the compaction.
* @throws IOException
*/
- StoreFile.Writer compact(final Store store,
+ public StoreFile.Writer compactForTesting(final Store store, Configuration conf,
final Collection<StoreFile> filesToCompact,
- final boolean majorCompaction, final long maxId)
- throws IOException {
+ boolean isMajor, long maxId) throws IOException {
+ return compact(CompactionRequest.getRequestForTesting(store, conf, filesToCompact, isMajor),
+ maxId);
+ }
+
+ /**
+ * Do a minor/major compaction on an explicit set of storefiles from a Store.
+ * @param request the requested compaction that contains all necessary information to complete the
+ * compaction (i.e. the store, the files, etc.)
+ * @return Product of compaction or null if all cells expired or deleted and nothing made it
+ * through the compaction.
+ * @throws IOException
+ */
+ StoreFile.Writer compact(CompactionRequest request, long maxId) throws IOException {
// Calculate maximum key count after compaction (for blooms)
// Also calculate earliest put timestamp if major compaction
int maxKeyCount = 0;
long earliestPutTs = HConstants.LATEST_TIMESTAMP;
+
+ // pull out the interesting things from the CR for ease later
+ final Store store = request.getStore();
+ final boolean majorCompaction = request.isMajor();
+ final List<StoreFile> filesToCompact = request.getFiles();
+
for (StoreFile file : filesToCompact) {
StoreFile.Reader r = file.getReader();
if (r == null) {
@@ -76,7 +98,8 @@ class Compactor extends Configured {
}
// NOTE: getFilterEntries could cause under-sized blooms if the user
// switches bloom type (e.g. from ROW to ROWCOL)
- long keyCount = (r.getBloomFilterType() == store.getFamily().getBloomFilterType()) ?
+ long keyCount = (r.getBloomFilterType() == store.getFamily()
+ .getBloomFilterType()) ?
r.getFilterEntries() : r.getEntries();
maxKeyCount += keyCount;
// For major compactions calculate the earliest put timestamp
@@ -130,7 +153,8 @@ class Compactor extends Configured {
scanner = store.getHRegion()
.getCoprocessorHost()
.preCompactScannerOpen(store, scanners,
- majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT, earliestPutTs);
+ majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT, earliestPutTs,
+ request);
}
if (scanner == null) {
Scan scan = new Scan();
@@ -142,7 +166,7 @@ class Compactor extends Configured {
}
if (store.getHRegion().getCoprocessorHost() != null) {
InternalScanner cpScanner =
- store.getHRegion().getCoprocessorHost().preCompact(store, scanner);
+ store.getHRegion().getCoprocessorHost().preCompact(store, scanner, request);
// NULL scanner returned from coprocessor hooks means skip normal processing
if (cpScanner == null) {
return null;
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Thu Feb 28 00:33:06 2013
@@ -1272,17 +1272,17 @@ public class HRegionServer implements HR
try {
if (s.needsCompaction()) {
// Queue a compaction. Will recognize if major is needed.
- this.instance.compactSplitThread.requestCompaction(r, s,
- getName() + " requests compaction");
+ this.instance.compactSplitThread.requestCompaction(r, s, getName()
+ + " requests compaction", null);
} else if (s.isMajorCompaction()) {
- if (majorCompactPriority == DEFAULT_PRIORITY ||
- majorCompactPriority > r.getCompactPriority()) {
- this.instance.compactSplitThread.requestCompaction(r, s,
- getName() + " requests major compaction; use default priority");
+ if (majorCompactPriority == DEFAULT_PRIORITY
+ || majorCompactPriority > r.getCompactPriority()) {
+ this.instance.compactSplitThread.requestCompaction(r, s, getName()
+ + " requests major compaction; use default priority", null);
} else {
- this.instance.compactSplitThread.requestCompaction(r, s,
- getName() + " requests major compaction; use configured priority",
- this.majorCompactPriority);
+ this.instance.compactSplitThread.requestCompaction(r, s, getName()
+ + " requests major compaction; use configured priority",
+ this.majorCompactPriority, null);
}
}
} catch (IOException e) {
@@ -1705,7 +1705,7 @@ public class HRegionServer implements HR
// Do checks to see if we need to compact (references or too many files)
for (Store s : r.getStores().values()) {
if (s.hasReferences() || s.needsCompaction()) {
- getCompactionRequester().requestCompaction(r, s, "Opening Region");
+ getCompactionRequester().requestCompaction(r, s, "Opening Region", null);
}
}
// Update ZK, ROOT or META
@@ -3138,10 +3138,10 @@ public class HRegionServer implements HR
String log = "User-triggered " + (major ? "major " : "") + "compaction" + familyLogMsg;
if (family != null) {
compactSplitThread.requestCompaction(region, store, log,
- Store.PRIORITY_USER);
+ Store.PRIORITY_USER, null);
} else {
compactSplitThread.requestCompaction(region, log,
- Store.PRIORITY_USER);
+ Store.PRIORITY_USER, null);
}
}
@@ -3962,4 +3962,11 @@ public class HRegionServer implements HR
String healthScriptLocation = this.conf.get(HConstants.HEALTH_SCRIPT_LOC);
return org.apache.commons.lang.StringUtils.isNotBlank(healthScriptLocation);
}
+
+ /**
+ * @return the underlying {@link CompactSplitThread} for the servers
+ */
+ public CompactSplitThread getCompactSplitThread() {
+ return this.compactSplitThread;
+ }
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java Thu Feb 28 00:33:06 2013
@@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.filter.Co
import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.util.Bytes;
@@ -343,10 +344,10 @@ public class RegionCoprocessorHost
/**
* See
- * {@link RegionObserver#preCompactScannerOpen(ObserverContext, Store, List, ScanType, long, InternalScanner)}
+ * {@link RegionObserver#preCompactScannerOpen(ObserverContext, Store, List, ScanType, long, InternalScanner, CompactionRequest)}
*/
public InternalScanner preCompactScannerOpen(Store store, List<StoreFileScanner> scanners,
- ScanType scanType, long earliestPutTs) throws IOException {
+ ScanType scanType, long earliestPutTs, CompactionRequest request) throws IOException {
ObserverContext<RegionCoprocessorEnvironment> ctx = null;
InternalScanner s = null;
for (RegionEnvironment env: coprocessors) {
@@ -354,7 +355,7 @@ public class RegionCoprocessorHost
ctx = ObserverContext.createAndPrepare(env, ctx);
try {
s = ((RegionObserver) env.getInstance()).preCompactScannerOpen(ctx, store, scanners,
- scanType, earliestPutTs, s);
+ scanType, earliestPutTs, s, request);
} catch (Throwable e) {
handleCoprocessorThrowable(env,e);
}
@@ -367,25 +368,25 @@ public class RegionCoprocessorHost
}
/**
- * Called prior to selecting the {@link StoreFile}s for compaction from
- * the list of currently available candidates.
+ * Called prior to selecting the {@link StoreFile}s for compaction from the list of currently
+ * available candidates.
* @param store The store where compaction is being requested
* @param candidates The currently available store files
+ * @param request custom compaction request
* @return If {@code true}, skip the normal selection process and use the current list
* @throws IOException
*/
- public boolean preCompactSelection(Store store, List<StoreFile> candidates) throws IOException {
+ public boolean preCompactSelection(Store store, List<StoreFile> candidates,
+ CompactionRequest request) throws IOException {
ObserverContext<RegionCoprocessorEnvironment> ctx = null;
boolean bypass = false;
for (RegionEnvironment env: coprocessors) {
if (env.getInstance() instanceof RegionObserver) {
ctx = ObserverContext.createAndPrepare(env, ctx);
try {
- ((RegionObserver)env.getInstance()).preCompactSelection(
- ctx, store, candidates);
+ ((RegionObserver) env.getInstance()).preCompactSelection(ctx, store, candidates, request);
} catch (Throwable e) {
handleCoprocessorThrowable(env,e);
-
}
bypass |= ctx.shouldBypass();
if (ctx.shouldComplete()) {
@@ -397,20 +398,20 @@ public class RegionCoprocessorHost
}
/**
- * Called after the {@link StoreFile}s to be compacted have been selected
- * from the available candidates.
+ * Called after the {@link StoreFile}s to be compacted have been selected from the available
+ * candidates.
* @param store The store where compaction is being requested
* @param selected The store files selected to compact
+ * @param request custom compaction
*/
- public void postCompactSelection(Store store,
- ImmutableList<StoreFile> selected) {
+ public void postCompactSelection(Store store, ImmutableList<StoreFile> selected,
+ CompactionRequest request) {
ObserverContext<RegionCoprocessorEnvironment> ctx = null;
for (RegionEnvironment env: coprocessors) {
if (env.getInstance() instanceof RegionObserver) {
ctx = ObserverContext.createAndPrepare(env, ctx);
try {
- ((RegionObserver)env.getInstance()).postCompactSelection(
- ctx, store, selected);
+ ((RegionObserver) env.getInstance()).postCompactSelection(ctx, store, selected, request);
} catch (Throwable e) {
handleCoprocessorThrowableNoRethrow(env,e);
}
@@ -425,19 +426,20 @@ public class RegionCoprocessorHost
* Called prior to rewriting the store files selected for compaction
* @param store the store being compacted
* @param scanner the scanner used to read store data during compaction
- * @throws IOException
+ * @param request the compaction that will be executed
+ * @throws IOException
*/
- public InternalScanner preCompact(Store store, InternalScanner scanner) throws IOException {
+ public InternalScanner preCompact(Store store, InternalScanner scanner,
+ CompactionRequest request) throws IOException {
ObserverContext<RegionCoprocessorEnvironment> ctx = null;
boolean bypass = false;
- for (RegionEnvironment env: coprocessors) {
+ for (RegionEnvironment env : coprocessors) {
if (env.getInstance() instanceof RegionObserver) {
ctx = ObserverContext.createAndPrepare(env, ctx);
try {
- scanner = ((RegionObserver)env.getInstance()).preCompact(
- ctx, store, scanner);
+ scanner = ((RegionObserver) env.getInstance()).preCompact(ctx, store, scanner, request);
} catch (Throwable e) {
- handleCoprocessorThrowable(env,e);
+ handleCoprocessorThrowable(env, e);
}
bypass |= ctx.shouldBypass();
if (ctx.shouldComplete()) {
@@ -452,15 +454,17 @@ public class RegionCoprocessorHost
* Called after the store compaction has completed.
* @param store the store being compacted
* @param resultFile the new store file written during compaction
+ * @param request the compaction that is being executed
* @throws IOException
*/
- public void postCompact(Store store, StoreFile resultFile) throws IOException {
+ public void postCompact(Store store, StoreFile resultFile, CompactionRequest request)
+ throws IOException {
ObserverContext<RegionCoprocessorEnvironment> ctx = null;
for (RegionEnvironment env: coprocessors) {
if (env.getInstance() instanceof RegionObserver) {
ctx = ObserverContext.createAndPrepare(env, ctx);
try {
- ((RegionObserver)env.getInstance()).postCompact(ctx, store, resultFile);
+ ((RegionObserver) env.getInstance()).postCompact(ctx, store, resultFile, request);
} catch (Throwable e) {
handleCoprocessorThrowable(env, e);
}
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Thu Feb 28 00:33:06 2013
@@ -1106,13 +1106,12 @@ public class Store extends SchemaConfigu
StoreFile sf = null;
try {
- StoreFile.Writer writer =
- this.compactor.compact(this, filesToCompact, cr.isMajor(), maxId);
+ StoreFile.Writer writer = this.compactor.compact(cr, maxId);
// Move the compaction into place.
if (this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
sf = completeCompaction(filesToCompact, writer);
if (region.getCoprocessorHost() != null) {
- region.getCoprocessorHost().postCompact(this, sf);
+ region.getCoprocessorHost().postCompact(this, sf, cr);
}
} else {
// Create storefile around what we wrote with a reader on it.
@@ -1175,12 +1174,12 @@ public class Store extends SchemaConfigu
try {
// Ready to go. Have list of files to compact.
- StoreFile.Writer writer =
- this.compactor.compact(this, filesToCompact, isMajor, maxId);
+ StoreFile.Writer writer = this.compactor.compactForTesting(this, conf, filesToCompact,
+ isMajor, maxId);
// Move the compaction into place.
StoreFile sf = completeCompaction(filesToCompact, writer);
if (region.getCoprocessorHost() != null) {
- region.getCoprocessorHost().postCompact(this, sf);
+ region.getCoprocessorHost().postCompact(this, sf, null);
}
} finally {
synchronized (filesCompacting) {
@@ -1331,16 +1330,16 @@ public class Store extends SchemaConfigu
}
public CompactionRequest requestCompaction() throws IOException {
- return requestCompaction(NO_PRIORITY);
+ return requestCompaction(NO_PRIORITY, null);
}
- public CompactionRequest requestCompaction(int priority) throws IOException {
+ public CompactionRequest requestCompaction(int priority, CompactionRequest request)
+ throws IOException {
// don't even select for compaction if writes are disabled
if (!this.region.areWritesEnabled()) {
return null;
}
- CompactionRequest ret = null;
this.lock.readLock().lock();
try {
synchronized (filesCompacting) {
@@ -1357,8 +1356,7 @@ public class Store extends SchemaConfigu
boolean override = false;
if (region.getCoprocessorHost() != null) {
- override = region.getCoprocessorHost().preCompactSelection(
- this, candidates);
+ override = region.getCoprocessorHost().preCompactSelection(this, candidates, request);
}
CompactSelection filesToCompact;
if (override) {
@@ -1370,7 +1368,7 @@ public class Store extends SchemaConfigu
if (region.getCoprocessorHost() != null) {
region.getCoprocessorHost().postCompactSelection(this,
- ImmutableList.copyOf(filesToCompact.getFilesToCompact()));
+ ImmutableList.copyOf(filesToCompact.getFilesToCompact()), request);
}
// no files to compact
@@ -1396,15 +1394,24 @@ public class Store extends SchemaConfigu
// everything went better than expected. create a compaction request
int pri = getCompactPriority(priority);
- ret = new CompactionRequest(region, this, filesToCompact, isMajor, pri);
+ //not a special compaction request, so we need to make one
+ if(request == null){
+ request = new CompactionRequest(region, this, filesToCompact, isMajor, pri);
+ } else {
+ // update the request with what the system thinks the request should be
+ // its up to the request if it wants to listen
+ request.setSelection(filesToCompact);
+ request.setIsMajor(isMajor);
+ request.setPriority(pri);
+ }
}
} finally {
this.lock.readLock().unlock();
}
- if (ret != null) {
- CompactionRequest.preRequest(ret);
+ if (request != null) {
+ CompactionRequest.preRequest(request);
}
- return ret;
+ return request;
}
public void finishRequest(CompactionRequest cr) {
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java Thu Feb 28 00:33:06 2013
@@ -20,6 +20,8 @@
package org.apache.hadoop.hbase.regionserver.compactions;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.RejectedExecutionHandler;
@@ -28,6 +30,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.RemoteExceptionHandler;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -51,9 +54,9 @@ public class CompactionRequest implement
static final Log LOG = LogFactory.getLog(CompactionRequest.class);
private final HRegion r;
private final Store s;
- private final CompactSelection compactSelection;
- private final long totalSize;
- private final boolean isMajor;
+ private CompactSelection compactSelection;
+ private long totalSize;
+ private boolean isMajor;
private int p;
private final Long timeInNanos;
private HRegionServer server = null;
@@ -66,23 +69,51 @@ public class CompactionRequest implement
private static final ConcurrentHashMap<Long, AtomicInteger>
minorCompactions = new ConcurrentHashMap<Long, AtomicInteger>();
- public CompactionRequest(HRegion r, Store s,
- CompactSelection files, boolean isMajor, int p) {
- Preconditions.checkNotNull(r);
- Preconditions.checkNotNull(files);
+ /**
+ * Create a simple compaction request just for testing - this lets you specify everything you
+ * would need in the general case of testing compactions from an external perspective (e.g.
+ * requesting a compaction through the HRegion).
+ * @param store
+ * @param conf
+ * @param selection
+ * @param isMajor
+ * @return a request that is useful in requesting compactions for testing
+ */
+ public static CompactionRequest getRequestForTesting(Store store, Configuration conf,
+ Collection<StoreFile> selection, boolean isMajor) {
+ return new CompactionRequest(store.getHRegion(), store, new CompactSelection(conf,
+ new ArrayList<StoreFile>(
+ selection)), isMajor, 0, System.nanoTime());
+ }
- this.r = r;
- this.s = s;
- this.compactSelection = files;
- long sz = 0;
- for (StoreFile sf : files.getFilesToCompact()) {
- sz += sf.getReader().length();
- }
- this.totalSize = sz;
- this.isMajor = isMajor;
- this.p = p;
- this.timeInNanos = System.nanoTime();
+ /**
+ * Constructor for a custom compaction. Uses the setXXX methods to update the state of the
+ * compaction before being used. Uses the current system time on creation as the start time.
+ * @param region region that is being compacted
+ * @param store store which is being compacted
+ * @param priority specified priority with which this compaction should enter the queue.
+ */
+ public CompactionRequest(HRegion region, Store store, int priority) {
+ this(region, store, null, false, priority, System.nanoTime());
+ }
+
+ public CompactionRequest(HRegion r, Store s, CompactSelection files, boolean isMajor, int p) {
+ // delegate to the internal constructor after checking basic preconditions
+ this(Preconditions.checkNotNull(r), s, Preconditions.checkNotNull(files), isMajor, p, System
+ .nanoTime());
+ }
+
+ private CompactionRequest(HRegion region, Store store, CompactSelection files, boolean isMajor,
+ int priority, long startTime) {
+ this.r = region;
+ this.s = store;
+ this.isMajor = isMajor;
+ this.p = priority;
+ this.timeInNanos = startTime;
+ if (files != null) {
+ this.setSelection(files);
}
+ }
/**
* Find out if a given region in compaction now.
@@ -216,6 +247,28 @@ public class CompactionRequest implement
this.server = hrs;
}
+ /**
+ * Set the files (and, implicitly, the size of the compaction based on those files)
+ * @param files files that should be included in the compaction
+ */
+ public void setSelection(CompactSelection files) {
+ long sz = 0;
+ for (StoreFile sf : files.getFilesToCompact()) {
+ sz += sf.getReader().length();
+ }
+ this.totalSize = sz;
+ this.compactSelection = files;
+ }
+
+ /**
+ * Specify if this compaction should be a major compaction based on the state of the store
+ * @param isMajor <tt>true</tt> if the system determines that this compaction should be a major
+ * compaction
+ */
+ public void setIsMajor(boolean isMajor) {
+ this.isMajor = isMajor;
+ }
+
@Override
public String toString() {
String fsList = Joiner.on(", ").join(
@@ -255,11 +308,11 @@ public class CompactionRequest implement
server.getMetrics().addCompaction(now - start, this.totalSize);
// degenerate case: blocked regions require recursive enqueues
if (s.getCompactPriority() <= 0) {
- server.compactSplitThread
- .requestCompaction(r, s, "Recursive enqueue");
+ server.getCompactSplitThread()
+ .requestCompaction(r, s, "Recursive enqueue", null);
} else {
// see if the compaction has caused us to exceed max region size
- server.compactSplitThread.requestSplit(r);
+ server.getCompactSplitThread().requestSplit(r);
}
}
} catch (IOException ex) {
@@ -271,7 +324,7 @@ public class CompactionRequest implement
server.checkFileSystem();
} finally {
s.finishRequest(this);
- LOG.debug("CompactSplitThread status: " + server.compactSplitThread);
+ LOG.debug("CompactSplitThread status: " + server.getCompactSplitThread());
}
}
@@ -299,4 +352,4 @@ public class CompactionRequest implement
}
}
}
- }
+}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java Thu Feb 28 00:33:06 2013
@@ -24,10 +24,12 @@ import static org.mockito.Mockito.spy;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.concurrent.CountDownLatch;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -52,9 +54,11 @@ import org.apache.hadoop.hbase.io.hfile.
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -566,8 +570,12 @@ public class TestCompaction extends HBas
}
private void createStoreFile(final HRegion region) throws IOException {
+ createStoreFile(region, Bytes.toString(COLUMN_FAMILY));
+ }
+
+ private void createStoreFile(final HRegion region, String family) throws IOException {
HRegionIncommon loader = new HRegionIncommon(region);
- addContent(loader, Bytes.toString(COLUMN_FAMILY));
+ addContent(loader, family);
loader.flushcache();
}
@@ -589,8 +597,8 @@ public class TestCompaction extends HBas
long maxId = StoreFile.getMaxSequenceIdInList(storeFiles);
Compactor tool = new Compactor(this.conf);
- StoreFile.Writer compactedFile =
- tool.compact(store, storeFiles, false, maxId);
+ StoreFile.Writer compactedFile = tool.compactForTesting(store, this.conf, storeFiles, false,
+ maxId);
// Now lets corrupt the compacted file.
FileSystem fs = FileSystem.get(conf);
@@ -628,7 +636,7 @@ public class TestCompaction extends HBas
}
store.triggerMajorCompaction();
- CompactionRequest request = store.requestCompaction(Store.NO_PRIORITY);
+ CompactionRequest request = store.requestCompaction(Store.NO_PRIORITY, null);
assertNotNull("Expected to receive a compaction request", request);
assertEquals(
"System-requested major compaction should not occur if there are too many store files",
@@ -646,7 +654,7 @@ public class TestCompaction extends HBas
createStoreFile(r);
}
store.triggerMajorCompaction();
- CompactionRequest request = store.requestCompaction(Store.PRIORITY_USER);
+ CompactionRequest request = store.requestCompaction(Store.PRIORITY_USER, null);
assertNotNull("Expected to receive a compaction request", request);
assertEquals(
"User-requested major compaction should always occur, even if there are too many store files",
@@ -654,6 +662,90 @@ public class TestCompaction extends HBas
request.isMajor());
}
+ /**
+ * Create a custom compaction request and be sure that we can track it through the queue, knowing
+ * when the compaction is completed.
+ */
+ public void testTrackingCompactionRequest() throws Exception {
+ // setup a compact/split thread on a mock server
+ HRegionServer mockServer = Mockito.mock(HRegionServer.class);
+ Mockito.when(mockServer.getConfiguration()).thenReturn(r.getConf());
+ CompactSplitThread thread = new CompactSplitThread(mockServer);
+ Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);
+ // simple stop for the metrics - we ignore any updates in the test
+ RegionServerMetrics mockMetrics = Mockito.mock(RegionServerMetrics.class);
+ Mockito.when(mockServer.getMetrics()).thenReturn(mockMetrics);
+
+ // setup a region/store with some files
+ Store store = r.getStore(COLUMN_FAMILY);
+ createStoreFile(r);
+ for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) {
+ createStoreFile(r);
+ }
+
+ CountDownLatch latch = new CountDownLatch(1);
+ TrackableCompactionRequest request = new TrackableCompactionRequest(r, store, latch);
+ thread.requestCompaction(r, store, "test custom comapction", Store.PRIORITY_USER, request);
+ // wait for the latch to complete.
+ latch.await();
+
+ thread.interruptIfNecessary();
+ }
+
+ public void testMultipleCustomCompactionRequests() throws Exception {
+ // setup a compact/split thread on a mock server
+ HRegionServer mockServer = Mockito.mock(HRegionServer.class);
+ Mockito.when(mockServer.getConfiguration()).thenReturn(r.getConf());
+ CompactSplitThread thread = new CompactSplitThread(mockServer);
+ Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);
+ // simple stop for the metrics - we ignore any updates in the test
+ RegionServerMetrics mockMetrics = Mockito.mock(RegionServerMetrics.class);
+ Mockito.when(mockServer.getMetrics()).thenReturn(mockMetrics);
+
+ // setup a region/store with some files
+ int numStores = r.getStores().size();
+ List<CompactionRequest> requests = new ArrayList<CompactionRequest>(numStores);
+ CountDownLatch latch = new CountDownLatch(numStores);
+ // create some store files and setup requests for each store on which we want to do a
+ // compaction
+ for (Store store : r.getStores().values()) {
+ createStoreFile(r, store.getColumnFamilyName());
+ createStoreFile(r, store.getColumnFamilyName());
+ createStoreFile(r, store.getColumnFamilyName());
+ requests.add(new TrackableCompactionRequest(r, store, latch));
+ }
+
+ thread.requestCompaction(r, "test mulitple custom comapctions", Store.PRIORITY_USER,
+ Collections.unmodifiableList(requests));
+
+ // wait for the latch to complete.
+ latch.await();
+
+ thread.interruptIfNecessary();
+ }
+
+ /**
+ * Simple {@link CompactionRequest} on which you can wait until the requested compaction finishes.
+ */
+ public static class TrackableCompactionRequest extends CompactionRequest {
+ private CountDownLatch done;
+
+ /**
+ * Constructor for a custom compaction. Uses the setXXX methods to update the state of the
+ * compaction before being used.
+ */
+ public TrackableCompactionRequest(HRegion region, Store store, CountDownLatch finished) {
+ super(region, store, Store.PRIORITY_USER);
+ this.done = finished;
+ }
+
+ @Override
+ public void run() {
+ super.run();
+ this.done.countDown();
+ }
+ }
+
@org.junit.Rule
public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();