You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jy...@apache.org on 2013/02/28 01:33:06 UTC

svn commit: r1451057 - in /hbase/branches/0.94/src: main/java/org/apache/hadoop/hbase/coprocessor/ main/java/org/apache/hadoop/hbase/regionserver/ main/java/org/apache/hadoop/hbase/regionserver/compactions/ test/java/org/apache/hadoop/hbase/regionserver/

Author: jyates
Date: Thu Feb 28 00:33:06 2013
New Revision: 1451057

URL: http://svn.apache.org/r1451057
Log:
HBASE-7725: Add ability to create custom compaction request

Modified:
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
    hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/BaseRegionObserver.java Thu Feb 28 00:33:06 2013
@@ -41,6 +41,8 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.regionserver.ScanType;
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactSelection;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.util.Pair;
@@ -110,10 +112,23 @@ public abstract class BaseRegionObserver
 
   @Override
   public void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final List<StoreFile> candidates, final CompactionRequest request)
+      throws IOException {
+    preCompactSelection(c, store, candidates);
+  }
+
+  @Override
+  public void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Store store, final List<StoreFile> candidates) throws IOException { }
 
   @Override
   public void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final ImmutableList<StoreFile> selected, CompactionRequest request) {
+    postCompactSelection(c, store, selected);
+  }
+
+  @Override
+  public void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Store store, final ImmutableList<StoreFile> selected) { }
 
   @Override
@@ -123,6 +138,13 @@ public abstract class BaseRegionObserver
   }
 
   @Override
+  public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
+      final Store store, final InternalScanner scanner, CompactionRequest request)
+      throws IOException {
+    return preCompact(e, store, scanner);
+  }
+
+  @Override
   public InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Store store, List<? extends KeyValueScanner> scanners, final ScanType scanType,
       final long earliestPutTs, final InternalScanner s) throws IOException {
@@ -130,11 +152,25 @@ public abstract class BaseRegionObserver
   }
 
   @Override
+  public InternalScanner preCompactScannerOpen(
+      final ObserverContext<RegionCoprocessorEnvironment> c, final Store store,
+      List<? extends KeyValueScanner> scanners, final ScanType scanType, final long earliestPutTs,
+      final InternalScanner s, CompactionRequest request) throws IOException {
+    return preCompactScannerOpen(c, store, scanners, scanType, earliestPutTs, s);
+  }
+
+  @Override
   public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, final Store store,
       final StoreFile resultFile) throws IOException {
   }
 
   @Override
+  public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e, final Store store,
+      final StoreFile resultFile, CompactionRequest request) throws IOException {
+    postCompact(e, store, resultFile);
+  }
+
+  @Override
   public void preGetClosestRowBefore(final ObserverContext<RegionCoprocessorEnvironment> e,
       final byte [] row, final byte [] family, final Result result)
     throws IOException {

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionObserver.java Thu Feb 28 00:33:06 2013
@@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.regionser
 import org.apache.hadoop.hbase.regionserver.Store;
 import org.apache.hadoop.hbase.regionserver.StoreFile;
 import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 
@@ -125,9 +126,23 @@ public interface RegionObserver extends 
       final StoreFile resultFile) throws IOException;
 
   /**
-   * Called prior to selecting the {@link StoreFile}s to compact from the list
-   * of available candidates.  To alter the files used for compaction, you may
-   * mutate the passed in list of candidates.
+   * Called prior to selecting the {@link StoreFile StoreFiles} to compact from the list of
+   * available candidates. To alter the files used for compaction, you may mutate the passed in list
+   * of candidates.
+   * @param c the environment provided by the region server
+   * @param store the store where compaction is being requested
+   * @param candidates the store files currently available for compaction
+   * @param request custom compaction request
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  void preCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final List<StoreFile> candidates, final CompactionRequest request)
+      throws IOException;
+
+  /**
+   * Called prior to selecting the {@link StoreFile}s to compact from the list of available
+   * candidates. To alter the files used for compaction, you may mutate the passed in list of
+   * candidates.
    * @param c the environment provided by the region server
    * @param store the store where compaction is being requested
    * @param candidates the store files currently available for compaction
@@ -147,9 +162,20 @@ public interface RegionObserver extends 
       final Store store, final ImmutableList<StoreFile> selected);
 
   /**
-   * Called prior to writing the {@link StoreFile}s selected for compaction into
-   * a new {@code StoreFile}.  To override or modify the compaction process,
-   * implementing classes have two options:
+   * Called after the {@link StoreFile}s to compact have been selected from the available
+   * candidates.
+   * @param c the environment provided by the region server
+   * @param store the store being compacted
+   * @param selected the store files selected to compact
+   * @param request custom compaction request
+   */
+  void postCompactSelection(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final ImmutableList<StoreFile> selected, CompactionRequest request);
+
+  /**
+   * Called prior to writing the {@link StoreFile}s selected for compaction into a new
+   * {@code StoreFile}. To override or modify the compaction process, implementing classes have two
+   * options:
    * <ul>
    *   <li>Wrap the provided {@link InternalScanner} with a custom
    *   implementation that is returned from this method.  The custom scanner
@@ -164,32 +190,55 @@ public interface RegionObserver extends 
    * </ul>
    * @param c the environment provided by the region server
    * @param store the store being compacted
-   * @param scanner the scanner over existing data used in the store file
-   * rewriting
-   * @return the scanner to use during compaction.  Should not be {@code null}
-   * unless the implementation is writing new store files on its own.
+   * @param scanner the scanner over existing data used in the store file rewriting
+   * @return the scanner to use during compaction. Should not be {@code null} unless the
+   *         implementation is writing new store files on its own.
    * @throws IOException if an error occurred on the coprocessor
    */
   InternalScanner preCompact(final ObserverContext<RegionCoprocessorEnvironment> c,
       final Store store, final InternalScanner scanner) throws IOException;
 
   /**
-   * Called prior to writing the {@link StoreFile}s selected for compaction into
-   * a new {@code StoreFile} and prior to creating the scanner used to read the
-   * input files.  To override or modify the compaction process,
-   * implementing classes can return a new scanner to provide the KeyValues to be
-   * stored into the new {@code StoreFile} or null to perform the default processing.
-   * Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
+   * Called prior to writing the {@link StoreFile}s selected for compaction into a new
+   * {@code StoreFile}. To override or modify the compaction process, implementing classes have two
+   * options:
+   * <ul>
+   * <li>Wrap the provided {@link InternalScanner} with a custom implementation that is returned
+   * from this method. The custom scanner can then inspect {@link KeyValue}s from the wrapped
+   * scanner, applying its own policy to what gets written.</li>
+   * <li>Call {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} and provide a
+   * custom implementation for writing of new {@link StoreFile}s. <strong>Note: any implementations
+   * bypassing core compaction using this approach must write out new store files themselves or the
+   * existing data will no longer be available after compaction.</strong></li>
+   * </ul>
+   * @param c the environment provided by the region server
+   * @param store the store being compacted
+   * @param scanner the scanner over existing data used in the store file rewriting
+   * @param request the requested compaction
+   * @return the scanner to use during compaction. Should not be {@code null} unless the
+   *         implementation is writing new store files on its own.
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  InternalScanner preCompact(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, final InternalScanner scanner, CompactionRequest request)
+      throws IOException;
+
+  /**
+   * Called prior to writing the {@link StoreFile}s selected for compaction into a new
+   * {@code StoreFile} and prior to creating the scanner used to read the input files. To override
+   * or modify the compaction process, implementing classes can return a new scanner to provide the
+   * KeyValues to be stored into the new {@code StoreFile} or null to perform the default
+   * processing. Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
    * effect in this hook.
    * @param c the environment provided by the region server
    * @param store the store being compacted
    * @param scanners the list {@link StoreFileScanner}s to be read from
    * @param scantype the {@link ScanType} indicating whether this is a major or minor compaction
-   * @param earliestPutTs timestamp of the earliest put that was found in any of the involved
-   * store files
+   * @param earliestPutTs timestamp of the earliest put that was found in any of the involved store
+   *          files
    * @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain
-   * @return the scanner to use during compaction.  {@code null} if the default implementation
-   * is to be used.
+   * @return the scanner to use during compaction. {@code null} if the default implementation is to
+   *         be used.
    * @throws IOException if an error occurred on the coprocessor
    */
   InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
@@ -197,6 +246,30 @@ public interface RegionObserver extends 
       final long earliestPutTs, final InternalScanner s) throws IOException;
 
   /**
+   * Called prior to writing the {@link StoreFile}s selected for compaction into a new
+   * {@code StoreFile} and prior to creating the scanner used to read the input files. To override
+   * or modify the compaction process, implementing classes can return a new scanner to provide the
+   * KeyValues to be stored into the new {@code StoreFile} or null to perform the default
+   * processing. Calling {@link org.apache.hadoop.hbase.coprocessor.ObserverContext#bypass()} has no
+   * effect in this hook.
+   * @param c the environment provided by the region server
+   * @param store the store being compacted
+   * @param scanners the list {@link StoreFileScanner}s to be read from
+   * @param scanType the {@link ScanType} indicating whether this is a major or minor compaction
+   * @param earliestPutTs timestamp of the earliest put that was found in any of the involved store
+   *          files
+   * @param s the base scanner, if not {@code null}, from previous RegionObserver in the chain
+   * @param request the requested compaction
+   * @return the scanner to use during compaction. {@code null} if the default implementation is to
+   *         be used.
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  InternalScanner preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
+      final Store store, List<? extends KeyValueScanner> scanners, final ScanType scanType,
+      final long earliestPutTs, final InternalScanner s, CompactionRequest request)
+      throws IOException;
+
+  /**
    * Called after compaction has completed and the new store file has been
    * moved in to place.
    * @param c the environment provided by the region server
@@ -208,9 +281,20 @@ public interface RegionObserver extends 
       StoreFile resultFile) throws IOException;
 
   /**
-   * Called before the region is split.
+   * Called after compaction has completed and the new store file has been moved in to place.
    * @param c the environment provided by the region server
-   * (e.getRegion() returns the parent region)
+   * @param store the store being compacted
+   * @param resultFile the new store file written out during compaction
+   * @param request the requested compaction
+   * @throws IOException if an error occurred on the coprocessor
+   */
+  void postCompact(final ObserverContext<RegionCoprocessorEnvironment> c, final Store store,
+      StoreFile resultFile, CompactionRequest request) throws IOException;
+
+  /**
+   * Called before the region is split.
+   * @param c the environment provided by the region server (e.getRegion() returns the parent
+   *          region)
    * @throws IOException if an error occurred on the coprocessor
    */
   void preSplit(final ObserverContext<RegionCoprocessorEnvironment> c) throws IOException;

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java Thu Feb 28 00:33:06 2013
@@ -21,7 +21,9 @@ package org.apache.hadoop.hbase.regionse
 
 import java.io.IOException;
 import java.util.concurrent.Executors;
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.PriorityBlockingQueue;
 import java.util.concurrent.RejectedExecutionException;
@@ -184,37 +186,51 @@ public class CompactSplitThread implemen
     }
   }
 
-  public synchronized void requestCompaction(final HRegion r,
-      final String why) throws IOException {
-    for(Store s : r.getStores().values()) {
-      requestCompaction(r, s, why, Store.NO_PRIORITY);
-    }
+  @Override
+  public synchronized List<CompactionRequest> requestCompaction(final HRegion r, final String why)
+      throws IOException {
+    return requestCompaction(r, why, null);
   }
 
-  public synchronized void requestCompaction(final HRegion r, final Store s,
-      final String why) throws IOException {
-    requestCompaction(r, s, why, Store.NO_PRIORITY);
+  @Override
+  public synchronized List<CompactionRequest> requestCompaction(final HRegion r, final String why,
+      List<CompactionRequest> requests) throws IOException {
+    return requestCompaction(r, why, Store.NO_PRIORITY, requests);
   }
 
-  public synchronized void requestCompaction(final HRegion r, final String why,
-      int p) throws IOException {
-    for(Store s : r.getStores().values()) {
-      requestCompaction(r, s, why, p);
+  @Override
+  public synchronized CompactionRequest requestCompaction(final HRegion r, final Store s,
+      final String why,
+      CompactionRequest request) throws IOException {
+    return requestCompaction(r, s, why, Store.NO_PRIORITY, request);
+  }
+
+  @Override
+  public synchronized List<CompactionRequest> requestCompaction(final HRegion r, final String why,
+      int pri, final List<CompactionRequest> requests) throws IOException {
+    List<CompactionRequest> ret;
+    // not a special compaction request, so make out own list
+    if (requests == null) {
+      ret = new ArrayList<CompactionRequest>(r.getStores().size());
+      for (Store s : r.getStores().values()) {
+        ret.add(requestCompaction(r, s, why, pri, null));
+      }
+    } else {
+      ret = new ArrayList<CompactionRequest>(requests.size());
+      for (CompactionRequest request : requests) {
+        ret.add(requestCompaction(r, request.getStore(), why, pri, request));
+      }
     }
+    return ret;
   }
 
-  /**
-   * @param r HRegion store belongs to
-   * @param s Store to request compaction on
-   * @param why Why compaction requested -- used in debug messages
-   * @param priority override the default priority (NO_PRIORITY == decide)
-   */
-  public synchronized void requestCompaction(final HRegion r, final Store s,
-      final String why, int priority) throws IOException {
+  @Override
+  public synchronized CompactionRequest requestCompaction(final HRegion r, final Store s,
+      final String why, int priority, CompactionRequest request) throws IOException {
     if (this.server.isStopped()) {
-      return;
+      return null;
     }
-    CompactionRequest cr = s.requestCompaction(priority);
+    CompactionRequest cr = s.requestCompaction(priority, request);
     if (cr != null) {
       cr.setServer(server);
       if (priority != Store.NO_PRIORITY) {
@@ -235,6 +251,7 @@ public class CompactSplitThread implemen
             " because compaction request was cancelled");
       }
     }
+    return cr;
   }
 
   /**

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionRequestor.java Thu Feb 28 00:33:06 2013
@@ -20,39 +20,71 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 
 public interface CompactionRequestor {
   /**
    * @param r Region to compact
    * @param why Why compaction was requested -- used in debug messages
+   * @return the created {@link CompactionRequest CompactionRequets} or, if no compactions were
+   *         started, an empty list
+   * @throws IOException
+   */
+  public List<CompactionRequest> requestCompaction(final HRegion r, final String why)
+      throws IOException;
+
+  /**
+   * @param r Region to compact
+   * @param why Why compaction was requested -- used in debug messages
+   * @param requests custom compaction requests. Each compaction must specify the store on which it
+   *          is acting. Can be <tt>null</tt> in which case a compaction will be attempted on all
+   *          stores for the region.
+   * @return The created {@link CompactionRequest CompactionRequests} or an empty list if no
+   *         compactions were started
    * @throws IOException
    */
-  public void requestCompaction(final HRegion r, final String why) throws IOException;
+  public List<CompactionRequest> requestCompaction(final HRegion r, final String why,
+      List<CompactionRequest> requests) throws IOException;
 
   /**
    * @param r Region to compact
    * @param s Store within region to compact
    * @param why Why compaction was requested -- used in debug messages
+   * @param request custom compaction request for the {@link HRegion} and {@link Store}. Custom
+   *          request must be <tt>null</tt> or be constructed with matching region and store.
+   * @return The created {@link CompactionRequest} or <tt>null</tt> if no compaction was started.
    * @throws IOException
    */
-  public void requestCompaction(final HRegion r, final Store s, final String why) throws IOException;
+  public CompactionRequest requestCompaction(final HRegion r, final Store s, final String why,
+      CompactionRequest request) throws IOException;
 
   /**
    * @param r Region to compact
    * @param why Why compaction was requested -- used in debug messages
    * @param pri Priority of this compaction. minHeap. <=0 is critical
+   * @param requests custom compaction requests. Each compaction must specify the store on which it
+   *          is acting. Can be <tt>null</tt> in which case a compaction will be attempted on all
+   *          stores for the region.
+   * @return The created {@link CompactionRequest CompactionRequests} or an empty list if no
+   *         compactions were started.
    * @throws IOException
    */
-  public void requestCompaction(final HRegion r, final String why, int pri) throws IOException;
+  public List<CompactionRequest> requestCompaction(final HRegion r, final String why, int pri,
+      List<CompactionRequest> requests) throws IOException;
 
   /**
    * @param r Region to compact
    * @param s Store within region to compact
    * @param why Why compaction was requested -- used in debug messages
    * @param pri Priority of this compaction. minHeap. <=0 is critical
+   * @param request request custom compaction request to run. {@link Store} and {@link HRegion} for
+   *          the request must match the region and store specified here.
+   * @return The created {@link CompactionRequest} or <tt>null</tt> if no compaction was started
    * @throws IOException
    */
-  public void requestCompaction(final HRegion r, final Store s,
-      final String why, int pri) throws IOException;
+  public CompactionRequest requestCompaction(final HRegion r, final Store s, final String why,
+      int pri, CompactionRequest request) throws IOException;
 
 }

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Compactor.java Thu Feb 28 00:33:06 2013
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.util.StringUtils;
 
@@ -50,24 +51,45 @@ class Compactor extends Configured {
   }
 
   /**
-   * Do a minor/major compaction on an explicit set of storefiles from a Store.
-   *
-   * @param store Store the files belong to
-   * @param filesToCompact which files to compact
-   * @param majorCompaction true to major compact (prune all deletes, max versions, etc)
-   * @param maxId Readers maximum sequence id.
-   * @return Product of compaction or null if all cells expired or deleted and
-   * nothing made it through the compaction.
+   * Compact a list of files for testing. Creates a fake {@link CompactionRequest} to pass to the
+   * actual compaction method
+   * @param store store which should be compacted
+   * @param conf configuration to use when generating the compaction selection
+   * @param filesToCompact the files to compact. They are used a the compaction selection for the
+   *          generated {@link CompactionRequest}
+   * @param isMajor <tt>true</tt> to initiate a major compaction (prune all deletes, max versions,
+   *          etc)
+   * @param maxId maximum sequenceID == the last key of all files in the compaction
+   * @return product of the compaction or null if all cells expired or deleted and nothing made it
+   *         through the compaction.
    * @throws IOException
    */
-  StoreFile.Writer compact(final Store store,
+  public StoreFile.Writer compactForTesting(final Store store, Configuration conf,
       final Collection<StoreFile> filesToCompact,
-      final boolean majorCompaction, final long maxId)
-  throws IOException {
+      boolean isMajor, long maxId) throws IOException {
+    return compact(CompactionRequest.getRequestForTesting(store, conf, filesToCompact, isMajor),
+      maxId);
+  }
+
+  /**
+   * Do a minor/major compaction on an explicit set of storefiles from a Store.
+   * @param request the requested compaction that contains all necessary information to complete the
+   *          compaction (i.e. the store, the files, etc.)
+   * @return Product of compaction or null if all cells expired or deleted and nothing made it
+   *         through the compaction.
+   * @throws IOException
+   */
+  StoreFile.Writer compact(CompactionRequest request, long maxId) throws IOException {
     // Calculate maximum key count after compaction (for blooms)
     // Also calculate earliest put timestamp if major compaction
     int maxKeyCount = 0;
     long earliestPutTs = HConstants.LATEST_TIMESTAMP;
+
+    // pull out the interesting things from the CR for ease later
+    final Store store = request.getStore();
+    final boolean majorCompaction = request.isMajor();
+    final List<StoreFile> filesToCompact = request.getFiles();
+
     for (StoreFile file : filesToCompact) {
       StoreFile.Reader r = file.getReader();
       if (r == null) {
@@ -76,7 +98,8 @@ class Compactor extends Configured {
       }
       // NOTE: getFilterEntries could cause under-sized blooms if the user
       //       switches bloom type (e.g. from ROW to ROWCOL)
-      long keyCount = (r.getBloomFilterType() == store.getFamily().getBloomFilterType()) ?
+      long keyCount = (r.getBloomFilterType() == store.getFamily()
+          .getBloomFilterType()) ?
           r.getFilterEntries() : r.getEntries();
       maxKeyCount += keyCount;
       // For major compactions calculate the earliest put timestamp
@@ -130,7 +153,8 @@ class Compactor extends Configured {
           scanner = store.getHRegion()
               .getCoprocessorHost()
               .preCompactScannerOpen(store, scanners,
-                  majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT, earliestPutTs);
+                majorCompaction ? ScanType.MAJOR_COMPACT : ScanType.MINOR_COMPACT, earliestPutTs,
+                request);
         }
         if (scanner == null) {
           Scan scan = new Scan();
@@ -142,7 +166,7 @@ class Compactor extends Configured {
         }
         if (store.getHRegion().getCoprocessorHost() != null) {
           InternalScanner cpScanner =
-            store.getHRegion().getCoprocessorHost().preCompact(store, scanner);
+            store.getHRegion().getCoprocessorHost().preCompact(store, scanner, request);
           // NULL scanner returned from coprocessor hooks means skip normal processing
           if (cpScanner == null) {
             return null;

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Thu Feb 28 00:33:06 2013
@@ -1272,17 +1272,17 @@ public class HRegionServer implements HR
           try {
             if (s.needsCompaction()) {
               // Queue a compaction. Will recognize if major is needed.
-              this.instance.compactSplitThread.requestCompaction(r, s,
-                getName() + " requests compaction");
+              this.instance.compactSplitThread.requestCompaction(r, s, getName()
+                  + " requests compaction", null);
             } else if (s.isMajorCompaction()) {
-              if (majorCompactPriority == DEFAULT_PRIORITY ||
-                  majorCompactPriority > r.getCompactPriority()) {
-                this.instance.compactSplitThread.requestCompaction(r, s,
-                    getName() + " requests major compaction; use default priority");
+              if (majorCompactPriority == DEFAULT_PRIORITY
+                  || majorCompactPriority > r.getCompactPriority()) {
+                this.instance.compactSplitThread.requestCompaction(r, s, getName()
+                    + " requests major compaction; use default priority", null);
               } else {
-               this.instance.compactSplitThread.requestCompaction(r, s,
-                  getName() + " requests major compaction; use configured priority",
-                  this.majorCompactPriority);
+                this.instance.compactSplitThread.requestCompaction(r, s, getName()
+                    + " requests major compaction; use configured priority",
+                  this.majorCompactPriority, null);
               }
             }
           } catch (IOException e) {
@@ -1705,7 +1705,7 @@ public class HRegionServer implements HR
     // Do checks to see if we need to compact (references or too many files)
     for (Store s : r.getStores().values()) {
       if (s.hasReferences() || s.needsCompaction()) {
-        getCompactionRequester().requestCompaction(r, s, "Opening Region");
+        getCompactionRequester().requestCompaction(r, s, "Opening Region", null);
       }
     }
     // Update ZK, ROOT or META
@@ -3138,10 +3138,10 @@ public class HRegionServer implements HR
     String log = "User-triggered " + (major ? "major " : "") + "compaction" + familyLogMsg;
     if (family != null) {
       compactSplitThread.requestCompaction(region, store, log,
-        Store.PRIORITY_USER);
+        Store.PRIORITY_USER, null);
     } else {
       compactSplitThread.requestCompaction(region, log,
-        Store.PRIORITY_USER);
+        Store.PRIORITY_USER, null);
     }
   }
 
@@ -3962,4 +3962,11 @@ public class HRegionServer implements HR
     String healthScriptLocation = this.conf.get(HConstants.HEALTH_SCRIPT_LOC);
     return org.apache.commons.lang.StringUtils.isNotBlank(healthScriptLocation);
   }
+
+  /**
+   * @return the underlying {@link CompactSplitThread} for the servers
+   */
+  public CompactSplitThread getCompactSplitThread() {
+    return this.compactSplitThread;
+  }
 }

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/RegionCoprocessorHost.java Thu Feb 28 00:33:06 2013
@@ -57,6 +57,7 @@ import org.apache.hadoop.hbase.filter.Co
 import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.ipc.CoprocessorProtocol;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -343,10 +344,10 @@ public class RegionCoprocessorHost
 
   /**
    * See
-   * {@link RegionObserver#preCompactScannerOpen(ObserverContext, Store, List, ScanType, long, InternalScanner)}
+   * {@link RegionObserver#preCompactScannerOpen(ObserverContext, Store, List, ScanType, long, InternalScanner, CompactionRequest)}
    */
   public InternalScanner preCompactScannerOpen(Store store, List<StoreFileScanner> scanners,
-      ScanType scanType, long earliestPutTs) throws IOException {
+      ScanType scanType, long earliestPutTs, CompactionRequest request) throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     InternalScanner s = null;
     for (RegionEnvironment env: coprocessors) {
@@ -354,7 +355,7 @@ public class RegionCoprocessorHost
         ctx = ObserverContext.createAndPrepare(env, ctx);
         try {
           s = ((RegionObserver) env.getInstance()).preCompactScannerOpen(ctx, store, scanners,
-              scanType, earliestPutTs, s);
+            scanType, earliestPutTs, s, request);
         } catch (Throwable e) {
           handleCoprocessorThrowable(env,e);
         }
@@ -367,25 +368,25 @@ public class RegionCoprocessorHost
   }
 
   /**
-   * Called prior to selecting the {@link StoreFile}s for compaction from
-   * the list of currently available candidates.
+   * Called prior to selecting the {@link StoreFile}s for compaction from the list of currently
+   * available candidates.
    * @param store The store where compaction is being requested
    * @param candidates The currently available store files
+   * @param request custom compaction request
    * @return If {@code true}, skip the normal selection process and use the current list
    * @throws IOException
    */
-  public boolean preCompactSelection(Store store, List<StoreFile> candidates) throws IOException {
+  public boolean preCompactSelection(Store store, List<StoreFile> candidates,
+      CompactionRequest request) throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     boolean bypass = false;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
         ctx = ObserverContext.createAndPrepare(env, ctx);
         try {
-          ((RegionObserver)env.getInstance()).preCompactSelection(
-              ctx, store, candidates);
+          ((RegionObserver) env.getInstance()).preCompactSelection(ctx, store, candidates, request);
         } catch (Throwable e) {
           handleCoprocessorThrowable(env,e);
-
         }
         bypass |= ctx.shouldBypass();
         if (ctx.shouldComplete()) {
@@ -397,20 +398,20 @@ public class RegionCoprocessorHost
   }
 
   /**
-   * Called after the {@link StoreFile}s to be compacted have been selected
-   * from the available candidates.
+   * Called after the {@link StoreFile}s to be compacted have been selected from the available
+   * candidates.
    * @param store The store where compaction is being requested
    * @param selected The store files selected to compact
+   * @param request custom compaction
    */
-  public void postCompactSelection(Store store,
-      ImmutableList<StoreFile> selected) {
+  public void postCompactSelection(Store store, ImmutableList<StoreFile> selected,
+      CompactionRequest request) {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
         ctx = ObserverContext.createAndPrepare(env, ctx);
         try {
-          ((RegionObserver)env.getInstance()).postCompactSelection(
-              ctx, store, selected);
+          ((RegionObserver) env.getInstance()).postCompactSelection(ctx, store, selected, request);
         } catch (Throwable e) {
           handleCoprocessorThrowableNoRethrow(env,e);
         }
@@ -425,19 +426,20 @@ public class RegionCoprocessorHost
    * Called prior to rewriting the store files selected for compaction
    * @param store the store being compacted
    * @param scanner the scanner used to read store data during compaction
-   * @throws IOException 
+   * @param request the compaction that will be executed
+   * @throws IOException
    */
-  public InternalScanner preCompact(Store store, InternalScanner scanner) throws IOException {
+  public InternalScanner preCompact(Store store, InternalScanner scanner, 
+      CompactionRequest request) throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     boolean bypass = false;
-    for (RegionEnvironment env: coprocessors) {
+    for (RegionEnvironment env : coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
         ctx = ObserverContext.createAndPrepare(env, ctx);
         try {
-          scanner = ((RegionObserver)env.getInstance()).preCompact(
-              ctx, store, scanner);
+          scanner = ((RegionObserver) env.getInstance()).preCompact(ctx, store, scanner, request);
         } catch (Throwable e) {
-          handleCoprocessorThrowable(env,e);
+          handleCoprocessorThrowable(env, e);
         }
         bypass |= ctx.shouldBypass();
         if (ctx.shouldComplete()) {
@@ -452,15 +454,17 @@ public class RegionCoprocessorHost
    * Called after the store compaction has completed.
    * @param store the store being compacted
    * @param resultFile the new store file written during compaction
+   * @param request the compaction that is being executed
    * @throws IOException
    */
-  public void postCompact(Store store, StoreFile resultFile) throws IOException {
+  public void postCompact(Store store, StoreFile resultFile, CompactionRequest request)
+      throws IOException {
     ObserverContext<RegionCoprocessorEnvironment> ctx = null;
     for (RegionEnvironment env: coprocessors) {
       if (env.getInstance() instanceof RegionObserver) {
         ctx = ObserverContext.createAndPrepare(env, ctx);
         try {
-          ((RegionObserver)env.getInstance()).postCompact(ctx, store, resultFile);
+          ((RegionObserver) env.getInstance()).postCompact(ctx, store, resultFile, request);
         } catch (Throwable e) {
           handleCoprocessorThrowable(env, e);
         }

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Thu Feb 28 00:33:06 2013
@@ -1106,13 +1106,12 @@ public class Store extends SchemaConfigu
 
     StoreFile sf = null;
     try {
-      StoreFile.Writer writer =
-        this.compactor.compact(this, filesToCompact, cr.isMajor(), maxId);
+      StoreFile.Writer writer = this.compactor.compact(cr, maxId);
       // Move the compaction into place.
       if (this.conf.getBoolean("hbase.hstore.compaction.complete", true)) {
         sf = completeCompaction(filesToCompact, writer);
         if (region.getCoprocessorHost() != null) {
-          region.getCoprocessorHost().postCompact(this, sf);
+          region.getCoprocessorHost().postCompact(this, sf, cr);
         }
       } else {
         // Create storefile around what we wrote with a reader on it.
@@ -1175,12 +1174,12 @@ public class Store extends SchemaConfigu
 
     try {
       // Ready to go. Have list of files to compact.
-      StoreFile.Writer writer =
-        this.compactor.compact(this, filesToCompact, isMajor, maxId);
+      StoreFile.Writer writer = this.compactor.compactForTesting(this, conf, filesToCompact,
+        isMajor, maxId);
       // Move the compaction into place.
       StoreFile sf = completeCompaction(filesToCompact, writer);
       if (region.getCoprocessorHost() != null) {
-        region.getCoprocessorHost().postCompact(this, sf);
+        region.getCoprocessorHost().postCompact(this, sf, null);
       }
     } finally {
       synchronized (filesCompacting) {
@@ -1331,16 +1330,16 @@ public class Store extends SchemaConfigu
   }
 
   public CompactionRequest requestCompaction() throws IOException {
-    return requestCompaction(NO_PRIORITY);
+    return requestCompaction(NO_PRIORITY, null);
   }
 
-  public CompactionRequest requestCompaction(int priority) throws IOException {
+  public CompactionRequest requestCompaction(int priority, CompactionRequest request)
+      throws IOException {
     // don't even select for compaction if writes are disabled
     if (!this.region.areWritesEnabled()) {
       return null;
     }
 
-    CompactionRequest ret = null;
     this.lock.readLock().lock();
     try {
       synchronized (filesCompacting) {
@@ -1357,8 +1356,7 @@ public class Store extends SchemaConfigu
 
         boolean override = false;
         if (region.getCoprocessorHost() != null) {
-          override = region.getCoprocessorHost().preCompactSelection(
-              this, candidates);
+          override = region.getCoprocessorHost().preCompactSelection(this, candidates, request);
         }
         CompactSelection filesToCompact;
         if (override) {
@@ -1370,7 +1368,7 @@ public class Store extends SchemaConfigu
 
         if (region.getCoprocessorHost() != null) {
           region.getCoprocessorHost().postCompactSelection(this,
-              ImmutableList.copyOf(filesToCompact.getFilesToCompact()));
+            ImmutableList.copyOf(filesToCompact.getFilesToCompact()), request);
         }
 
         // no files to compact
@@ -1396,15 +1394,24 @@ public class Store extends SchemaConfigu
 
         // everything went better than expected. create a compaction request
         int pri = getCompactPriority(priority);
-        ret = new CompactionRequest(region, this, filesToCompact, isMajor, pri);
+        //not a special compaction request, so we need to make one
+        if(request == null){
+          request = new CompactionRequest(region, this, filesToCompact, isMajor, pri);
+        } else {
+          // update the request with what the system thinks the request should be
+          // its up to the request if it wants to listen
+          request.setSelection(filesToCompact);
+          request.setIsMajor(isMajor);
+          request.setPriority(pri);
+        }
       }
     } finally {
       this.lock.readLock().unlock();
     }
-    if (ret != null) {
-      CompactionRequest.preRequest(ret);
+    if (request != null) {
+      CompactionRequest.preRequest(request);
     }
-    return ret;
+    return request;
   }
 
   public void finishRequest(CompactionRequest cr) {

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/CompactionRequest.java Thu Feb 28 00:33:06 2013
@@ -20,6 +20,8 @@
 package org.apache.hadoop.hbase.regionserver.compactions;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.RejectedExecutionHandler;
@@ -28,6 +30,7 @@ import java.util.concurrent.atomic.Atomi
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.KeyValue.Type;
 import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -51,9 +54,9 @@ public class CompactionRequest implement
     static final Log LOG = LogFactory.getLog(CompactionRequest.class);
     private final HRegion r;
     private final Store s;
-    private final CompactSelection compactSelection;
-    private final long totalSize;
-    private final boolean isMajor;
+    private CompactSelection compactSelection;
+    private long totalSize;
+    private boolean isMajor;
     private int p;
     private final Long timeInNanos;
     private HRegionServer server = null;
@@ -66,23 +69,51 @@ public class CompactionRequest implement
     private static final ConcurrentHashMap<Long, AtomicInteger>
       minorCompactions = new ConcurrentHashMap<Long, AtomicInteger>();
 
-    public CompactionRequest(HRegion r, Store s,
-        CompactSelection files, boolean isMajor, int p) {
-      Preconditions.checkNotNull(r);
-      Preconditions.checkNotNull(files);
+  /**
+   * Create a simple compaction request just for testing - this lets you specify everything you
+   * would need in the general case of testing compactions from an external perspective (e.g.
+   * requesting a compaction through the HRegion).
+   * @param store
+   * @param conf
+   * @param selection
+   * @param isMajor
+   * @return a request that is useful in requesting compactions for testing
+   */
+  public static CompactionRequest getRequestForTesting(Store store, Configuration conf,
+      Collection<StoreFile> selection, boolean isMajor) {
+    return new CompactionRequest(store.getHRegion(), store, new CompactSelection(conf,
+        new ArrayList<StoreFile>(
+        selection)), isMajor, 0, System.nanoTime());
+  }
 
-      this.r = r;
-      this.s = s;
-      this.compactSelection = files;
-      long sz = 0;
-      for (StoreFile sf : files.getFilesToCompact()) {
-        sz += sf.getReader().length();
-      }
-      this.totalSize = sz;
-      this.isMajor = isMajor;
-      this.p = p;
-      this.timeInNanos = System.nanoTime();
+  /**
+   * Constructor for a custom compaction. Uses the setXXX methods to update the state of the
+   * compaction before being used. Uses the current system time on creation as the start time.
+   * @param region region that is being compacted
+   * @param store store which is being compacted
+   * @param priority specified priority with which this compaction should enter the queue.
+   */
+  public CompactionRequest(HRegion region, Store store, int priority) {
+    this(region, store, null, false, priority, System.nanoTime());
+  }
+
+  public CompactionRequest(HRegion r, Store s, CompactSelection files, boolean isMajor, int p) {
+    // delegate to the internal constructor after checking basic preconditions
+    this(Preconditions.checkNotNull(r), s, Preconditions.checkNotNull(files), isMajor, p, System
+        .nanoTime());
+  }
+
+  private CompactionRequest(HRegion region, Store store, CompactSelection files, boolean isMajor,
+      int priority, long startTime) {
+    this.r = region;
+    this.s = store;
+    this.isMajor = isMajor;
+    this.p = priority;
+    this.timeInNanos = startTime;
+    if (files != null) {
+      this.setSelection(files);
     }
+  }
 
     /**
      * Find out if a given region in compaction now.
@@ -216,6 +247,28 @@ public class CompactionRequest implement
       this.server = hrs;
     }
 
+    /**
+     * Set the files (and, implicitly, the size of the compaction based on those files)
+     * @param files files that should be included in the compaction
+     */
+    public void setSelection(CompactSelection files) {
+      long sz = 0;
+      for (StoreFile sf : files.getFilesToCompact()) {
+        sz += sf.getReader().length();
+      }
+      this.totalSize = sz;
+      this.compactSelection = files;
+    }
+
+    /**
+     * Specify if this compaction should be a major compaction based on the state of the store
+     * @param isMajor <tt>true</tt> if the system determines that this compaction should be a major
+     *          compaction
+     */
+    public void setIsMajor(boolean isMajor) {
+      this.isMajor = isMajor;
+    }
+
     @Override
     public String toString() {
       String fsList = Joiner.on(", ").join(
@@ -255,11 +308,11 @@ public class CompactionRequest implement
           server.getMetrics().addCompaction(now - start, this.totalSize);
           // degenerate case: blocked regions require recursive enqueues
           if (s.getCompactPriority() <= 0) {
-            server.compactSplitThread
-              .requestCompaction(r, s, "Recursive enqueue");
+            server.getCompactSplitThread()
+              .requestCompaction(r, s, "Recursive enqueue", null);
           } else {
             // see if the compaction has caused us to exceed max region size
-            server.compactSplitThread.requestSplit(r);
+            server.getCompactSplitThread().requestSplit(r);
           }
         }
       } catch (IOException ex) {
@@ -271,7 +324,7 @@ public class CompactionRequest implement
         server.checkFileSystem();
       } finally {
         s.finishRequest(this);
-        LOG.debug("CompactSplitThread status: " + server.compactSplitThread);
+      LOG.debug("CompactSplitThread status: " + server.getCompactSplitThread());
       }
     }
 
@@ -299,4 +352,4 @@ public class CompactionRequest implement
         }
       }
     }
-  }
+}

Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java?rev=1451057&r1=1451056&r2=1451057&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java Thu Feb 28 00:33:06 2013
@@ -24,10 +24,12 @@ import static org.mockito.Mockito.spy;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.concurrent.CountDownLatch;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -52,9 +54,11 @@ import org.apache.hadoop.hbase.io.hfile.
 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+import org.apache.hadoop.hbase.regionserver.metrics.RegionServerMetrics;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
 import org.mockito.invocation.InvocationOnMock;
 import org.mockito.stubbing.Answer;
 
@@ -566,8 +570,12 @@ public class TestCompaction extends HBas
   }
 
   private void createStoreFile(final HRegion region) throws IOException {
+    createStoreFile(region, Bytes.toString(COLUMN_FAMILY));
+  }
+
+  private void createStoreFile(final HRegion region, String family) throws IOException {
     HRegionIncommon loader = new HRegionIncommon(region);
-    addContent(loader, Bytes.toString(COLUMN_FAMILY));
+    addContent(loader, family);
     loader.flushcache();
   }
 
@@ -589,8 +597,8 @@ public class TestCompaction extends HBas
     long maxId = StoreFile.getMaxSequenceIdInList(storeFiles);
     Compactor tool = new Compactor(this.conf);
 
-    StoreFile.Writer compactedFile =
-      tool.compact(store, storeFiles, false, maxId);
+    StoreFile.Writer compactedFile = tool.compactForTesting(store, this.conf, storeFiles, false,
+      maxId);
 
     // Now lets corrupt the compacted file.
     FileSystem fs = FileSystem.get(conf);
@@ -628,7 +636,7 @@ public class TestCompaction extends HBas
     }
     store.triggerMajorCompaction();
 
-    CompactionRequest request = store.requestCompaction(Store.NO_PRIORITY);
+    CompactionRequest request = store.requestCompaction(Store.NO_PRIORITY, null);
     assertNotNull("Expected to receive a compaction request", request);
     assertEquals(
       "System-requested major compaction should not occur if there are too many store files",
@@ -646,7 +654,7 @@ public class TestCompaction extends HBas
       createStoreFile(r);
     }
     store.triggerMajorCompaction();
-    CompactionRequest request = store.requestCompaction(Store.PRIORITY_USER);
+    CompactionRequest request = store.requestCompaction(Store.PRIORITY_USER, null);
     assertNotNull("Expected to receive a compaction request", request);
     assertEquals(
       "User-requested major compaction should always occur, even if there are too many store files",
@@ -654,6 +662,90 @@ public class TestCompaction extends HBas
       request.isMajor());
   }
 
+  /**
+   * Create a custom compaction request and be sure that we can track it through the queue, knowing
+   * when the compaction is completed.
+   */
+  public void testTrackingCompactionRequest() throws Exception {
+    // setup a compact/split thread on a mock server
+    HRegionServer mockServer = Mockito.mock(HRegionServer.class);
+    Mockito.when(mockServer.getConfiguration()).thenReturn(r.getConf());
+    CompactSplitThread thread = new CompactSplitThread(mockServer);
+    Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);
+    // simple stop for the metrics - we ignore any updates in the test
+    RegionServerMetrics mockMetrics = Mockito.mock(RegionServerMetrics.class);
+    Mockito.when(mockServer.getMetrics()).thenReturn(mockMetrics);
+
+    // setup a region/store with some files
+    Store store = r.getStore(COLUMN_FAMILY);
+    createStoreFile(r);
+    for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) {
+      createStoreFile(r);
+    }
+
+    CountDownLatch latch = new CountDownLatch(1);
+    TrackableCompactionRequest request = new TrackableCompactionRequest(r, store, latch);
+    thread.requestCompaction(r, store, "test custom comapction", Store.PRIORITY_USER, request);
+    // wait for the latch to complete.
+    latch.await();
+
+    thread.interruptIfNecessary();
+  }
+
+  public void testMultipleCustomCompactionRequests() throws Exception {
+    // setup a compact/split thread on a mock server
+    HRegionServer mockServer = Mockito.mock(HRegionServer.class);
+    Mockito.when(mockServer.getConfiguration()).thenReturn(r.getConf());
+    CompactSplitThread thread = new CompactSplitThread(mockServer);
+    Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);
+    // simple stop for the metrics - we ignore any updates in the test
+    RegionServerMetrics mockMetrics = Mockito.mock(RegionServerMetrics.class);
+    Mockito.when(mockServer.getMetrics()).thenReturn(mockMetrics);
+
+    // setup a region/store with some files
+    int numStores = r.getStores().size();
+    List<CompactionRequest> requests = new ArrayList<CompactionRequest>(numStores);
+    CountDownLatch latch = new CountDownLatch(numStores);
+    // create some store files and setup requests for each store on which we want to do a
+    // compaction
+    for (Store store : r.getStores().values()) {
+      createStoreFile(r, store.getColumnFamilyName());
+      createStoreFile(r, store.getColumnFamilyName());
+      createStoreFile(r, store.getColumnFamilyName());
+      requests.add(new TrackableCompactionRequest(r, store, latch));
+    }
+
+    thread.requestCompaction(r, "test mulitple custom comapctions", Store.PRIORITY_USER,
+      Collections.unmodifiableList(requests));
+
+    // wait for the latch to complete.
+    latch.await();
+
+    thread.interruptIfNecessary();
+  }
+
+  /**
+   * Simple {@link CompactionRequest} on which you can wait until the requested compaction finishes.
+   */
+  public static class TrackableCompactionRequest extends CompactionRequest {
+    private CountDownLatch done;
+
+    /**
+     * Constructor for a custom compaction. Uses the setXXX methods to update the state of the
+     * compaction before being used.
+     */
+    public TrackableCompactionRequest(HRegion region, Store store, CountDownLatch finished) {
+      super(region, store, Store.PRIORITY_USER);
+      this.done = finished;
+    }
+
+    @Override
+    public void run() {
+      super.run();
+      this.done.countDown();
+    }
+  }
+
   @org.junit.Rule
   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();