You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by se...@apache.org on 2013/09/26 00:17:35 UTC
svn commit: r1526320 - in /hbase/trunk/hbase-server/src:
main/java/org/apache/hadoop/hbase/regionserver/compactions/
test/java/org/apache/hadoop/hbase/
test/java/org/apache/hadoop/hbase/coprocessor/
test/java/org/apache/hadoop/hbase/master/cleaner/
Author: sershe
Date: Wed Sep 25 22:17:35 2013
New Revision: 1526320
URL: http://svn.apache.org/r1526320
Log:
HBASE-8870 Store.needsCompaction() should include minFilesToCompact (Liang Xie)
Modified:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java?rev=1526320&r1=1526319&r2=1526320&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/RatioBasedCompactionPolicy.java Wed Sep 25 22:17:35 2013
@@ -392,6 +392,6 @@ public class RatioBasedCompactionPolicy
public boolean needsCompaction(final Collection<StoreFile> storeFiles,
final List<StoreFile> filesCompacting) {
int numCandidates = storeFiles.size() - filesCompacting.size();
- return numCandidates > comConf.getMinFilesToCompact();
+ return numCandidates >= comConf.getMinFilesToCompact();
}
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1526320&r1=1526319&r2=1526320&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Wed Sep 25 22:17:35 2013
@@ -1167,28 +1167,40 @@ public class HBaseTestingUtility extends
/**
* Create a table.
- * @param tableName
+ * @param htd
* @param families
* @param c Configuration to use
* @return An HTable instance for the created table.
* @throws IOException
*/
- public HTable createTable(TableName tableName, byte[][] families,
- final Configuration c)
+ public HTable createTable(HTableDescriptor htd, byte[][] families, Configuration c)
throws IOException {
- HTableDescriptor desc = new HTableDescriptor(tableName);
for(byte[] family : families) {
HColumnDescriptor hcd = new HColumnDescriptor(family);
// Disable blooms (they are on by default as of 0.95) but we disable them here because
// tests have hard coded counts of what to expect in block cache, etc., and blooms being
// on is interfering.
hcd.setBloomFilterType(BloomType.NONE);
- desc.addFamily(hcd);
+ htd.addFamily(hcd);
}
- getHBaseAdmin().createTable(desc);
+ getHBaseAdmin().createTable(htd);
// HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned
- waitUntilAllRegionsAssigned(tableName);
- return new HTable(c, tableName);
+ waitUntilAllRegionsAssigned(htd.getTableName());
+ return new HTable(c, htd.getTableName());
+ }
+
+ /**
+ * Create a table.
+ * @param tableName
+ * @param families
+ * @param c Configuration to use
+ * @return An HTable instance for the created table.
+ * @throws IOException
+ */
+ public HTable createTable(TableName tableName, byte[][] families,
+ final Configuration c)
+ throws IOException {
+ return createTable(new HTableDescriptor(tableName), families, c);
}
/**
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java?rev=1526320&r1=1526319&r2=1526320&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java Wed Sep 25 22:17:35 2013
@@ -55,9 +55,7 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.ScanType;
import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.regionserver.StoreScanner;
-import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -249,14 +247,8 @@ public class TestRegionObserverScannerOp
admin.flush(region.getRegionName());
// run a compaction, which normally would should get rid of the data
- Store s = region.getStores().get(A);
- CountDownLatch latch = new CountDownLatch(1);
- WaitableCompactionRequest request = new WaitableCompactionRequest(s.getStorefiles(), latch);
- rs.compactSplitThread.requestCompaction(region, s,
- "compact for testRegionObserverCompactionTimeStacking", Store.PRIORITY_USER, request);
- // wait for the compaction to complete
- latch.await();
-
+ // wait for the compaction checker to complete
+ Thread.sleep(1000);
// check both rows to ensure that they aren't there
Get get = new Get(ROW);
Result r = table.get(get);
@@ -273,26 +265,4 @@ public class TestRegionObserverScannerOp
table.close();
UTIL.shutdownMiniCluster();
}
-
- /**
- * A simple compaction on which you can wait for the passed in latch until the compaction finishes
- * (either successfully or if it failed).
- */
- public static class WaitableCompactionRequest extends CompactionRequest {
- private CountDownLatch done;
-
- /**
- * Constructor for a custom compaction. Uses the setXXX methods to update the state of the
- * compaction before being used.
- */
- public WaitableCompactionRequest(Collection<StoreFile> files, CountDownLatch finished) {
- super(files);
- this.done = finished;
- }
-
- @Override
- public void afterExecute() {
- this.done.countDown();
- }
- }
}
\ No newline at end of file
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java?rev=1526320&r1=1526319&r2=1526320&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/cleaner/TestSnapshotFromMaster.java Wed Sep 25 22:17:35 2013
@@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
@@ -280,11 +281,19 @@ public class TestSnapshotFromMaster {
HBaseAdmin admin = UTIL.getHBaseAdmin();
// make sure we don't fail on listing snapshots
SnapshotTestingUtils.assertNoSnapshots(admin);
+
+ // recreate test table with disabled compactions; otherwise compaction may happen before
+ // snapshot, the call after snapshot will be a no-op and checks will fail
+ UTIL.deleteTable(TABLE_NAME);
+ HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
+ htd.setCompactionEnabled(false);
+ UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
// load the table (creates 4 hfiles)
UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
// disable the table so we can take a snapshot
admin.disableTable(TABLE_NAME);
+ htd.setCompactionEnabled(true);
// take a snapshot of the table
String snapshotName = "snapshot";
@@ -298,6 +307,9 @@ public class TestSnapshotFromMaster {
// ensure we only have one snapshot
SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshotNameBytes, TABLE_NAME);
+ // enable compactions now
+ admin.modifyTable(TABLE_NAME, htd);
+
// renable the table so we can compact the regions
admin.enableTable(TABLE_NAME);