You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by jx...@apache.org on 2012/10/01 19:21:24 UTC
svn commit: r1392451 - in /hbase/trunk/hbase-server/src:
main/java/org/apache/hadoop/hbase/regionserver/HStore.java
test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
Author: jxiang
Date: Mon Oct 1 17:21:23 2012
New Revision: 1392451
URL: http://svn.apache.org/viewvc?rev=1392451&view=rev
Log:
HBASE-6901 Store file compactSelection throws ArrayIndexOutOfBoundsException
Modified:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=1392451&r1=1392450&r2=1392451&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java Mon Oct 1 17:21:23 2012
@@ -54,7 +54,6 @@ import org.apache.hadoop.hbase.backup.HF
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.io.HFileLink;
-import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.io.hfile.HFile;
@@ -1420,6 +1419,15 @@ public class HStore extends SchemaConfig
int start = 0;
double r = compactSelection.getCompactSelectionRatio();
+ // remove bulk import files that request to be excluded from minors
+ compactSelection.getFilesToCompact().removeAll(Collections2.filter(
+ compactSelection.getFilesToCompact(),
+ new Predicate<StoreFile>() {
+ public boolean apply(StoreFile input) {
+ return input.excludeFromMinorCompaction();
+ }
+ }));
+
// skip selection algorithm if we don't have enough files
if (compactSelection.getFilesToCompact().size() < this.minFilesToCompact) {
if(LOG.isDebugEnabled()) {
@@ -1431,15 +1439,6 @@ public class HStore extends SchemaConfig
return compactSelection;
}
- // remove bulk import files that request to be excluded from minors
- compactSelection.getFilesToCompact().removeAll(Collections2.filter(
- compactSelection.getFilesToCompact(),
- new Predicate<StoreFile>() {
- public boolean apply(StoreFile input) {
- return input.excludeFromMinorCompaction();
- }
- }));
-
/* TODO: add sorting + unit test back in when HBASE-2856 is fixed
// Sort files by size to correct when normal skew is altered by bulk load.
Collections.sort(filesToCompact, StoreFile.Comparators.FILE_SIZE);
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java?rev=1392451&r1=1392450&r2=1392451&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java Mon Oct 1 17:21:23 2012
@@ -671,12 +671,85 @@ public class TestHFileOutputFormat {
}
}
+ /**
+ * This test is to test the scenario happened in HBASE-6901.
+ * All files are bulk loaded and excluded from minor compaction.
+ * Without the fix of HBASE-6901, an ArrayIndexOutOfBoundsException
+ * will be thrown.
+ */
+ @Test
+ public void testExcludeAllFromMinorCompaction() throws Exception {
+ Configuration conf = util.getConfiguration();
+ conf.setInt("hbase.hstore.compaction.min", 2);
+ generateRandomStartKeys(5);
+
+ try {
+ util.startMiniCluster();
+ final FileSystem fs = util.getDFSCluster().getFileSystem();
+ HBaseAdmin admin = new HBaseAdmin(conf);
+ HTable table = util.createTable(TABLE_NAME, FAMILIES);
+ assertEquals("Should start with empty table", 0, util.countRows(table));
+
+ // deep inspection: get the StoreFile dir
+ final Path storePath = HStore.getStoreHomedir(
+ HTableDescriptor.getTableDir(FSUtils.getRootDir(conf), TABLE_NAME),
+ admin.getTableRegions(TABLE_NAME).get(0).getEncodedName(),
+ FAMILIES[0]);
+ assertEquals(0, fs.listStatus(storePath).length);
+
+ // Generate two bulk load files
+ conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
+ true);
+ util.startMiniMapReduceCluster();
+
+ for (int i = 0; i < 2; i++) {
+ Path testDir = util.getDataTestDir("testExcludeAllFromMinorCompaction_" + i);
+ runIncrementalPELoad(conf, table, testDir);
+ // Perform the actual load
+ new LoadIncrementalHFiles(conf).doBulkLoad(testDir, table);
+ }
+
+ // Ensure data shows up
+ int expectedRows = 2 * NMapInputFormat.getNumMapTasks(conf) * ROWSPERSPLIT;
+ assertEquals("LoadIncrementalHFiles should put expected data in table",
+ expectedRows, util.countRows(table));
+
+ // should have a second StoreFile now
+ assertEquals(2, fs.listStatus(storePath).length);
+
+ // minor compactions shouldn't get rid of the file
+ admin.compact(TABLE_NAME);
+ try {
+ quickPoll(new Callable<Boolean>() {
+ public Boolean call() throws Exception {
+ return fs.listStatus(storePath).length == 1;
+ }
+ }, 5000);
+ throw new IOException("SF# = " + fs.listStatus(storePath).length);
+ } catch (AssertionError ae) {
+ // this is expected behavior
+ }
+
+ // a major compaction should work though
+ admin.majorCompact(TABLE_NAME);
+ quickPoll(new Callable<Boolean>() {
+ public Boolean call() throws Exception {
+ return fs.listStatus(storePath).length == 1;
+ }
+ }, 5000);
+
+ } finally {
+ util.shutdownMiniMapReduceCluster();
+ util.shutdownMiniCluster();
+ }
+ }
+
@Test
public void testExcludeMinorCompaction() throws Exception {
Configuration conf = util.getConfiguration();
conf.setInt("hbase.hstore.compaction.min", 2);
Path testDir = util.getDataTestDir("testExcludeMinorCompaction");
- byte[][] startKeys = generateRandomStartKeys(5);
+ generateRandomStartKeys(5);
try {
util.startMiniCluster();