You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2018/01/31 22:24:04 UTC
hbase git commit: HBASE-19528 Major Compaction Tool; ADDENDUM
Repository: hbase
Updated Branches:
refs/heads/master 4b3b627ab -> 60827fc1e
HBASE-19528 Major Compaction Tool; ADDENDUM
Add @ClassRule to new tests and rename those added here so they
align w/ how we name other Test classes.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/60827fc1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/60827fc1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/60827fc1
Branch: refs/heads/master
Commit: 60827fc1ea93ce3c93ea9b86e618e419babed42f
Parents: 4b3b627
Author: Michael Stack <st...@apache.org>
Authored: Wed Jan 31 14:18:20 2018 -0800
Committer: Michael Stack <st...@apache.org>
Committed: Wed Jan 31 14:18:24 2018 -0800
----------------------------------------------------------------------
.../compaction/MajorCompactionRequestTest.java | 166 ------------------
.../util/compaction/MajorCompactorTest.java | 81 ---------
.../compaction/TestMajorCompactionRequest.java | 170 +++++++++++++++++++
.../util/compaction/TestMajorCompactor.java | 86 ++++++++++
4 files changed, 256 insertions(+), 247 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/60827fc1/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequestTest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequestTest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequestTest.java
deleted file mode 100644
index c5ce4e3..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactionRequestTest.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.util.compaction;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Optional;
-import java.util.Set;
-import java.util.stream.Collectors;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.RegionInfo;
-import org.apache.hadoop.hbase.client.RegionInfoBuilder;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
-import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
-import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
-import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.eq;
-import static org.mockito.ArgumentMatchers.isA;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
-@Category({SmallTests.class})
-public class MajorCompactionRequestTest {
-
- private static final HBaseTestingUtility UTILITY = new HBaseTestingUtility();
- private static final String FAMILY = "a";
- private Path rootRegionDir;
- private Path regionStoreDir;
-
- @Before public void setUp() throws Exception {
- rootRegionDir = UTILITY.getDataTestDirOnTestFS("MajorCompactionRequestTest");
- regionStoreDir = new Path(rootRegionDir, FAMILY);
- }
-
- @Test public void testStoresNeedingCompaction() throws Exception {
- // store files older than timestamp
- List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 5, 10);
- MajorCompactionRequest request = makeMockRequest(100, storeFiles, false);
- Optional<MajorCompactionRequest> result =
- request.createRequest(mock(Configuration.class), Sets.newHashSet(FAMILY));
- assertTrue(result.isPresent());
-
- // store files newer than timestamp
- storeFiles = mockStoreFiles(regionStoreDir, 5, 101);
- request = makeMockRequest(100, storeFiles, false);
- result = request.createRequest(mock(Configuration.class), Sets.newHashSet(FAMILY));
- assertFalse(result.isPresent());
- }
-
- @Test public void testIfWeHaveNewReferenceFilesButOldStoreFiles() throws Exception {
- // this tests that reference files that are new, but have older timestamps for the files
- // they reference still will get compacted.
- TableName table = TableName.valueOf("MajorCompactorTest");
- TableDescriptor htd = UTILITY.createTableDescriptor(table, Bytes.toBytes(FAMILY));
- RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
- HRegion region =
- HBaseTestingUtility.createRegionAndWAL(hri, rootRegionDir, UTILITY.getConfiguration(), htd);
-
- Configuration configuration = mock(Configuration.class);
- // the reference file timestamp is newer
- List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 4, 101);
- List<Path> paths = storeFiles.stream().map(StoreFileInfo::getPath).collect(Collectors.toList());
- // the files that are referenced are older, thus we still compact.
- HRegionFileSystem fileSystem =
- mockFileSystem(region.getRegionInfo(), true, storeFiles, 50);
- MajorCompactionRequest majorCompactionRequest = spy(new MajorCompactionRequest(configuration,
- region.getRegionInfo(), Sets.newHashSet(FAMILY), 100));
- doReturn(mock(Connection.class)).when(majorCompactionRequest).getConnection(eq(configuration));
- doReturn(paths).when(majorCompactionRequest).getReferenceFilePaths(any(FileSystem.class),
- any(Path.class));
- doReturn(fileSystem).when(majorCompactionRequest).getFileSystem(any(Connection.class));
- Set<String> result = majorCompactionRequest.getStoresRequiringCompaction(Sets.newHashSet("a"));
- assertEquals(FAMILY, Iterables.getOnlyElement(result));
- }
-
- private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
- List<StoreFileInfo> storeFiles) throws IOException {
- long timestamp = storeFiles.stream().findFirst().get().getModificationTime();
- return mockFileSystem(info, hasReferenceFiles, storeFiles, timestamp);
- }
-
- private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
- List<StoreFileInfo> storeFiles, long referenceFileTimestamp) throws IOException {
- FileSystem fileSystem = mock(FileSystem.class);
- if (hasReferenceFiles) {
- FileStatus fileStatus = mock(FileStatus.class);
- doReturn(referenceFileTimestamp).when(fileStatus).getModificationTime();
- doReturn(fileStatus).when(fileSystem).getFileLinkStatus(isA(Path.class));
- }
- HRegionFileSystem mockSystem = mock(HRegionFileSystem.class);
- doReturn(info).when(mockSystem).getRegionInfo();
- doReturn(regionStoreDir).when(mockSystem).getStoreDir(FAMILY);
- doReturn(hasReferenceFiles).when(mockSystem).hasReferences(anyString());
- doReturn(storeFiles).when(mockSystem).getStoreFiles(anyString());
- doReturn(fileSystem).when(mockSystem).getFileSystem();
- return mockSystem;
- }
-
- private List<StoreFileInfo> mockStoreFiles(Path regionStoreDir, int howMany, long timestamp)
- throws IOException {
- List<StoreFileInfo> infos = Lists.newArrayList();
- int i = 0;
- while (i < howMany) {
- StoreFileInfo storeFileInfo = mock(StoreFileInfo.class);
- doReturn(timestamp).doReturn(timestamp).when(storeFileInfo).getModificationTime();
- doReturn(new Path(regionStoreDir, RandomStringUtils.randomAlphabetic(10))).when(storeFileInfo)
- .getPath();
- infos.add(storeFileInfo);
- i++;
- }
- return infos;
- }
-
- private MajorCompactionRequest makeMockRequest(long timestamp, List<StoreFileInfo> storeFiles,
- boolean references) throws IOException {
- Configuration configuration = mock(Configuration.class);
- RegionInfo regionInfo = mock(RegionInfo.class);
- when(regionInfo.getEncodedName()).thenReturn("HBase");
- when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo"));
- MajorCompactionRequest request =
- new MajorCompactionRequest(configuration, regionInfo, Sets.newHashSet("a"), timestamp);
- MajorCompactionRequest spy = spy(request);
- HRegionFileSystem fileSystem = mockFileSystem(regionInfo, references, storeFiles);
- doReturn(fileSystem).when(spy).getFileSystem(isA(Connection.class));
- doReturn(mock(Connection.class)).when(spy).getConnection(eq(configuration));
- return spy;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/60827fc1/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactorTest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactorTest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactorTest.java
deleted file mode 100644
index 3fb37ec..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/MajorCompactorTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.util.compaction;
-
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
-import org.apache.hadoop.hbase.testclassification.MiscTests;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
-import org.junit.After;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-
-@Category({ MiscTests.class, MediumTests.class })
-public class MajorCompactorTest {
-
- public static final byte[] FAMILY = Bytes.toBytes("a");
- private HBaseTestingUtility utility;
-
- @Before public void setUp() throws Exception {
- utility = new HBaseTestingUtility();
- utility.getConfiguration().setInt("hbase.hfile.compaction.discharger.interval", 10);
- utility.startMiniCluster();
- }
-
- @After public void tearDown() throws Exception {
- utility.shutdownMiniCluster();
- }
-
- @Test public void testCompactingATable() throws Exception {
- TableName tableName = TableName.valueOf("MajorCompactorTest");
- utility.createMultiRegionTable(tableName, FAMILY, 5);
- utility.waitTableAvailable(tableName);
- Connection connection = utility.getConnection();
- Table table = connection.getTable(tableName);
- // write data and flush multiple store files:
- for (int i = 0; i < 5; i++) {
- utility.loadRandomRows(table, FAMILY, 50, 100);
- utility.flush(tableName);
- }
- table.close();
- int numberOfRegions = utility.getAdmin().getRegions(tableName).size();
- int numHFiles = utility.getNumHFiles(tableName, FAMILY);
- // we should have a table with more store files than we would before we major compacted.
- assertTrue(numberOfRegions < numHFiles);
-
- MajorCompactor compactor =
- new MajorCompactor(utility.getConfiguration(), tableName,
- Sets.newHashSet(Bytes.toString(FAMILY)), 1, System.currentTimeMillis(), 200);
- compactor.initializeWorkQueues();
- compactor.compactAllRegions();
- compactor.shutdown();
-
- // verify that the store has been completely major compacted.
- numberOfRegions = utility.getAdmin().getRegions(tableName).size();
- numHFiles = utility.getNumHFiles(tableName, FAMILY);
- assertEquals(numHFiles, numberOfRegions);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/60827fc1/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionRequest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionRequest.java
new file mode 100644
index 0000000..b626481
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactionRequest.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util.compaction;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
+import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isA;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+@Category({SmallTests.class})
+public class TestMajorCompactionRequest {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestMajorCompactionRequest.class);
+
+ private static final HBaseTestingUtility UTILITY = new HBaseTestingUtility();
+ private static final String FAMILY = "a";
+ private Path rootRegionDir;
+ private Path regionStoreDir;
+
+ @Before public void setUp() throws Exception {
+ rootRegionDir = UTILITY.getDataTestDirOnTestFS("TestMajorCompactionRequest");
+ regionStoreDir = new Path(rootRegionDir, FAMILY);
+ }
+
+ @Test public void testStoresNeedingCompaction() throws Exception {
+ // store files older than timestamp
+ List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 5, 10);
+ MajorCompactionRequest request = makeMockRequest(100, storeFiles, false);
+ Optional<MajorCompactionRequest> result =
+ request.createRequest(mock(Configuration.class), Sets.newHashSet(FAMILY));
+ assertTrue(result.isPresent());
+
+ // store files newer than timestamp
+ storeFiles = mockStoreFiles(regionStoreDir, 5, 101);
+ request = makeMockRequest(100, storeFiles, false);
+ result = request.createRequest(mock(Configuration.class), Sets.newHashSet(FAMILY));
+ assertFalse(result.isPresent());
+ }
+
+ @Test public void testIfWeHaveNewReferenceFilesButOldStoreFiles() throws Exception {
+ // this tests that reference files that are new, but have older timestamps for the files
+ // they reference still will get compacted.
+ TableName table = TableName.valueOf("TestMajorCompactor");
+ TableDescriptor htd = UTILITY.createTableDescriptor(table, Bytes.toBytes(FAMILY));
+ RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName()).build();
+ HRegion region =
+ HBaseTestingUtility.createRegionAndWAL(hri, rootRegionDir, UTILITY.getConfiguration(), htd);
+
+ Configuration configuration = mock(Configuration.class);
+ // the reference file timestamp is newer
+ List<StoreFileInfo> storeFiles = mockStoreFiles(regionStoreDir, 4, 101);
+ List<Path> paths = storeFiles.stream().map(StoreFileInfo::getPath).collect(Collectors.toList());
+ // the files that are referenced are older, thus we still compact.
+ HRegionFileSystem fileSystem =
+ mockFileSystem(region.getRegionInfo(), true, storeFiles, 50);
+ MajorCompactionRequest majorCompactionRequest = spy(new MajorCompactionRequest(configuration,
+ region.getRegionInfo(), Sets.newHashSet(FAMILY), 100));
+ doReturn(mock(Connection.class)).when(majorCompactionRequest).getConnection(eq(configuration));
+ doReturn(paths).when(majorCompactionRequest).getReferenceFilePaths(any(FileSystem.class),
+ any(Path.class));
+ doReturn(fileSystem).when(majorCompactionRequest).getFileSystem(any(Connection.class));
+ Set<String> result = majorCompactionRequest.getStoresRequiringCompaction(Sets.newHashSet("a"));
+ assertEquals(FAMILY, Iterables.getOnlyElement(result));
+ }
+
+ private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
+ List<StoreFileInfo> storeFiles) throws IOException {
+ long timestamp = storeFiles.stream().findFirst().get().getModificationTime();
+ return mockFileSystem(info, hasReferenceFiles, storeFiles, timestamp);
+ }
+
+ private HRegionFileSystem mockFileSystem(RegionInfo info, boolean hasReferenceFiles,
+ List<StoreFileInfo> storeFiles, long referenceFileTimestamp) throws IOException {
+ FileSystem fileSystem = mock(FileSystem.class);
+ if (hasReferenceFiles) {
+ FileStatus fileStatus = mock(FileStatus.class);
+ doReturn(referenceFileTimestamp).when(fileStatus).getModificationTime();
+ doReturn(fileStatus).when(fileSystem).getFileLinkStatus(isA(Path.class));
+ }
+ HRegionFileSystem mockSystem = mock(HRegionFileSystem.class);
+ doReturn(info).when(mockSystem).getRegionInfo();
+ doReturn(regionStoreDir).when(mockSystem).getStoreDir(FAMILY);
+ doReturn(hasReferenceFiles).when(mockSystem).hasReferences(anyString());
+ doReturn(storeFiles).when(mockSystem).getStoreFiles(anyString());
+ doReturn(fileSystem).when(mockSystem).getFileSystem();
+ return mockSystem;
+ }
+
+ private List<StoreFileInfo> mockStoreFiles(Path regionStoreDir, int howMany, long timestamp)
+ throws IOException {
+ List<StoreFileInfo> infos = Lists.newArrayList();
+ int i = 0;
+ while (i < howMany) {
+ StoreFileInfo storeFileInfo = mock(StoreFileInfo.class);
+ doReturn(timestamp).doReturn(timestamp).when(storeFileInfo).getModificationTime();
+ doReturn(new Path(regionStoreDir, RandomStringUtils.randomAlphabetic(10))).when(storeFileInfo)
+ .getPath();
+ infos.add(storeFileInfo);
+ i++;
+ }
+ return infos;
+ }
+
+ private MajorCompactionRequest makeMockRequest(long timestamp, List<StoreFileInfo> storeFiles,
+ boolean references) throws IOException {
+ Configuration configuration = mock(Configuration.class);
+ RegionInfo regionInfo = mock(RegionInfo.class);
+ when(regionInfo.getEncodedName()).thenReturn("HBase");
+ when(regionInfo.getTable()).thenReturn(TableName.valueOf("foo"));
+ MajorCompactionRequest request =
+ new MajorCompactionRequest(configuration, regionInfo, Sets.newHashSet("a"), timestamp);
+ MajorCompactionRequest spy = spy(request);
+ HRegionFileSystem fileSystem = mockFileSystem(regionInfo, references, storeFiles);
+ doReturn(fileSystem).when(spy).getFileSystem(isA(Connection.class));
+ doReturn(mock(Connection.class)).when(spy).getConnection(eq(configuration));
+ return spy;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/60827fc1/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactor.java
new file mode 100644
index 0000000..ccf0146
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/compaction/TestMajorCompactor.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util.compaction;
+
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
+import org.junit.After;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+
+@Category({ MiscTests.class, MediumTests.class })
+public class TestMajorCompactor {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestMajorCompactor.class);
+
+ public static final byte[] FAMILY = Bytes.toBytes("a");
+ private HBaseTestingUtility utility;
+
+ @Before public void setUp() throws Exception {
+ utility = new HBaseTestingUtility();
+ utility.getConfiguration().setInt("hbase.hfile.compaction.discharger.interval", 10);
+ utility.startMiniCluster();
+ }
+
+ @After public void tearDown() throws Exception {
+ utility.shutdownMiniCluster();
+ }
+
+ @Test public void testCompactingATable() throws Exception {
+ TableName tableName = TableName.valueOf("TestMajorCompactor");
+ utility.createMultiRegionTable(tableName, FAMILY, 5);
+ utility.waitTableAvailable(tableName);
+ Connection connection = utility.getConnection();
+ Table table = connection.getTable(tableName);
+ // write data and flush multiple store files:
+ for (int i = 0; i < 5; i++) {
+ utility.loadRandomRows(table, FAMILY, 50, 100);
+ utility.flush(tableName);
+ }
+ table.close();
+ int numberOfRegions = utility.getAdmin().getRegions(tableName).size();
+ int numHFiles = utility.getNumHFiles(tableName, FAMILY);
+ // we should have a table with more store files than we would before we major compacted.
+ assertTrue(numberOfRegions < numHFiles);
+
+ MajorCompactor compactor =
+ new MajorCompactor(utility.getConfiguration(), tableName,
+ Sets.newHashSet(Bytes.toString(FAMILY)), 1, System.currentTimeMillis(), 200);
+ compactor.initializeWorkQueues();
+ compactor.compactAllRegions();
+ compactor.shutdown();
+
+ // verify that the store has been completely major compacted.
+ numberOfRegions = utility.getAdmin().getRegions(tableName).size();
+ numHFiles = utility.getNumHFiles(tableName, FAMILY);
+ assertEquals(numHFiles, numberOfRegions);
+ }
+}
\ No newline at end of file