You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2021/07/07 14:02:44 UTC
[hbase] branch branch-2 updated: HBASE-26069 Remove HStore.compactRecentForTestingAssumingDefaultPolic… (#3462)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new b86d2d4 HBASE-26069 Remove HStore.compactRecentForTestingAssumingDefaultPolic… (#3462)
b86d2d4 is described below
commit b86d2d4020f9e6eacb31a17ee28ade7ce56189b5
Author: Duo Zhang <zh...@apache.org>
AuthorDate: Wed Jul 7 21:34:52 2021 +0800
HBASE-26069 Remove HStore.compactRecentForTestingAssumingDefaultPolic… (#3462)
Signed-off-by: Yulin Niu <ni...@apache.org>
---
.../apache/hadoop/hbase/regionserver/HStore.java | 61 ---------
.../regionserver/compactions/DefaultCompactor.java | 22 +--
.../io/hfile/TestScannerSelectionUsingTTL.java | 20 +--
.../hadoop/hbase/regionserver/TestCompaction.java | 63 ++++-----
.../hbase/regionserver/TestMinorCompaction.java | 149 ++++++++++++---------
5 files changed, 125 insertions(+), 190 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index f8efbae..a2abfda 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -1,5 +1,4 @@
/*
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -53,7 +52,6 @@ import java.util.function.Predicate;
import java.util.function.ToLongFunction;
import java.util.stream.Collectors;
import java.util.stream.LongStream;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -89,7 +87,6 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress;
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
-import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
import org.apache.hadoop.hbase.regionserver.compactions.OffPeakHours;
import org.apache.hadoop.hbase.regionserver.querymatcher.ScanQueryMatcher;
import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
@@ -1761,64 +1758,6 @@ public class HStore implements Store, HeapSize, StoreConfigInformation,
}
}
- /**
- * This method tries to compact N recent files for testing.
- * Note that because compacting "recent" files only makes sense for some policies,
- * e.g. the default one, it assumes default policy is used. It doesn't use policy,
- * but instead makes a compaction candidate list by itself.
- * @param N Number of files.
- */
- public void compactRecentForTestingAssumingDefaultPolicy(int N) throws IOException {
- List<HStoreFile> filesToCompact;
- boolean isMajor;
-
- this.lock.readLock().lock();
- try {
- synchronized (filesCompacting) {
- filesToCompact = Lists.newArrayList(storeEngine.getStoreFileManager().getStorefiles());
- if (!filesCompacting.isEmpty()) {
- // exclude all files older than the newest file we're currently
- // compacting. this allows us to preserve contiguity (HBASE-2856)
- HStoreFile last = filesCompacting.get(filesCompacting.size() - 1);
- int idx = filesToCompact.indexOf(last);
- Preconditions.checkArgument(idx != -1);
- filesToCompact.subList(0, idx + 1).clear();
- }
- int count = filesToCompact.size();
- if (N > count) {
- throw new RuntimeException("Not enough files");
- }
-
- filesToCompact = filesToCompact.subList(count - N, count);
- isMajor = (filesToCompact.size() == storeEngine.getStoreFileManager().getStorefileCount());
- filesCompacting.addAll(filesToCompact);
- Collections.sort(filesCompacting, storeEngine.getStoreFileManager()
- .getStoreFileComparator());
- }
- } finally {
- this.lock.readLock().unlock();
- }
-
- try {
- // Ready to go. Have list of files to compact.
- List<Path> newFiles = ((DefaultCompactor)this.storeEngine.getCompactor())
- .compactForTesting(filesToCompact, isMajor);
- for (Path newFile: newFiles) {
- // Move the compaction into place.
- HStoreFile sf = moveFileIntoPlace(newFile);
- if (this.getCoprocessorHost() != null) {
- this.getCoprocessorHost().postCompact(this, sf, null, null, null);
- }
- replaceStoreFiles(filesToCompact, Collections.singletonList(sf));
- refreshStoreSizeAndTotalBytes();
- }
- } finally {
- synchronized (filesCompacting) {
- filesCompacting.removeAll(filesToCompact);
- }
- }
- }
-
@Override
public boolean hasReferences() {
// Grab the read lock here, because we need to ensure that: only when the atomic
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
index 839d067..49d3e8e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DefaultCompactor.java
@@ -18,21 +18,18 @@
package org.apache.hadoop.hbase.regionserver.compactions;
import java.io.IOException;
-import java.util.Collection;
import java.util.List;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.regionserver.HStore;
-import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.StoreFileWriter;
-import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
import org.apache.hadoop.hbase.regionserver.throttle.ThroughputController;
import org.apache.hadoop.hbase.security.User;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
/**
@@ -65,23 +62,6 @@ public class DefaultCompactor extends Compactor<StoreFileWriter> {
return compact(request, defaultScannerFactory, writerFactory, throughputController, user);
}
- /**
- * Compact a list of files for testing. Creates a fake {@link CompactionRequestImpl} to pass to
- * {@link #compact(CompactionRequestImpl, ThroughputController, User)};
- * @param filesToCompact the files to compact. These are used as the compactionSelection for the
- * generated {@link CompactionRequestImpl}.
- * @param isMajor true to major compact (prune all deletes, max versions, etc)
- * @return Product of compaction or an empty list if all cells expired or deleted and nothing \
- * made it through the compaction.
- * @throws IOException
- */
- public List<Path> compactForTesting(Collection<HStoreFile> filesToCompact, boolean isMajor)
- throws IOException {
- CompactionRequestImpl cr = new CompactionRequestImpl(filesToCompact);
- cr.setIsMajor(isMajor, isMajor);
- return compact(cr, NoLimitThroughputController.INSTANCE, null);
- }
-
@Override
protected List<Path> commitWriter(StoreFileWriter writer, FileDetails fd,
CompactionRequestImpl request) throws IOException {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java
index 262c8ec..65c6ee8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerSelectionUsingTTL.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.testclassification.IOTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
@@ -82,25 +81,18 @@ public class TestScannerSelectionUsingTTL {
public final int numFreshFiles, totalNumFiles;
- /** Whether we are specifying the exact files to compact */
- private final boolean explicitCompaction;
-
@Parameters
public static Collection<Object[]> parameters() {
List<Object[]> params = new ArrayList<>();
for (int numFreshFiles = 1; numFreshFiles <= 3; ++numFreshFiles) {
- for (boolean explicitCompaction : new boolean[] { false, true }) {
- params.add(new Object[] { numFreshFiles, explicitCompaction });
- }
+ params.add(new Object[] { numFreshFiles });
}
return params;
}
- public TestScannerSelectionUsingTTL(int numFreshFiles,
- boolean explicitCompaction) {
+ public TestScannerSelectionUsingTTL(int numFreshFiles) {
this.numFreshFiles = numFreshFiles;
this.totalNumFiles = numFreshFiles + NUM_EXPIRED_FILES;
- this.explicitCompaction = explicitCompaction;
}
@Test
@@ -152,13 +144,7 @@ public class TestScannerSelectionUsingTTL {
Set<String> accessedFiles = cache.getCachedFileNamesForTest();
LOG.debug("Files accessed during scan: " + accessedFiles);
- // Exercise both compaction codepaths.
- if (explicitCompaction) {
- HStore store = region.getStore(FAMILY_BYTES);
- store.compactRecentForTestingAssumingDefaultPolicy(totalNumFiles);
- } else {
- region.compact(false);
- }
+ region.compact(false);
HBaseTestingUtility.closeRegionAndWAL(region);
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
index cbc43f0..163560f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
@@ -81,6 +81,8 @@ import org.junit.rules.TestName;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Test compaction framework and common functions
@@ -90,10 +92,13 @@ public class TestCompaction {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestCompaction.class);
+ HBaseClassTestRule.forClass(TestCompaction.class);
- @Rule public TestName name = new TestName();
- private static final HBaseTestingUtility UTIL = HBaseTestingUtility.createLocalHTU();
+ private static final Logger LOG = LoggerFactory.getLogger(TestCompaction.class);
+
+ @Rule
+ public TestName name = new TestName();
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
protected Configuration conf = UTIL.getConfiguration();
private HRegion r = null;
@@ -150,7 +155,6 @@ public class TestCompaction {
/**
* Verify that you can stop a long-running compaction
* (used during RS shutdown)
- * @throws Exception
*/
@Test
public void testInterruptCompaction() throws Exception {
@@ -177,7 +181,7 @@ public class TestCompaction {
}
HRegion spyR = spy(r);
- doAnswer(new Answer() {
+ doAnswer(new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
r.writestate.writesEnabled = false;
@@ -231,15 +235,14 @@ public class TestCompaction {
private int count() throws IOException {
int count = 0;
- for (HStoreFile f: this.r.stores.
- get(COLUMN_FAMILY_TEXT).getStorefiles()) {
+ for (HStoreFile f : this.r.stores.get(COLUMN_FAMILY_TEXT).getStorefiles()) {
HFileScanner scanner = f.getReader().getScanner(false, false);
if (!scanner.seekTo()) {
continue;
}
do {
count++;
- } while(scanner.next());
+ } while (scanner.next());
}
return count;
}
@@ -264,7 +267,8 @@ public class TestCompaction {
Collection<HStoreFile> storeFiles = store.getStorefiles();
DefaultCompactor tool = (DefaultCompactor)store.storeEngine.getCompactor();
- tool.compactForTesting(storeFiles, false);
+ CompactionRequestImpl request = new CompactionRequestImpl(storeFiles);
+ tool.compact(request, NoLimitThroughputController.INSTANCE, null);
// Now lets corrupt the compacted file.
FileSystem fs = store.getFileSystem();
@@ -283,7 +287,7 @@ public class TestCompaction {
// in the 'tmp' directory;
assertTrue(fs.exists(origPath));
assertFalse(fs.exists(dstPath));
- System.out.println("testCompactionWithCorruptResult Passed");
+ LOG.info("testCompactionWithCorruptResult Passed");
return;
}
fail("testCompactionWithCorruptResult failed since no exception was" +
@@ -338,28 +342,27 @@ public class TestCompaction {
Mockito.when(mockRegion.checkSplit()).
thenThrow(new RuntimeException("Thrown intentionally by test!"));
- MetricsRegionWrapper metricsWrapper = new MetricsRegionWrapperImpl(r);
+ try (MetricsRegionWrapperImpl metricsWrapper = new MetricsRegionWrapperImpl(r)) {
- long preCompletedCount = metricsWrapper.getNumCompactionsCompleted();
- long preFailedCount = metricsWrapper.getNumCompactionsFailed();
+ long preCompletedCount = metricsWrapper.getNumCompactionsCompleted();
+ long preFailedCount = metricsWrapper.getNumCompactionsFailed();
- CountDownLatch latch = new CountDownLatch(1);
- Tracker tracker = new Tracker(latch);
- thread.requestCompaction(mockRegion, store, "test custom comapction", PRIORITY_USER,
- tracker, null);
- // wait for the latch to complete.
- latch.await(120, TimeUnit.SECONDS);
-
- // compaction should have completed and been marked as failed due to error in split request
- long postCompletedCount = metricsWrapper.getNumCompactionsCompleted();
- long postFailedCount = metricsWrapper.getNumCompactionsFailed();
-
- assertTrue("Completed count should have increased (pre=" + preCompletedCount +
- ", post="+postCompletedCount+")",
- postCompletedCount > preCompletedCount);
- assertTrue("Failed count should have increased (pre=" + preFailedCount +
- ", post=" + postFailedCount + ")",
- postFailedCount > preFailedCount);
+ CountDownLatch latch = new CountDownLatch(1);
+ Tracker tracker = new Tracker(latch);
+ thread.requestCompaction(mockRegion, store, "test custom comapction", PRIORITY_USER, tracker,
+ null);
+ // wait for the latch to complete.
+ latch.await(120, TimeUnit.SECONDS);
+
+ // compaction should have completed and been marked as failed due to error in split request
+ long postCompletedCount = metricsWrapper.getNumCompactionsCompleted();
+ long postFailedCount = metricsWrapper.getNumCompactionsFailed();
+
+ assertTrue("Completed count should have increased (pre=" + preCompletedCount + ", post=" +
+ postCompletedCount + ")", postCompletedCount > preCompletedCount);
+ assertTrue("Failed count should have increased (pre=" + preFailedCount + ", post=" +
+ postFailedCount + ")", postFailedCount > preFailedCount);
+ }
}
/**
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
index aa9cb53..a650964 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinorCompaction.java
@@ -22,29 +22,38 @@ import static org.apache.hadoop.hbase.HBaseTestingUtility.fam1;
import static org.apache.hadoop.hbase.HBaseTestingUtility.fam2;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequestImpl;
+import org.apache.hadoop.hbase.regionserver.compactions.RatioBasedCompactionPolicy;
+import org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.WAL;
import org.junit.After;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Test minor compactions
@@ -56,35 +65,51 @@ public class TestMinorCompaction {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestMinorCompaction.class);
- @Rule public TestName name = new TestName();
- private static final Logger LOG = LoggerFactory.getLogger(TestMinorCompaction.class.getName());
- private static final HBaseTestingUtility UTIL = HBaseTestingUtility.createLocalHTU();
- protected Configuration conf = UTIL.getConfiguration();
+ @Rule
+ public TestName name = new TestName();
+ private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
+ private static Configuration CONF = UTIL.getConfiguration();
private HRegion r = null;
- private HTableDescriptor htd = null;
- private int compactionThreshold;
- private byte[] firstRowBytes, secondRowBytes, thirdRowBytes;
- final private byte[] col1, col2;
+ private TableDescriptor htd = null;
+ private static int COMPACTION_THRESHOLD;
+ private static byte[] FIRST_ROW_BYTES, SECOND_ROW_BYTES, THIRD_ROW_BYTES;
+ private static byte[] COL1, COL2;
+
+ public static final class MyCompactionPolicy extends RatioBasedCompactionPolicy {
+
+ public MyCompactionPolicy(Configuration conf, StoreConfigInformation storeConfigInfo) {
+ super(conf, storeConfigInfo);
+ }
- /** constructor */
- public TestMinorCompaction() {
- super();
+ @Override
+ public CompactionRequestImpl selectCompaction(Collection<HStoreFile> candidateFiles,
+ List<HStoreFile> filesCompacting, boolean isUserCompaction, boolean mayUseOffPeak,
+ boolean forceMajor) throws IOException {
+ return new CompactionRequestImpl(
+ candidateFiles.stream().filter(f -> !filesCompacting.contains(f))
+ .limit(COMPACTION_THRESHOLD).collect(Collectors.toList()));
+ }
+ }
+ @BeforeClass
+ public static void setUpBeforeClass() {
// Set cache flush size to 1MB
- conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024);
- conf.setInt(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 100);
- compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
+ CONF.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024);
+ CONF.setInt(HConstants.HREGION_MEMSTORE_BLOCK_MULTIPLIER, 100);
+ COMPACTION_THRESHOLD = CONF.getInt("hbase.hstore.compactionThreshold", 3);
+ CONF.setClass(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, MyCompactionPolicy.class,
+ RatioBasedCompactionPolicy.class);
- firstRowBytes = START_KEY_BYTES;
- secondRowBytes = START_KEY_BYTES.clone();
+ FIRST_ROW_BYTES = START_KEY_BYTES;
+ SECOND_ROW_BYTES = START_KEY_BYTES.clone();
// Increment the least significant character so we get to next row.
- secondRowBytes[START_KEY_BYTES.length - 1]++;
- thirdRowBytes = START_KEY_BYTES.clone();
- thirdRowBytes[START_KEY_BYTES.length - 1] =
- (byte) (thirdRowBytes[START_KEY_BYTES.length - 1] + 2);
- col1 = Bytes.toBytes("column1");
- col2 = Bytes.toBytes("column2");
+ SECOND_ROW_BYTES[START_KEY_BYTES.length - 1]++;
+ THIRD_ROW_BYTES = START_KEY_BYTES.clone();
+ THIRD_ROW_BYTES[START_KEY_BYTES.length - 1] =
+ (byte) (THIRD_ROW_BYTES[START_KEY_BYTES.length - 1] + 2);
+ COL1 = Bytes.toBytes("column1");
+ COL2 = Bytes.toBytes("column2");
}
@Before
@@ -95,29 +120,29 @@ public class TestMinorCompaction {
@After
public void tearDown() throws Exception {
- WAL wal = ((HRegion)r).getWAL();
- ((HRegion)r).close();
+ WAL wal = ((HRegion) r).getWAL();
+ ((HRegion) r).close();
wal.close();
}
@Test
public void testMinorCompactionWithDeleteRow() throws Exception {
- Delete deleteRow = new Delete(secondRowBytes);
+ Delete deleteRow = new Delete(SECOND_ROW_BYTES);
testMinorCompactionWithDelete(deleteRow);
}
@Test
public void testMinorCompactionWithDeleteColumn1() throws Exception {
- Delete dc = new Delete(secondRowBytes);
+ Delete dc = new Delete(SECOND_ROW_BYTES);
/* delete all timestamps in the column */
- dc.addColumns(fam2, col2);
+ dc.addColumns(fam2, COL2);
testMinorCompactionWithDelete(dc);
}
@Test
public void testMinorCompactionWithDeleteColumn2() throws Exception {
- Delete dc = new Delete(secondRowBytes);
- dc.addColumn(fam2, col2);
+ Delete dc = new Delete(SECOND_ROW_BYTES);
+ dc.addColumn(fam2, COL2);
/* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3.
* we only delete the latest version. One might expect to see only
* versions 1 and 2. HBase differs, and gives us 0, 1 and 2.
@@ -129,15 +154,15 @@ public class TestMinorCompaction {
@Test
public void testMinorCompactionWithDeleteColumnFamily() throws Exception {
- Delete deleteCF = new Delete(secondRowBytes);
+ Delete deleteCF = new Delete(SECOND_ROW_BYTES);
deleteCF.addFamily(fam2);
testMinorCompactionWithDelete(deleteCF);
}
@Test
public void testMinorCompactionWithDeleteVersion1() throws Exception {
- Delete deleteVersion = new Delete(secondRowBytes);
- deleteVersion.addColumns(fam2, col2, 2);
+ Delete deleteVersion = new Delete(SECOND_ROW_BYTES);
+ deleteVersion.addColumns(fam2, COL2, 2);
/* compactionThreshold is 3. The table has 4 versions: 0, 1, 2, and 3.
* We delete versions 0 ... 2. So, we still have one remaining.
*/
@@ -146,8 +171,8 @@ public class TestMinorCompaction {
@Test
public void testMinorCompactionWithDeleteVersion2() throws Exception {
- Delete deleteVersion = new Delete(secondRowBytes);
- deleteVersion.addColumn(fam2, col2, 1);
+ Delete deleteVersion = new Delete(SECOND_ROW_BYTES);
+ deleteVersion.addColumn(fam2, COL2, 1);
/*
* the table has 4 versions: 0, 1, 2, and 3.
* We delete 1.
@@ -169,22 +194,22 @@ public class TestMinorCompaction {
private void testMinorCompactionWithDelete(Delete delete, int expectedResultsAfterDelete)
throws Exception {
Table loader = new RegionAsTable(r);
- for (int i = 0; i < compactionThreshold + 1; i++) {
- HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col1), firstRowBytes,
- thirdRowBytes, i);
- HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(col2), firstRowBytes,
- thirdRowBytes, i);
- HBaseTestCase.addContent(loader, Bytes.toString(fam2), Bytes.toString(col1), firstRowBytes,
- thirdRowBytes, i);
- HBaseTestCase.addContent(loader, Bytes.toString(fam2), Bytes.toString(col2), firstRowBytes,
- thirdRowBytes, i);
+ for (int i = 0; i < COMPACTION_THRESHOLD + 1; i++) {
+ HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(COL1), FIRST_ROW_BYTES,
+ THIRD_ROW_BYTES, i);
+ HBaseTestCase.addContent(loader, Bytes.toString(fam1), Bytes.toString(COL2), FIRST_ROW_BYTES,
+ THIRD_ROW_BYTES, i);
+ HBaseTestCase.addContent(loader, Bytes.toString(fam2), Bytes.toString(COL1), FIRST_ROW_BYTES,
+ THIRD_ROW_BYTES, i);
+ HBaseTestCase.addContent(loader, Bytes.toString(fam2), Bytes.toString(COL2), FIRST_ROW_BYTES,
+ THIRD_ROW_BYTES, i);
r.flush(true);
}
- Result result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
- assertEquals(compactionThreshold, result.size());
- result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
- assertEquals(compactionThreshold, result.size());
+ Result result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
+ assertEquals(COMPACTION_THRESHOLD, result.size());
+ result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
+ assertEquals(COMPACTION_THRESHOLD, result.size());
// Now add deletes to memstore and then flush it. That will put us over
// the compaction threshold of 3 store files. Compacting these store files
@@ -193,28 +218,30 @@ public class TestMinorCompaction {
r.delete(delete);
// Make sure that we have only deleted family2 from secondRowBytes
- result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
+ result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
assertEquals(expectedResultsAfterDelete, result.size());
// but we still have firstrow
- result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
- assertEquals(compactionThreshold, result.size());
+ result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
+ assertEquals(COMPACTION_THRESHOLD, result.size());
r.flush(true);
// should not change anything.
// Let us check again
// Make sure that we have only deleted family2 from secondRowBytes
- result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
+ result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
assertEquals(expectedResultsAfterDelete, result.size());
// but we still have firstrow
- result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
- assertEquals(compactionThreshold, result.size());
+ result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
+ assertEquals(COMPACTION_THRESHOLD, result.size());
// do a compaction
HStore store2 = r.getStore(fam2);
int numFiles1 = store2.getStorefiles().size();
- assertTrue("Was expecting to see 4 store files", numFiles1 > compactionThreshold); // > 3
- ((HStore)store2).compactRecentForTestingAssumingDefaultPolicy(compactionThreshold); // = 3
+ assertTrue("Was expecting to see 4 store files", numFiles1 > COMPACTION_THRESHOLD); // > 3
+ Optional<CompactionContext> compaction = store2.requestCompaction();
+ assertTrue(compaction.isPresent());
+ store2.compact(compaction.get(), NoLimitThroughputController.INSTANCE, null); // = 3
int numFiles2 = store2.getStorefiles().size();
// Check that we did compact
assertTrue("Number of store files should go down", numFiles1 > numFiles2);
@@ -222,10 +249,10 @@ public class TestMinorCompaction {
assertTrue("Was not supposed to be a major compaction", numFiles2 > 1);
// Make sure that we have only deleted family2 from secondRowBytes
- result = r.get(new Get(secondRowBytes).addColumn(fam2, col2).setMaxVersions(100));
+ result = r.get(new Get(SECOND_ROW_BYTES).addColumn(fam2, COL2).readVersions(100));
assertEquals(expectedResultsAfterDelete, result.size());
// but we still have firstrow
- result = r.get(new Get(firstRowBytes).addColumn(fam1, col1).setMaxVersions(100));
- assertEquals(compactionThreshold, result.size());
+ result = r.get(new Get(FIRST_ROW_BYTES).addColumn(fam1, COL1).readVersions(100));
+ assertEquals(COMPACTION_THRESHOLD, result.size());
}
}