You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2016/12/05 04:23:13 UTC
hbase git commit: HBASE-17241 Avoid compacting already compacted mob
files with _del files (huaxiang sun)
Repository: hbase
Updated Branches:
refs/heads/master c73180d45 -> 0d6b872d9
HBASE-17241 Avoid compacting already compacted mob files with _del files (huaxiang sun)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0d6b872d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0d6b872d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0d6b872d
Branch: refs/heads/master
Commit: 0d6b872d9dca4c040a172eed7b0153ece36c0a0d
Parents: c73180d
Author: tedyu <yu...@gmail.com>
Authored: Sun Dec 4 20:23:06 2016 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Sun Dec 4 20:23:06 2016 -0800
----------------------------------------------------------------------
.../PartitionedMobCompactionRequest.java | 4 +++
.../compactions/PartitionedMobCompactor.java | 20 +++++++++++
.../TestPartitionedMobCompactor.java | 38 +++++++++++++++++---
3 files changed, 57 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d6b872d/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
index e288571..665b5e2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactionRequest.java
@@ -86,6 +86,10 @@ public class PartitionedMobCompactionRequest extends MobCompactionRequest {
public List<FileStatus> listFiles() {
return Collections.unmodifiableList(files);
}
+
+ public int getFileCount () {
+ return files.size();
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d6b872d/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
index 33f5e71..2f7d961 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/compactions/PartitionedMobCompactor.java
@@ -25,6 +25,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -187,6 +188,25 @@ public class PartitionedMobCompactor extends MobCompactor {
selectedFileCount++;
}
}
+
+ /*
+ * If it is not a major mob compaction with del files, and the file number in Partition is 1,
+ * remove the partition from filesToCompact list to avoid re-compacting files which has been
+ * compacted with del files.
+ */
+ if (!allFiles && (allDelFiles.size() > 0)) {
+ Iterator<Map.Entry<CompactionPartitionId, CompactionPartition>> it =
+ filesToCompact.entrySet().iterator();
+
+ while(it.hasNext()) {
+ Map.Entry<CompactionPartitionId, CompactionPartition> entry = it.next();
+ if (entry.getValue().getFileCount() == 1) {
+ it.remove();
+ --selectedFileCount;
+ }
+ }
+ }
+
PartitionedMobCompactionRequest request = new PartitionedMobCompactionRequest(
filesToCompact.values(), allDelFiles);
if (candidates.size() == (allDelFiles.size() + selectedFileCount + irrelevantFileCount)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d6b872d/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
index 7da8544..c112289 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mob/compactions/TestPartitionedMobCompactor.java
@@ -106,11 +106,23 @@ public class TestPartitionedMobCompactor {
@Test
public void testCompactionSelectWithAllFiles() throws Exception {
String tableName = "testCompactionSelectWithAllFiles";
+ // If there is only 1 file, it will not be compacted with _del files, so
+ // It wont be CompactionType.ALL_FILES in this case, do not create with _del files.
testCompactionAtMergeSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD,
- CompactionType.ALL_FILES, false);
+ CompactionType.ALL_FILES, false, false);
}
@Test
+ public void testCompactionSelectToAvoidCompactOneFileWithDelete() throws Exception {
+ String tableName = "testCompactionSelectToAvoidCompactOneFileWithDelete";
+ // If there is only 1 file, it will not be compacted with _del files, so
+ // It wont be CompactionType.ALL_FILES in this case, and expected compact file count will be 0.
+ testCompactionAtMergeSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_MERGEABLE_THRESHOLD,
+ CompactionType.PART_FILES, false);
+ }
+
+
+ @Test
public void testCompactionSelectWithPartFiles() throws Exception {
String tableName = "testCompactionSelectWithPartFiles";
testCompactionAtMergeSize(tableName, 4000, CompactionType.PART_FILES, false);
@@ -125,20 +137,36 @@ public class TestPartitionedMobCompactor {
private void testCompactionAtMergeSize(final String tableName,
final long mergeSize, final CompactionType type, final boolean isForceAllFiles)
throws Exception {
+ testCompactionAtMergeSize(tableName, mergeSize, type, isForceAllFiles, true);
+ }
+
+ private void testCompactionAtMergeSize(final String tableName,
+ final long mergeSize, final CompactionType type, final boolean isForceAllFiles,
+ final boolean createDelFiles)
+ throws Exception {
resetConf();
init(tableName);
int count = 10;
// create 10 mob files.
createStoreFiles(basePath, family, qf, count, Type.Put);
- // create 10 del files
- createStoreFiles(basePath, family, qf, count, Type.Delete);
+
+ if (createDelFiles) {
+ // create 10 del files
+ createStoreFiles(basePath, family, qf, count, Type.Delete);
+ }
+
listFiles();
List<String> expectedStartKeys = new ArrayList<>();
for(FileStatus file : mobFiles) {
if(file.getLen() < mergeSize) {
String fileName = file.getPath().getName();
String startKey = fileName.substring(0, 32);
- expectedStartKeys.add(startKey);
+
+ // If it is not an major mob compaction and del files are there,
+ // these mob files wont be compacted.
+ if (isForceAllFiles || !createDelFiles) {
+ expectedStartKeys.add(startKey);
+ }
}
}
// set the mob compaction mergeable threshold
@@ -150,7 +178,7 @@ public class TestPartitionedMobCompactor {
public void testCompactDelFilesWithDefaultBatchSize() throws Exception {
String tableName = "testCompactDelFilesWithDefaultBatchSize";
testCompactDelFilesAtBatchSize(tableName, MobConstants.DEFAULT_MOB_COMPACTION_BATCH_SIZE,
- MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
+ MobConstants.DEFAULT_MOB_DELFILE_MAX_COUNT);
}
@Test