You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ni...@apache.org on 2021/03/04 02:15:40 UTC
[hbase] branch branch-2 updated: HBASE-25603 Add switch for
compaction after bulkload (#2982)
This is an automated email from the ASF dual-hosted git repository.
niuyulin pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 621edfe HBASE-25603 Add switch for compaction after bulkload (#2982)
621edfe is described below
commit 621edfe1da59ebc2b45bfbcc2486400573a784d3
Author: niuyulin <yu...@gmail.com>
AuthorDate: Wed Mar 3 20:09:08 2021 -0600
HBASE-25603 Add switch for compaction after bulkload (#2982)
Signed-off-by: Duo Zhang <zh...@apache.org>
Signed-off-by: Wellington Chevreuil <wc...@apache.org>
---
hbase-common/src/main/resources/hbase-default.xml | 7 ++++
.../apache/hadoop/hbase/regionserver/HRegion.java | 32 +++++++++------
.../regionserver/TestCompactionAfterBulkLoad.java | 46 ++++++++++++----------
src/main/asciidoc/_chapters/hbase-default.adoc | 11 ++++++
4 files changed, 64 insertions(+), 32 deletions(-)
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index 0908ca6..83427f0 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -594,6 +594,13 @@ possible configurations would overwhelm and obscure the important.
0 means never give up.</description>
</property>
<property>
+ <name>hbase.compaction.after.bulkload.enable</name>
+ <value>true</value>
+ <description>Request Compaction after bulkload immediately.
+ If bulkload is continuous, the triggered compactions may increase load,
+ bring about performance side effect.</description>
+ </property>
+ <property>
<name>hbase.master.balancer.maxRitPercent</name>
<value>1.0</value>
<description>The max percent of regions in transition when balancing.
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index fbaaeb0..a582294 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -238,6 +238,10 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
public static final String WAL_HSYNC_CONF_KEY = "hbase.wal.hsync";
public static final boolean DEFAULT_WAL_HSYNC = false;
+ /** Parameter name for compaction after bulkload */
+ public static final String COMPACTION_AFTER_BULKLOAD_ENABLE =
+ "hbase.compaction.after.bulkload.enable";
+
/**
* This is for for using HRegion as a local storage, where we may put the recovered edits in a
* special place. Once this is set, we will only replay the recovered edits under this directory
@@ -6971,19 +6975,23 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
}
isSuccessful = true;
- //request compaction
- familyWithFinalPath.keySet().forEach(family -> {
- HStore store = getStore(family);
- try {
- if (this.rsServices != null && store.needsCompaction()) {
- this.rsServices.getCompactionRequestor().requestCompaction(this, store,
- "bulkload hfiles request compaction", Store.PRIORITY_USER + 1,
- CompactionLifeCycleTracker.DUMMY, null);
+ if (conf.getBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, true)) {
+ // request compaction
+ familyWithFinalPath.keySet().forEach(family -> {
+ HStore store = getStore(family);
+ try {
+ if (this.rsServices != null && store.needsCompaction()) {
+ this.rsServices.getCompactionRequestor().requestCompaction(this, store,
+ "bulkload hfiles request compaction", Store.PRIORITY_USER + 1,
+ CompactionLifeCycleTracker.DUMMY, null);
+ LOG.debug("bulkload hfiles request compaction region : {}, family : {}",
+ this.getRegionInfo(), family);
+ }
+ } catch (IOException e) {
+ LOG.error("bulkload hfiles request compaction error ", e);
}
- } catch (IOException e) {
- LOG.error("bulkload hfiles request compaction error ", e);
- }
- });
+ });
+ }
} finally {
if (wal != null && !storeFiles.isEmpty()) {
// Write a bulk load event for hfiles that are loaded
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionAfterBulkLoad.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionAfterBulkLoad.java
index 423659e..c736513 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionAfterBulkLoad.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactionAfterBulkLoad.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import static org.apache.hadoop.hbase.regionserver.HRegion.COMPACTION_AFTER_BULKLOAD_ENABLE;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.eq;
@@ -84,27 +85,32 @@ public class TestCompactionAfterBulkLoad extends TestBulkloadBase {
for (int i = 0; i < 5; i++) {
familyPaths.addAll(withFamilyPathsFor(family1, family2, family3));
}
- when(regionServerServices.getConfiguration()).thenReturn(conf);
- when(regionServerServices.getCompactionRequestor()).thenReturn(compactionRequester);
- when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD))))
- .thenAnswer(new Answer() {
- @Override
- public Object answer(InvocationOnMock invocation) {
- WALKeyImpl walKey = invocation.getArgument(1);
- MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
- if (mvcc != null) {
- MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
- walKey.setWriteEntry(we);
+ try {
+ conf.setBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, true);
+ when(regionServerServices.getConfiguration()).thenReturn(conf);
+ when(regionServerServices.getCompactionRequestor()).thenReturn(compactionRequester);
+ when(log.appendMarker(any(), any(), argThat(bulkLogWalEditType(WALEdit.BULK_LOAD))))
+ .thenAnswer(new Answer() {
+ @Override
+ public Object answer(InvocationOnMock invocation) {
+ WALKeyImpl walKey = invocation.getArgument(1);
+ MultiVersionConcurrencyControl mvcc = walKey.getMvcc();
+ if (mvcc != null) {
+ MultiVersionConcurrencyControl.WriteEntry we = mvcc.begin();
+ walKey.setWriteEntry(we);
+ }
+ return 01L;
}
- return 01L;
- }
- });
+ });
- Mockito.doNothing().when(compactionRequester).requestCompaction(any(), any(), any(), anyInt(),
- any(), any());
- testRegionWithFamilies(family1, family2, family3).bulkLoadHFiles(familyPaths, false, null);
- // invoke three times for 3 families
- verify(compactionRequester, times(3)).requestCompaction(isA(HRegion.class), isA(HStore.class),
- isA(String.class), anyInt(), eq(CompactionLifeCycleTracker.DUMMY), eq(null));
+ Mockito.doNothing().when(compactionRequester).requestCompaction(any(), any(), any(), anyInt(),
+ any(), any());
+ testRegionWithFamilies(family1, family2, family3).bulkLoadHFiles(familyPaths, false, null);
+ // invoke three times for 3 families
+ verify(compactionRequester, times(3)).requestCompaction(isA(HRegion.class), isA(HStore.class),
+ isA(String.class), anyInt(), eq(CompactionLifeCycleTracker.DUMMY), eq(null));
+ } finally {
+ conf.setBoolean(COMPACTION_AFTER_BULKLOAD_ENABLE, false);
+ }
}
}
diff --git a/src/main/asciidoc/_chapters/hbase-default.adoc b/src/main/asciidoc/_chapters/hbase-default.adoc
index 1c373e9..acab813 100644
--- a/src/main/asciidoc/_chapters/hbase-default.adoc
+++ b/src/main/asciidoc/_chapters/hbase-default.adoc
@@ -756,6 +756,17 @@ Maximum retries. This is a maximum number of iterations
`10`
+[[hbase.compaction.after.bulkload.enable]]
+*`hbase.compaction.after.bulkload.enable`*::
++
+.Description
+Request Compaction after bulkload immediately.
+If bulkload is continuous, the triggered compactions may increase load,
+bring about performance side effect.
++
+.Default
+`true`
+
[[hbase.balancer.period
]]
*`hbase.balancer.period