You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2021/12/18 15:37:21 UTC
[hbase] branch branch-2.4 updated (e20b2f3 -> 9c01d04)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a change to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git.
from e20b2f3 HBASE-26488 Memory leak when MemStore retry flushing (#3899)
new 101d21f HBASE-26579 Set storage policy of recovered edits when hbase.wal.storage.type is configured (#3948)
new 9c01d04 HBASE-26580 The message of StoreTooBusy is confused (#3949)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../throttle/StoreHotnessProtector.java | 33 +++++++++++++---------
.../org/apache/hadoop/hbase/wal/WALSplitUtil.java | 5 ++++
.../org/apache/hadoop/hbase/wal/TestWALSplit.java | 12 ++++++++
3 files changed, 36 insertions(+), 14 deletions(-)
[hbase] 01/02: HBASE-26579 Set storage policy of recovered edits when hbase.wal.storage.type is configured (#3948)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 101d21f25f8d4300e8448348016ce363fe3a9b6a
Author: zhengzhuobinzzb <zh...@gmail.com>
AuthorDate: Sat Dec 18 23:17:40 2021 +0800
HBASE-26579 Set storage policy of recovered edits when hbase.wal.storage.type is configured (#3948)
Signed-off-by: Duo Zhang <zh...@apache.org>
Signed-off-by: Peter Somogyi <ps...@apache.org>
---
.../main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java | 5 +++++
.../test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java | 12 ++++++++++++
2 files changed, 17 insertions(+)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java
index 2354b46..bb9a513 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java
@@ -148,6 +148,7 @@ public final class WALSplitUtil {
* named for the sequenceid in the passed <code>logEntry</code>: e.g.
* /hbase/some_table/2323432434/recovered.edits/2332. This method also ensures existence of
* RECOVERED_EDITS_DIR under the region creating it if necessary.
+ * And also set storage policy for RECOVERED_EDITS_DIR if WAL_STORAGE_POLICY is configured.
* @param tableName the table name
* @param encodedRegionName the encoded region name
* @param seqId the sequence id which used to generate file name
@@ -182,6 +183,10 @@ public final class WALSplitUtil {
if (!walFS.exists(dir) && !walFS.mkdirs(dir)) {
LOG.warn("mkdir failed on {}", dir);
+ } else {
+ String storagePolicy =
+ conf.get(HConstants.WAL_STORAGE_POLICY, HConstants.DEFAULT_WAL_STORAGE_POLICY);
+ CommonFSUtils.setStoragePolicy(walFS, dir, storagePolicy);
}
// Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.
// Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
index 5f22b45..a414f83 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
@@ -1174,6 +1174,18 @@ public class TestWALSplit {
}
}
+ @Test
+ public void testRecoveredEditsStoragePolicy() throws IOException {
+ conf.set(HConstants.WAL_STORAGE_POLICY, "ALL_SSD");
+ try {
+ Path path = createRecoveredEditsPathForRegion();
+ assertEquals("ALL_SSD", fs.getStoragePolicy(path.getParent()).getName());
+ } finally {
+ conf.unset(HConstants.WAL_STORAGE_POLICY);
+ }
+
+ }
+
private Writer generateWALs(int leaveOpen) throws IOException {
return generateWALs(NUM_WRITERS, ENTRIES, leaveOpen, 0);
}
[hbase] 02/02: HBASE-26580 The message of StoreTooBusy is confused (#3949)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 9c01d043541a8cff7eedfdfd789bbcf0fef2ae4a
Author: zhengzhuobinzzb <zh...@gmail.com>
AuthorDate: Sat Dec 18 23:22:14 2021 +0800
HBASE-26580 The message of StoreTooBusy is confused (#3949)
Signed-off-by: Duo Zhang <zh...@apache.org>
Reviewed-by: Bryan Beaudreault <bb...@hubspot.com>
---
.../throttle/StoreHotnessProtector.java | 33 +++++++++++++---------
1 file changed, 19 insertions(+), 14 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java
index f75fb3c..4cf0b59 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java
@@ -108,6 +108,8 @@ public class StoreHotnessProtector {
}
String tooBusyStore = null;
+ boolean aboveParallelThreadLimit = false;
+ boolean aboveParallelPrePutLimit = false;
for (Map.Entry<byte[], List<Cell>> e : familyMaps.entrySet()) {
Store store = this.region.getStore(e.getKey());
@@ -119,19 +121,19 @@ public class StoreHotnessProtector {
//we need to try to add #preparePutCount at first because preparePutToStoreMap will be
//cleared when changing the configuration.
- preparePutToStoreMap.putIfAbsent(e.getKey(), new AtomicInteger());
- AtomicInteger preparePutCounter = preparePutToStoreMap.get(e.getKey());
- if (preparePutCounter == null) {
- preparePutCounter = new AtomicInteger();
- preparePutToStoreMap.putIfAbsent(e.getKey(), preparePutCounter);
- }
- int preparePutCount = preparePutCounter.incrementAndGet();
- if (store.getCurrentParallelPutCount() > this.parallelPutToStoreThreadLimit
- || preparePutCount > this.parallelPreparePutToStoreThreadLimit) {
+ int preparePutCount = preparePutToStoreMap
+ .computeIfAbsent(e.getKey(), key -> new AtomicInteger())
+ .incrementAndGet();
+ boolean storeAboveThread =
+ store.getCurrentParallelPutCount() > this.parallelPutToStoreThreadLimit;
+ boolean storeAbovePrePut = preparePutCount > this.parallelPreparePutToStoreThreadLimit;
+ if (storeAboveThread || storeAbovePrePut) {
tooBusyStore = (tooBusyStore == null ?
store.getColumnFamilyName() :
tooBusyStore + "," + store.getColumnFamilyName());
}
+ aboveParallelThreadLimit |= storeAboveThread;
+ aboveParallelPrePutLimit |= storeAbovePrePut;
if (LOG.isTraceEnabled()) {
LOG.trace(store.getColumnFamilyName() + ": preparePutCount=" + preparePutCount
@@ -140,13 +142,16 @@ public class StoreHotnessProtector {
}
}
- if (tooBusyStore != null) {
+ if (aboveParallelThreadLimit || aboveParallelPrePutLimit) {
String msg =
"StoreTooBusy," + this.region.getRegionInfo().getRegionNameAsString() + ":" + tooBusyStore
- + " Above parallelPutToStoreThreadLimit(" + this.parallelPutToStoreThreadLimit + ")";
- if (LOG.isTraceEnabled()) {
- LOG.trace(msg);
- }
+ + " Above "
+ + (aboveParallelThreadLimit ? "parallelPutToStoreThreadLimit("
+ + this.parallelPutToStoreThreadLimit + ")" : "")
+ + (aboveParallelThreadLimit && aboveParallelPrePutLimit ? " or " : "")
+ + (aboveParallelPrePutLimit ? "parallelPreparePutToStoreThreadLimit("
+ + this.parallelPreparePutToStoreThreadLimit + ")" : "");
+ LOG.trace(msg);
throw new RegionTooBusyException(msg);
}
}