You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2021/12/18 15:39:05 UTC
[hbase] branch branch-2 updated (2d5d5b2 -> 68d8573)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a change to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git.
from 2d5d5b2 HBASE-26488 Memory leak when MemStore retry flushing (#3899)
new 12a286c HBASE-26579 Set storage policy of recovered edits when hbase.wal.storage.type is configured (#3948)
new 68d8573 HBASE-26580 The message of StoreTooBusy is confused (#3949)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../throttle/StoreHotnessProtector.java | 33 +++++++++++++---------
.../org/apache/hadoop/hbase/wal/WALSplitUtil.java | 5 ++++
.../org/apache/hadoop/hbase/wal/TestWALSplit.java | 12 ++++++++
3 files changed, 36 insertions(+), 14 deletions(-)
[hbase] 01/02: HBASE-26579 Set storage policy of recovered edits when hbase.wal.storage.type is configured (#3948)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 12a286ca6691c9e71fe6f8804b22e5dc6e84cd32
Author: zhengzhuobinzzb <zh...@gmail.com>
AuthorDate: Sat Dec 18 23:17:40 2021 +0800
HBASE-26579 Set storage policy of recovered edits when hbase.wal.storage.type is configured (#3948)
Signed-off-by: Duo Zhang <zh...@apache.org>
Signed-off-by: Peter Somogyi <ps...@apache.org>
---
.../main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java | 5 +++++
.../test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java | 12 ++++++++++++
2 files changed, 17 insertions(+)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java
index 11baddb..e5a6d70 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALSplitUtil.java
@@ -149,6 +149,7 @@ public final class WALSplitUtil {
* named for the sequenceid in the passed <code>logEntry</code>: e.g.
* /hbase/some_table/2323432434/recovered.edits/2332. This method also ensures existence of
* RECOVERED_EDITS_DIR under the region creating it if necessary.
+ * And also set storage policy for RECOVERED_EDITS_DIR if WAL_STORAGE_POLICY is configured.
* @param tableName the table name
* @param encodedRegionName the encoded region name
* @param seqId the sequence id which used to generate file name
@@ -183,6 +184,10 @@ public final class WALSplitUtil {
if (!walFS.exists(dir) && !walFS.mkdirs(dir)) {
LOG.warn("mkdir failed on {}", dir);
+ } else {
+ String storagePolicy =
+ conf.get(HConstants.WAL_STORAGE_POLICY, HConstants.DEFAULT_WAL_STORAGE_POLICY);
+ CommonFSUtils.setStoragePolicy(walFS, dir, storagePolicy);
}
// Append fileBeingSplit to prevent name conflict since we may have duplicate wal entries now.
// Append file name ends with RECOVERED_LOG_TMPFILE_SUFFIX to ensure
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
index fb278fd0..11781d3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALSplit.java
@@ -1174,6 +1174,18 @@ public class TestWALSplit {
}
}
+ @Test
+ public void testRecoveredEditsStoragePolicy() throws IOException {
+ conf.set(HConstants.WAL_STORAGE_POLICY, "ALL_SSD");
+ try {
+ Path path = createRecoveredEditsPathForRegion();
+ assertEquals("ALL_SSD", fs.getStoragePolicy(path.getParent()).getName());
+ } finally {
+ conf.unset(HConstants.WAL_STORAGE_POLICY);
+ }
+
+ }
+
private Writer generateWALs(int leaveOpen) throws IOException {
return generateWALs(NUM_WRITERS, ENTRIES, leaveOpen, 0);
}
[hbase] 02/02: HBASE-26580 The message of StoreTooBusy is confused (#3949)
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 68d8573c095df107aa0bfb52239c349ee314ff65
Author: zhengzhuobinzzb <zh...@gmail.com>
AuthorDate: Sat Dec 18 23:22:14 2021 +0800
HBASE-26580 The message of StoreTooBusy is confused (#3949)
Signed-off-by: Duo Zhang <zh...@apache.org>
Reviewed-by: Bryan Beaudreault <bb...@hubspot.com>
---
.../throttle/StoreHotnessProtector.java | 33 +++++++++++++---------
1 file changed, 19 insertions(+), 14 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java
index f75fb3c..4cf0b59 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/throttle/StoreHotnessProtector.java
@@ -108,6 +108,8 @@ public class StoreHotnessProtector {
}
String tooBusyStore = null;
+ boolean aboveParallelThreadLimit = false;
+ boolean aboveParallelPrePutLimit = false;
for (Map.Entry<byte[], List<Cell>> e : familyMaps.entrySet()) {
Store store = this.region.getStore(e.getKey());
@@ -119,19 +121,19 @@ public class StoreHotnessProtector {
//we need to try to add #preparePutCount at first because preparePutToStoreMap will be
//cleared when changing the configuration.
- preparePutToStoreMap.putIfAbsent(e.getKey(), new AtomicInteger());
- AtomicInteger preparePutCounter = preparePutToStoreMap.get(e.getKey());
- if (preparePutCounter == null) {
- preparePutCounter = new AtomicInteger();
- preparePutToStoreMap.putIfAbsent(e.getKey(), preparePutCounter);
- }
- int preparePutCount = preparePutCounter.incrementAndGet();
- if (store.getCurrentParallelPutCount() > this.parallelPutToStoreThreadLimit
- || preparePutCount > this.parallelPreparePutToStoreThreadLimit) {
+ int preparePutCount = preparePutToStoreMap
+ .computeIfAbsent(e.getKey(), key -> new AtomicInteger())
+ .incrementAndGet();
+ boolean storeAboveThread =
+ store.getCurrentParallelPutCount() > this.parallelPutToStoreThreadLimit;
+ boolean storeAbovePrePut = preparePutCount > this.parallelPreparePutToStoreThreadLimit;
+ if (storeAboveThread || storeAbovePrePut) {
tooBusyStore = (tooBusyStore == null ?
store.getColumnFamilyName() :
tooBusyStore + "," + store.getColumnFamilyName());
}
+ aboveParallelThreadLimit |= storeAboveThread;
+ aboveParallelPrePutLimit |= storeAbovePrePut;
if (LOG.isTraceEnabled()) {
LOG.trace(store.getColumnFamilyName() + ": preparePutCount=" + preparePutCount
@@ -140,13 +142,16 @@ public class StoreHotnessProtector {
}
}
- if (tooBusyStore != null) {
+ if (aboveParallelThreadLimit || aboveParallelPrePutLimit) {
String msg =
"StoreTooBusy," + this.region.getRegionInfo().getRegionNameAsString() + ":" + tooBusyStore
- + " Above parallelPutToStoreThreadLimit(" + this.parallelPutToStoreThreadLimit + ")";
- if (LOG.isTraceEnabled()) {
- LOG.trace(msg);
- }
+ + " Above "
+ + (aboveParallelThreadLimit ? "parallelPutToStoreThreadLimit("
+ + this.parallelPutToStoreThreadLimit + ")" : "")
+ + (aboveParallelThreadLimit && aboveParallelPrePutLimit ? " or " : "")
+ + (aboveParallelPrePutLimit ? "parallelPreparePutToStoreThreadLimit("
+ + this.parallelPreparePutToStoreThreadLimit + ")" : "");
+ LOG.trace(msg);
throw new RegionTooBusyException(msg);
}
}