You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by fe...@apache.org on 2020/12/28 06:18:46 UTC
[hadoop] branch trunk updated: HDFS-15749. Make size of
editPendingQ can be configurable (#2572)
This is an automated email from the ASF dual-hosted git repository.
ferhui pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 3b77cf4 HDFS-15749. Make size of editPendingQ can be configurable (#2572)
3b77cf4 is described below
commit 3b77cf444622a9ccc56e30a1dff6dfdb83676aca
Author: maobaolong <ma...@139.com>
AuthorDate: Mon Dec 28 14:18:18 2020 +0800
HDFS-15749. Make size of editPendingQ can be configurable (#2572)
---
.../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +++++
.../org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java | 10 ++++++++--
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 8 ++++++++
3 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 0a5caed..9260916 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -393,6 +393,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_NAMENODE_EDITS_ASYNC_LOGGING =
"dfs.namenode.edits.asynclogging";
public static final boolean DFS_NAMENODE_EDITS_ASYNC_LOGGING_DEFAULT = true;
+ public static final String
+ DFS_NAMENODE_EDITS_ASYNC_LOGGING_PENDING_QUEUE_SIZE =
+ "dfs.namenode.edits.asynclogging.pending.queue.size";
+ public static final int
+ DFS_NAMENODE_EDITS_ASYNC_LOGGING_PENDING_QUEUE_SIZE_DEFAULT = 4096;
public static final String DFS_NAMENODE_PROVIDED_ENABLED = "dfs.namenode.provided.enabled";
public static final boolean DFS_NAMENODE_PROVIDED_ENABLED_DEFAULT = false;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
index e73dfa7..68af0c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogAsync.java
@@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
@@ -45,8 +46,7 @@ class FSEditLogAsync extends FSEditLog implements Runnable {
private static final ThreadLocal<Edit> THREAD_EDIT = new ThreadLocal<Edit>();
// requires concurrent access from caller threads and syncing thread.
- private final BlockingQueue<Edit> editPendingQ =
- new ArrayBlockingQueue<Edit>(4096);
+ private final BlockingQueue<Edit> editPendingQ;
// only accessed by syncing thread so no synchronization required.
// queue is unbounded because it's effectively limited by the size
@@ -57,6 +57,12 @@ class FSEditLogAsync extends FSEditLog implements Runnable {
super(conf, storage, editsDirs);
// op instances cannot be shared due to queuing for background thread.
cache.disableCache();
+ int editPendingQSize = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_EDITS_ASYNC_LOGGING_PENDING_QUEUE_SIZE,
+ DFSConfigKeys.
+ DFS_NAMENODE_EDITS_ASYNC_LOGGING_PENDING_QUEUE_SIZE_DEFAULT);
+
+ editPendingQ = new ArrayBlockingQueue<>(editPendingQSize);
}
private boolean isSyncThreadAlive() {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index b1a0b1f..e5d626d 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4952,6 +4952,14 @@
</property>
<property>
+ <name>dfs.namenode.edits.asynclogging.pending.queue.size</name>
+ <value>4096</value>
+ <description>
+ The queue size of edit pending queue for FSEditLogAsync.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.edits.dir.minimum</name>
<value>1</value>
<description>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org