You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2019/06/11 00:24:20 UTC
[hadoop] branch trunk updated: HDFS-14553. Make queue size of
BlockReportProcessingThread configurable. Contributed by He Xiaoqiao.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new bd46bdf HDFS-14553. Make queue size of BlockReportProcessingThread configurable. Contributed by He Xiaoqiao.
bd46bdf is described below
commit bd46bdf9f9244f3f3474d316255ac98717ed5719
Author: He Xiaoqiao <he...@apache.org>
AuthorDate: Mon Jun 10 17:20:50 2019 -0700
HDFS-14553. Make queue size of BlockReportProcessingThread configurable. Contributed by He Xiaoqiao.
Signed-off-by: Wei-Chiu Chuang <we...@apache.org>
---
.../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++++
.../hadoop/hdfs/server/blockmanagement/BlockManager.java | 14 +++++++++-----
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 8 ++++++++
3 files changed, 21 insertions(+), 5 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index e9d0eec..7bb30a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -269,6 +269,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
= "dfs.namenode.storageinfo.defragment.ratio";
public static final double
DFS_NAMENODE_STORAGEINFO_DEFRAGMENT_RATIO_DEFAULT = 0.75;
+ public static final String DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_KEY
+ = "dfs.namenode.blockreport.queue.size";
+ public static final int DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT
+ = 1024;
public static final String DFS_WEBHDFS_AUTHENTICATION_FILTER_KEY = "dfs.web.authentication.filter";
/* Phrased as below to avoid javac inlining as a constant, to match the behavior when
this was AuthFilter.class.getName(). Note that if you change the import for AuthFilter, you
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 9cfa180..bc2141d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -317,8 +317,7 @@ public class BlockManager implements BlockStatsMXBean {
new Daemon(new StorageInfoDefragmenter());
/** Block report thread for handling async reports. */
- private final BlockReportProcessingThread blockReportThread =
- new BlockReportProcessingThread();
+ private final BlockReportProcessingThread blockReportThread;
/**
* Store blocks {@literal ->} datanodedescriptor(s) map of corrupt replicas.
@@ -574,6 +573,11 @@ public class BlockManager implements BlockStatsMXBean {
bmSafeMode = new BlockManagerSafeMode(this, namesystem, haEnabled, conf);
+ int queueSize = conf.getInt(
+ DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_KEY,
+ DFSConfigKeys.DFS_NAMENODE_BLOCKREPORT_QUEUE_SIZE_DEFAULT);
+ blockReportThread = new BlockReportProcessingThread(queueSize);
+
LOG.info("defaultReplication = {}", defaultReplication);
LOG.info("maxReplication = {}", maxReplication);
LOG.info("minReplication = {}", minReplication);
@@ -4966,11 +4970,11 @@ public class BlockManager implements BlockStatsMXBean {
private static final long MAX_LOCK_HOLD_MS = 4;
private long lastFull = 0;
- private final BlockingQueue<Runnable> queue =
- new ArrayBlockingQueue<Runnable>(1024);
+ private final BlockingQueue<Runnable> queue;
- BlockReportProcessingThread() {
+ BlockReportProcessingThread(int size) {
super("Block report processor");
+ queue = new ArrayBlockingQueue<>(size);
setDaemon(true);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 8c575af..76c0660 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -5337,4 +5337,12 @@
inter-DN QOP.
</description>
</property>
+
+ <property>
+ <name>dfs.namenode.blockreport.queue.size</name>
+ <value>1024</value>
+ <description>
+ The queue size of BlockReportProcessingThread in BlockManager.
+ </description>
+ </property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org