You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by he...@apache.org on 2022/04/21 04:32:24 UTC
[hadoop] branch trunk updated: HDFS-16500. Make asynchronous blocks deletion lock and unlock durtion threshold configurable. (#4061). Contributed by Chengwei Wang.
This is an automated email from the ASF dual-hosted git repository.
hexiaoqiao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new bf4730b4d69 HDFS-16500. Make asynchronous blocks deletion lock and unlock durtion threshold configurable. (#4061). Contributed by Chengwei Wang.
bf4730b4d69 is described below
commit bf4730b4d69e796cb0d968aa5893bdb506effb2f
Author: smarthan <11...@qq.com>
AuthorDate: Thu Apr 21 12:32:12 2022 +0800
HDFS-16500. Make asynchronous blocks deletion lock and unlock durtion threshold configurable. (#4061). Contributed by Chengwei Wang.
---
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 11 +++++++++++
.../hdfs/server/blockmanagement/BlockManager.java | 10 ++++++++--
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 19 +++++++++++++++++++
3 files changed, 38 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index cf1755cd9f9..9ed564ebd6f 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -481,6 +481,17 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.namenode.block.deletion.increment";
public static final int DFS_NAMENODE_BLOCK_DELETION_INCREMENT_DEFAULT = 1000;
+ /** The limit of single lock holding duration.*/
+ public static final String DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS =
+ "dfs.namenode.block.deletion.lock.threshold.ms";
+ public static final int DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS_DEFAULT =
+ 50;
+ /** The sleep interval for releasing lock.*/
+ public static final String DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS =
+ "dfs.namenode.block.deletion.unlock.interval.ms";
+ public static final int DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS_DEFAULT =
+ 10;
+
public static final String DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES =
HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES;
public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT =
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 59b3fb32e38..f3f8f734cc5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -192,8 +192,8 @@ public class BlockManager implements BlockStatsMXBean {
private volatile long lowRedundancyBlocksCount = 0L;
private volatile long scheduledReplicationBlocksCount = 0L;
- private final long deleteBlockLockTimeMs = 500;
- private final long deleteBlockUnlockIntervalTimeMs = 100;
+ private final long deleteBlockLockTimeMs;
+ private final long deleteBlockUnlockIntervalTimeMs;
/** flag indicating whether replication queues have been initialized */
private boolean initializedReplQueues;
@@ -495,6 +495,12 @@ public class BlockManager implements BlockStatsMXBean {
startupDelayBlockDeletionInMs = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_KEY,
DFSConfigKeys.DFS_NAMENODE_STARTUP_DELAY_BLOCK_DELETION_SEC_DEFAULT) * 1000L;
+ deleteBlockLockTimeMs = conf.getLong(
+ DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS,
+ DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_LOCK_THRESHOLD_MS_DEFAULT);
+ deleteBlockUnlockIntervalTimeMs = conf.getLong(
+ DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS,
+ DFSConfigKeys.DFS_NAMENODE_BLOCK_DELETION_UNLOCK_INTERVAL_MS_DEFAULT);
invalidateBlocks = new InvalidateBlocks(
datanodeManager.getBlockInvalidateLimit(),
startupDelayBlockDeletionInMs,
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index a75a76d7af4..b21134cbae2 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -6136,6 +6136,25 @@
</description>
</property>
+ <property>
+ <name>dfs.namenode.block.deletion.lock.threshold.ms</name>
+ <value>50</value>
+ <description>
+ The limit of single time lock holding duration for the block asynchronous
+ deletion thread.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.namenode.block.deletion.unlock.interval.ms</name>
+ <value>10</value>
+ <description>
+ The sleep interval for yield lock.
+ When the single time lock holding duration of the block asynchronous deletion
+ thread exceeds limit, sleeping to yield lock.
+ </description>
+ </property>
+
<property>
<name>dfs.namenode.rpc-address.auxiliary-ports</name>
<value></value>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org