You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by so...@apache.org on 2021/12/20 12:00:46 UTC
[hadoop] branch branch-3.3 updated: HDFS-16386. Reduce DataNode load when FsDatasetAsyncDiskService is working. (#3806)
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new a4496a0 HDFS-16386. Reduce DataNode load when FsDatasetAsyncDiskService is working. (#3806)
a4496a0 is described below
commit a4496a0187f448592ac96ca7e4037bcec091e4d7
Author: jianghuazhu <74...@qq.com>
AuthorDate: Mon Dec 20 19:28:55 2021 +0800
HDFS-16386. Reduce DataNode load when FsDatasetAsyncDiskService is working. (#3806)
(cherry picked from commit 746b32855496c054854c3e02d9993d97f10073d2)
---
.../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +++
.../datanode/fsdataset/impl/FsDatasetAsyncDiskService.java | 12 ++++++++++--
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 10 ++++++++++
3 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 9eb1fdf..7196def 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -143,6 +143,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long DFS_DATANODE_MAX_LOCKED_MEMORY_DEFAULT = 0;
public static final String DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_KEY = "dfs.datanode.fsdatasetcache.max.threads.per.volume";
public static final int DFS_DATANODE_FSDATASETCACHE_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
+ public static final String DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY =
+ "dfs.datanode.fsdatasetasyncdisk.max.threads.per.volume";
+ public static final int DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT = 4;
public static final String DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC = "dfs.datanode.lazywriter.interval.sec";
public static final int DFS_DATANODE_LAZY_WRITER_INTERVAL_DEFAULT_SEC = 60;
public static final String DFS_DATANODE_RAM_DISK_REPLICA_TRACKER_KEY = "dfs.datanode.ram.disk.replica.tracker";
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
index 706c078..aa577a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetAsyncDiskService.java
@@ -30,6 +30,8 @@ import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -65,7 +67,7 @@ class FsDatasetAsyncDiskService {
// ThreadPool core pool size
private static final int CORE_THREADS_PER_VOLUME = 1;
// ThreadPool maximum pool size
- private static final int MAXIMUM_THREADS_PER_VOLUME = 4;
+ private final int maxNumThreadsPerVolume;
// ThreadPool keep-alive time for threads over core pool size
private static final long THREADS_KEEP_ALIVE_SECONDS = 60;
@@ -90,6 +92,12 @@ class FsDatasetAsyncDiskService {
this.datanode = datanode;
this.fsdatasetImpl = fsdatasetImpl;
this.threadGroup = new ThreadGroup(getClass().getSimpleName());
+ maxNumThreadsPerVolume = datanode.getConf().getInt(
+ DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY,
+ DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_DEFAULT);
+ Preconditions.checkArgument(maxNumThreadsPerVolume > 0,
+ DFSConfigKeys.DFS_DATANODE_FSDATASETASYNCDISK_MAX_THREADS_PER_VOLUME_KEY +
+ " must be a positive integer.");
}
private void addExecutorForVolume(final FsVolumeImpl volume) {
@@ -110,7 +118,7 @@ class FsDatasetAsyncDiskService {
};
ThreadPoolExecutor executor = new ThreadPoolExecutor(
- CORE_THREADS_PER_VOLUME, MAXIMUM_THREADS_PER_VOLUME,
+ CORE_THREADS_PER_VOLUME, maxNumThreadsPerVolume,
THREADS_KEEP_ALIVE_SECONDS, TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(), threadFactory);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 582e010..80c4818 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2885,6 +2885,16 @@
</property>
<property>
+ <name>dfs.datanode.fsdatasetasyncdisk.max.threads.per.volume</name>
+ <value>4</value>
+ <description>
+ The maximum number of threads per volume used to process async disk
+ operations on the datanode. These threads consume I/O and CPU at the
+ same time. This will affect normal data node operations.
+ </description>
+</property>
+
+<property>
<name>dfs.cachereport.intervalMsec</name>
<value>10000</value>
<description>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org