You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by sh...@apache.org on 2020/01/08 09:22:02 UTC
[hadoop-ozone] branch HDDS-2854 created (now a2c8548)
This is an automated email from the ASF dual-hosted git repository.
shashikant pushed a change to branch HDDS-2854
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
at a2c8548 HDDS-2854. Add a config for the write chunk executor queue limit.
This branch includes the following new commits:
new a2c8548 HDDS-2854. Add a config for the write chunk executor queue limit.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org
[hadoop-ozone] 01/01: HDDS-2854. Add a config for the write chunk
executor queue limit.
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shashikant pushed a commit to branch HDDS-2854
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit a2c854846041e4299ff29555de919ff946ac5006
Author: Shashikant Banerjee <sh...@apache.org>
AuthorDate: Wed Jan 8 14:51:34 2020 +0530
HDDS-2854. Add a config for the write chunk executor queue limit.
---
.../src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 ++++++
.../src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java | 8 ++++++++
hadoop-hdds/common/src/main/resources/ozone-default.xml | 9 +++++++++
.../common/transport/server/ratis/XceiverServerRatis.java | 7 ++++++-
4 files changed, 29 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
index 00d326e..8c11b05 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java
@@ -48,6 +48,12 @@ public final class ScmConfigKeys {
= "dfs.container.ratis.num.write.chunk.threads";
public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT
= 60;
+ public static final String
+ DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT
+ = "dfs.container.ratis.num.write.chunk.executor.queue.limit";
+ public static final int
+ DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT_DEFAULT
+ = 1024;
public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
= "dfs.container.ratis.replication.level";
public static final ReplicationLevel
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 1700926..184124d 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -240,6 +240,14 @@ public final class OzoneConfigKeys {
= ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY;
public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT
= ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT;
+ public static final String
+ DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT
+ = ScmConfigKeys.
+ DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT;
+ public static final int
+ DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT_DEFAULT
+ = ScmConfigKeys.
+ DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT_DEFAULT;
public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY
= ScmConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY;
public static final ReplicationLevel
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index c83a5bf..a319f0c 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -204,6 +204,15 @@
</description>
</property>
<property>
+ <name>dfs.container.ratis.num.write.chunk.executor.queue.limit</name>
+ <value>60</value>
+ <tag>OZONE, RATIS, PERFORMANCE</tag>
+ <description>Maximum limit of the number of tasks that can be pending
+ in the thread pool that Ratis will use for writing chunks
+ (1024 by default).
+ </description>
+ </property>
+ <property>
<name>dfs.container.ratis.leader.num.pending.requests</name>
<value>4096</value>
<tag>OZONE, RATIS, PERFORMANCE</tag>
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
index dd9327a..341e2af 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
@@ -127,10 +127,15 @@ public final class XceiverServerRatis implements XceiverServerSpi {
final int numWriteChunkThreads = conf.getInt(
OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY,
OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT);
+ final int queueLimit = conf.getInt(
+ OzoneConfigKeys.
+ DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT,
+ OzoneConfigKeys.
+ DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT_DEFAULT);
chunkExecutor =
new ThreadPoolExecutor(numWriteChunkThreads, numWriteChunkThreads,
100, TimeUnit.SECONDS,
- new ArrayBlockingQueue<>(1024),
+ new ArrayBlockingQueue<>(queueLimit),
new ThreadPoolExecutor.CallerRunsPolicy());
this.context = context;
this.replicationLevel =
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org