You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by le...@apache.org on 2017/09/13 01:13:06 UTC
hadoop git commit: HDFS-12412. Change
ErasureCodingWorker.stripedReadPool to cached thread pool. (Lei (Eddy) Xu)
Repository: hadoop
Updated Branches:
refs/heads/trunk f4b626746 -> 123342cd0
HDFS-12412. Change ErasureCodingWorker.stripedReadPool to cached thread pool. (Lei (Eddy) Xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/123342cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/123342cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/123342cd
Branch: refs/heads/trunk
Commit: 123342cd0759ff88801d4f5ab10987f6e3f344b0
Parents: f4b6267
Author: Lei Xu <le...@apache.org>
Authored: Tue Sep 12 18:12:07 2017 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Tue Sep 12 18:12:07 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 --
.../datanode/erasurecode/ErasureCodingWorker.java | 14 +++++++-------
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 9 ---------
.../src/site/markdown/HDFSErasureCoding.md | 1 -
4 files changed, 7 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/123342cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index d06e378..322cae4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -571,8 +571,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.namenode.ec.system.default.policy";
public static final String DFS_NAMENODE_EC_SYSTEM_DEFAULT_POLICY_DEFAULT =
"RS-6-3-1024k";
- public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_KEY = "dfs.datanode.ec.reconstruction.stripedread.threads";
- public static final int DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_DEFAULT = 20;
public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.ec.reconstruction.stripedread.buffer.size";
public static final int DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_DEFAULT = 64 * 1024;
public static final String DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_TIMEOUT_MILLIS_KEY = "dfs.datanode.ec.reconstruction.stripedread.timeout.millis";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/123342cd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
index 72c224f..d3de82e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -53,19 +53,19 @@ public final class ErasureCodingWorker {
this.datanode = datanode;
this.conf = conf;
- initializeStripedReadThreadPool(conf.getInt(
- DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_KEY,
- DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_DEFAULT));
+ initializeStripedReadThreadPool();
initializeStripedBlkReconstructionThreadPool(conf.getInt(
DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_BLK_THREADS_KEY,
DFSConfigKeys.DFS_DN_EC_RECONSTRUCTION_STRIPED_BLK_THREADS_DEFAULT));
}
- private void initializeStripedReadThreadPool(int num) {
- LOG.debug("Using striped reads; pool threads={}", num);
+ private void initializeStripedReadThreadPool() {
+ LOG.debug("Using striped reads");
- stripedReadPool = new ThreadPoolExecutor(1, num, 60, TimeUnit.SECONDS,
- new SynchronousQueue<Runnable>(),
+ // Essentially, this is a cachedThreadPool.
+ stripedReadPool = new ThreadPoolExecutor(0, Integer.MAX_VALUE,
+ 60, TimeUnit.SECONDS,
+ new SynchronousQueue<>(),
new Daemon.DaemonFactory() {
private final AtomicInteger threadIndex = new AtomicInteger(0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/123342cd/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 4f37b56..15ee9bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -3049,15 +3049,6 @@
</property>
<property>
- <name>dfs.datanode.ec.reconstruction.stripedread.threads</name>
- <value>20</value>
- <description>
- Number of threads used by the Datanode to read striped block
- during background reconstruction work.
- </description>
-</property>
-
-<property>
<name>dfs.datanode.ec.reconstruction.stripedread.buffer.size</name>
<value>65536</value>
<description>Datanode striped read buffer size.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/123342cd/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 5bd7c6d..0a88a5f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -137,7 +137,6 @@ Deployment
Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters:
1. `dfs.datanode.ec.reconstruction.stripedread.timeout.millis` - Timeout for striped reads. Default value is 5000 ms.
- 1. `dfs.datanode.ec.reconstruction.stripedread.threads` - Number of concurrent reader threads. Default value is 20 threads.
1. `dfs.datanode.ec.reconstruction.stripedread.buffer.size` - Buffer size for reader service. Default value is 64KB.
1. `dfs.datanode.ec.reconstruction.stripedblock.threads.size` - Number of threads used by the Datanode for background reconstruction work. Default value is 8 threads.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org