You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2015/09/30 20:22:51 UTC
[17/52] [abbrv] hadoop git commit: HDFS-9086. Rename
dfs.datanode.stripedread.threshold.millis to
dfs.datanode.stripedread.timeout.millis. Contributed by Andrew Wang.
HDFS-9086. Rename dfs.datanode.stripedread.threshold.millis to dfs.datanode.stripedread.timeout.millis. Contributed by Andrew Wang.
Change-Id: Ice86c5d46d29e94087c0f186b941d5394e7ac0e6
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82a88b92
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82a88b92
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82a88b92
Branch: refs/heads/HDFS-7240
Commit: 82a88b92b46911c1b33d11898f8f678a134a9b69
Parents: 607bda2
Author: Zhe Zhang <zh...@cloudera.com>
Authored: Fri Sep 18 10:45:29 2015 -0700
Committer: Zhe Zhang <zh...@cloudera.com>
Committed: Fri Sep 18 10:45:29 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++
.../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ++--
.../server/datanode/erasurecode/ErasureCodingWorker.java | 10 +++++-----
.../org/apache/hadoop/hdfs/util/StripedBlockUtil.java | 6 +++---
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 8 ++++----
.../hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md | 2 +-
6 files changed, 18 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82a88b92/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 3a29db1..8ff696b 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -432,3 +432,6 @@
HDFS-9097. Erasure coding: update EC command "-s" flag to "-p" when
specifying policy. (zhz)
+
+ HDFS-9086. Rename dfs.datanode.stripedread.threshold.millis to
+ dfs.datanode.stripedread.timeout.millis. (wang via zhz)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82a88b92/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index f7cda18..53c6cdb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -399,8 +399,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_DATANODE_STRIPED_READ_THREADS_DEFAULT = 20;
public static final String DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.stripedread.buffer.size";
public static final int DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_DEFAULT = 64 * 1024;
- public static final String DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_KEY = "dfs.datanode.stripedread.threshold.millis";
- public static final int DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_DEFAULT = 5000; //5s
+ public static final String DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_KEY = "dfs.datanode.stripedread.timeout.millis";
+ public static final int DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_DEFAULT = 5000; //5s
public static final String DFS_DATANODE_STRIPED_BLK_RECOVERY_THREADS_KEY = "dfs.datanode.striped.blockrecovery.threads.size";
public static final int DFS_DATANODE_STRIPED_BLK_RECOVERY_THREADS_DEFAULT = 8;
public static final String DFS_DATANODE_DNS_INTERFACE_KEY = "dfs.datanode.dns.interface";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82a88b92/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
index 56b54f1..d1fa478 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -95,16 +95,16 @@ public final class ErasureCodingWorker {
private ThreadPoolExecutor STRIPED_BLK_RECOVERY_THREAD_POOL;
private ThreadPoolExecutor STRIPED_READ_THREAD_POOL;
- private final int STRIPED_READ_THRESHOLD_MILLIS;
+ private final int STRIPED_READ_TIMEOUT_MILLIS;
private final int STRIPED_READ_BUFFER_SIZE;
public ErasureCodingWorker(Configuration conf, DataNode datanode) {
this.datanode = datanode;
this.conf = conf;
- STRIPED_READ_THRESHOLD_MILLIS = conf.getInt(
- DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_KEY,
- DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_DEFAULT);
+ STRIPED_READ_TIMEOUT_MILLIS = conf.getInt(
+ DFSConfigKeys.DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_KEY,
+ DFSConfigKeys.DFS_DATANODE_STRIPED_READ_TIMEOUT_MILLIS_DEFAULT);
initializeStripedReadThreadPool(conf.getInt(
DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THREADS_KEY,
DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THREADS_DEFAULT));
@@ -556,7 +556,7 @@ public final class ErasureCodingWorker {
try {
StripingChunkReadResult result =
StripedBlockUtil.getNextCompletedStripedRead(
- readService, futures, STRIPED_READ_THRESHOLD_MILLIS);
+ readService, futures, STRIPED_READ_TIMEOUT_MILLIS);
int resultIndex = -1;
if (result.state == StripingChunkReadResult.SUCCESSFUL) {
resultIndex = result.index;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82a88b92/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index 4fded73..897b092 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -203,12 +203,12 @@ public class StripedBlockUtil {
*/
public static StripingChunkReadResult getNextCompletedStripedRead(
CompletionService<Void> readService, Map<Future<Void>, Integer> futures,
- final long threshold) throws InterruptedException {
+ final long timeoutMillis) throws InterruptedException {
Preconditions.checkArgument(!futures.isEmpty());
Future<Void> future = null;
try {
- if (threshold > 0) {
- future = readService.poll(threshold, TimeUnit.MILLISECONDS);
+ if (timeoutMillis > 0) {
+ future = readService.poll(timeoutMillis, TimeUnit.MILLISECONDS);
} else {
future = readService.take();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82a88b92/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 0db56dd..acbad07 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2403,23 +2403,23 @@
</property>
<property>
- <name>dfs.datanode.stripedread.threshold.millis</name>
+ <name>dfs.datanode.stripedread.timeout.millis</name>
<value>5000</value>
- <description>datanode striped read threshold in millisecond.
+ <description>Datanode striped read timeout in milliseconds.
</description>
</property>
<property>
<name>dfs.datanode.stripedread.threads</name>
<value>20</value>
- <description>datanode striped read thread pool size.
+ <description>Number of threads used by the Datanode for background recovery work.
</description>
</property>
<property>
<name>dfs.datanode.stripedread.buffer.size</name>
<value>262144</value>
- <description>datanode striped read buffer size.
+ <description>Datanode striped read buffer size.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82a88b92/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 2323a6d..5b3aa34 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -101,7 +101,7 @@ Deployment
Erasure coding background recovery work on the DataNodes can also be tuned via the following configuration parameters:
- 1. `dfs.datanode.stripedread.threshold.millis` - Timeout for striped reads. Default value is 5000 ms.
+ 1. `dfs.datanode.stripedread.timeout.millis` - Timeout for striped reads. Default value is 5000 ms.
1. `dfs.datanode.stripedread.threads` - Number of concurrent reader threads. Default value is 20 threads.
1. `dfs.datanode.stripedread.buffer.size` - Buffer size for reader service. Default value is 256KB.