You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by so...@apache.org on 2022/06/01 20:27:27 UTC
[hadoop] branch branch-3.2 updated: HDFS-16610. Make fsck read timeout configurable (#4384)
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 58513d367e2 HDFS-16610. Make fsck read timeout configurable (#4384)
58513d367e2 is described below
commit 58513d367e2c3ae7581306a670ce8197f5edc465
Author: Stephen O'Donnell <st...@gmail.com>
AuthorDate: Wed Jun 1 20:36:01 2022 +0100
HDFS-16610. Make fsck read timeout configurable (#4384)
(cherry picked from commit 34a973a90ef89b633c9b5c13a79aa1ac11c92eb5)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
(cherry picked from commit 7d6b133af3f7ee4b986bdd2b266b83693614b6ab)
---
.../hadoop/hdfs/client/HdfsClientConfigKeys.java | 8 ++++++++
.../main/java/org/apache/hadoop/hdfs/tools/DFSck.java | 13 ++++++++++++-
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 19 +++++++++++++++++++
3 files changed, 39 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index b74c1194cf5..26834ecf258 100755
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -205,6 +205,14 @@ public interface HdfsClientConfigKeys {
String DFS_LEASE_HARDLIMIT_KEY = "dfs.namenode.lease-hard-limit-sec";
long DFS_LEASE_HARDLIMIT_DEFAULT = 20 * 60;
+ String DFS_CLIENT_FSCK_CONNECT_TIMEOUT =
+ "dfs.client.fsck.connect.timeout";
+ int DFS_CLIENT_FSCK_CONNECT_TIMEOUT_DEFAULT = 60 * 1000;
+
+ String DFS_CLIENT_FSCK_READ_TIMEOUT =
+ "dfs.client.fsck.read.timeout";
+ int DFS_CLIENT_FSCK_READ_TIMEOUT_DEFAULT = 60 * 1000;
+
/**
* These are deprecated config keys to client code.
*/
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 8e6ed4a5726..ade461aa24c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -27,6 +27,7 @@ import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@@ -37,6 +38,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck;
import org.apache.hadoop.hdfs.web.URLConnectionFactory;
import org.apache.hadoop.security.UserGroupInformation;
@@ -132,8 +134,17 @@ public class DFSck extends Configured implements Tool {
super(conf);
this.ugi = UserGroupInformation.getCurrentUser();
this.out = out;
+ int connectTimeout = (int) conf.getTimeDuration(
+ HdfsClientConfigKeys.DFS_CLIENT_FSCK_CONNECT_TIMEOUT,
+ HdfsClientConfigKeys.DFS_CLIENT_FSCK_CONNECT_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ int readTimeout = (int) conf.getTimeDuration(
+ HdfsClientConfigKeys.DFS_CLIENT_FSCK_READ_TIMEOUT,
+ HdfsClientConfigKeys.DFS_CLIENT_FSCK_READ_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS);
+
this.connectionFactory = URLConnectionFactory
- .newDefaultURLConnectionFactory(conf);
+ .newDefaultURLConnectionFactory(connectTimeout, readTimeout, conf);
this.isSpnegoEnabled = UserGroupInformation.isSecurityEnabled();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index f304ed0a720..702618cffe6 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -5508,4 +5508,23 @@
Determines the namenode automatic lease recovery interval in seconds.
</description>
</property>
+
+ <property>
+ <name>dfs.client.fsck.connect.timeout</name>
+ <value>60000ms</value>
+ <description>
+ The amount of time the fsck client will wait to connect to the namenode
+ before timing out.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.client.fsck.read.timeout</name>
+ <value>60000ms</value>
+ <description>
+ The amount of time the fsck client will wait to read from the namenode
+ before timing out. If the namenode does not report progress more
+ frequently than this time, the client will give up waiting.
+ </description>
+ </property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org