You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by iw...@apache.org on 2016/02/12 19:38:25 UTC
hadoop git commit: HDFS-9700. DFSClient and DFSOutputStream should
set TCP_NODELAY on sockets for DataTransferProtocol (Gary Helmling via
iwasakims)
Repository: hadoop
Updated Branches:
refs/heads/trunk e6a7044b8 -> 372d1302c
HDFS-9700. DFSClient and DFSOutputStream should set TCP_NODELAY on sockets for DataTransferProtocol (Gary Helmling via iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/372d1302
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/372d1302
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/372d1302
Branch: refs/heads/trunk
Commit: 372d1302c63c6f49f99be5766c5da9647ebd9ca6
Parents: e6a7044
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Sat Feb 13 03:31:15 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Sat Feb 13 03:31:15 2016 +0900
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/DFSClient.java | 1 +
.../main/java/org/apache/hadoop/hdfs/DataStreamer.java | 1 +
.../hadoop/hdfs/client/HdfsClientConfigKeys.java | 4 ++++
.../apache/hadoop/hdfs/client/impl/DfsClientConf.java | 13 +++++++++++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
5 files changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/372d1302/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 9c00ea7..15a49f1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1890,6 +1890,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
String dnAddr = dn.getXferAddr(getConf().isConnectToDnViaHostname());
LOG.debug("Connecting to datanode {}", dnAddr);
NetUtils.connect(sock, NetUtils.createSocketAddr(dnAddr), timeout);
+ sock.setTcpNoDelay(dfsClientConf.getDataTransferTcpNoDelay());
sock.setSoTimeout(timeout);
OutputStream unbufOut = NetUtils.getOutputStream(sock);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/372d1302/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index e3843de..9d3cb55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -131,6 +131,7 @@ class DataStreamer extends Daemon {
final int timeout = client.getDatanodeReadTimeout(length);
NetUtils.connect(sock, isa, client.getRandomLocalInterfaceAddr(),
conf.getSocketTimeout());
+ sock.setTcpNoDelay(conf.getDataTransferTcpNoDelay());
sock.setSoTimeout(timeout);
sock.setKeepAlive(true);
if (conf.getSocketSendBufferSize() > 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/372d1302/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index fcfd49c..775907e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -160,6 +160,10 @@ public interface HdfsClientConfigKeys {
String DFS_USER_HOME_DIR_PREFIX_KEY = "dfs.user.home.dir.prefix";
String DFS_USER_HOME_DIR_PREFIX_DEFAULT = "/user";
+ String DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY =
+ "dfs.data.transfer.client.tcpnodelay";
+ boolean DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT = true;
+
/**
* These are deprecated config keys to client code.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/372d1302/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index 194f3ba..eae3c8e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -58,6 +58,8 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCK
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_CACHE_EXPIRY_MSEC_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_DEFAULT;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT;
@@ -135,6 +137,7 @@ public class DfsClientConf {
private final int stripedReadThreadpoolSize;
+ private final boolean dataTransferTcpNoDelay;
public DfsClientConf(Configuration conf) {
// The hdfsTimeout is currently the same as the ipc timeout
@@ -173,6 +176,9 @@ public class DfsClientConf {
CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
defaultChecksumOpt = getChecksumOptFromConf(conf);
+ dataTransferTcpNoDelay = conf.getBoolean(
+ DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_KEY,
+ DFS_DATA_TRANSFER_CLIENT_TCPNODELAY_DEFAULT);
socketTimeout = conf.getInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY,
HdfsConstants.READ_TIMEOUT);
socketSendBufferSize = conf.getInt(DFS_CLIENT_SOCKET_SEND_BUFFER_SIZE_KEY,
@@ -406,6 +412,13 @@ public class DfsClientConf {
}
/**
+ * @return whether TCP_NODELAY should be set on client sockets
+ */
+ public boolean getDataTransferTcpNoDelay() {
+ return dataTransferTcpNoDelay;
+ }
+
+ /**
* @return the socketTimeout
*/
public int getSocketTimeout() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/372d1302/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 27c4c0f..8d60d5e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1908,6 +1908,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9777. Fix typos in DFSAdmin command line and documentation.
(Wei-Chiu Chuang via umamahesh)
+ HDFS-9700. DFSClient and DFSOutputStream should set TCP_NODELAY on sockets
+ for DataTransferProtocol (Gary Helmling via iwasakims)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than