You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/03/13 00:04:46 UTC
[27/49] hadoop git commit: HADOOP-11226. Add a configuration to set
ipc.Client's traffic class with IPTOS_LOWDELAY|IPTOS_RELIABILITY. Contributed
by Gopal V.
HADOOP-11226. Add a configuration to set ipc.Client's traffic class with IPTOS_LOWDELAY|IPTOS_RELIABILITY. Contributed by Gopal V.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54639c7d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54639c7d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54639c7d
Branch: refs/heads/YARN-2928
Commit: 54639c7d7a34f4a46e8df50d57c79bab34b1ac07
Parents: c3003eb
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Tue Mar 10 13:08:29 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Tue Mar 10 13:08:29 2015 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../fs/CommonConfigurationKeysPublic.java | 6 +++-
.../main/java/org/apache/hadoop/ipc/Client.java | 33 ++++++++++++++++++--
.../src/main/resources/core-default.xml | 14 +++++++++
4 files changed, 52 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54639c7d/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0fe5b7c..fa73ba1 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -432,6 +432,9 @@ Release 2.8.0 - UNRELEASED
NEW FEATURES
+ HADOOP-11226. Add a configuration to set ipc.Client's traffic class with
+ IPTOS_LOWDELAY|IPTOS_RELIABILITY. (Gopal V via ozawa)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54639c7d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 00c8d78..470b4d0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -206,8 +206,12 @@ public class CommonConfigurationKeysPublic {
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String IPC_CLIENT_TCPNODELAY_KEY =
"ipc.client.tcpnodelay";
- /** Defalt value for IPC_CLIENT_TCPNODELAY_KEY */
+ /** Default value for IPC_CLIENT_TCPNODELAY_KEY */
public static final boolean IPC_CLIENT_TCPNODELAY_DEFAULT = true;
+ /** Enable low-latency connections from the client */
+ public static final String IPC_CLIENT_LOW_LATENCY = "ipc.client.low-latency";
+ /** Default value of IPC_CLIENT_LOW_LATENCY */
+ public static final boolean IPC_CLIENT_LOW_LATENCY_DEFAULT = false;
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String IPC_SERVER_LISTEN_QUEUE_SIZE_KEY =
"ipc.server.listen.queue.size";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54639c7d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 32558bc..97b715b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -384,7 +384,8 @@ public class Client {
private final RetryPolicy connectionRetryPolicy;
private final int maxRetriesOnSasl;
private int maxRetriesOnSocketTimeouts;
- private boolean tcpNoDelay; // if T then disable Nagle's Algorithm
+ private final boolean tcpNoDelay; // if T then disable Nagle's Algorithm
+ private final boolean tcpLowLatency; // if T then use low-delay QoS
private boolean doPing; //do we need to send ping message
private int pingInterval; // how often sends ping to the server in msecs
private ByteArrayOutputStream pingRequest; // ping message
@@ -413,6 +414,7 @@ public class Client {
this.maxRetriesOnSasl = remoteId.getMaxRetriesOnSasl();
this.maxRetriesOnSocketTimeouts = remoteId.getMaxRetriesOnSocketTimeouts();
this.tcpNoDelay = remoteId.getTcpNoDelay();
+ this.tcpLowLatency = remoteId.getTcpLowLatency();
this.doPing = remoteId.getDoPing();
if (doPing) {
// construct a RPC header with the callId as the ping callId
@@ -585,6 +587,20 @@ public class Client {
this.socket.setTcpNoDelay(tcpNoDelay);
this.socket.setKeepAlive(true);
+ if (tcpLowLatency) {
+ /*
+ * This allows intermediate switches to shape IPC traffic
+ * differently from Shuffle/HDFS DataStreamer traffic.
+ *
+ * IPTOS_RELIABILITY (0x04) | IPTOS_LOWDELAY (0x10)
+ *
+ * Prefer to optimize connect() speed & response latency over net
+ * throughput.
+ */
+ this.socket.setTrafficClass(0x04 | 0x10);
+ this.socket.setPerformancePreferences(1, 2, 0);
+ }
+
/*
* Bind the socket to the host specified in the principal name of the
* client, to ensure Server matching address of the client connection
@@ -1549,6 +1565,7 @@ public class Client {
// the max. no. of retries for socket connections on time out exceptions
private final int maxRetriesOnSocketTimeouts;
private final boolean tcpNoDelay; // if T then disable Nagle's Algorithm
+ private final boolean tcpLowLatency; // if T then use low-delay QoS
private final boolean doPing; //do we need to send ping message
private final int pingInterval; // how often sends ping to the server in msecs
private String saslQop; // here for testing
@@ -1575,6 +1592,10 @@ public class Client {
this.tcpNoDelay = conf.getBoolean(
CommonConfigurationKeysPublic.IPC_CLIENT_TCPNODELAY_KEY,
CommonConfigurationKeysPublic.IPC_CLIENT_TCPNODELAY_DEFAULT);
+ this.tcpLowLatency = conf.getBoolean(
+ CommonConfigurationKeysPublic.IPC_CLIENT_LOW_LATENCY,
+ CommonConfigurationKeysPublic.IPC_CLIENT_LOW_LATENCY_DEFAULT
+ );
this.doPing = conf.getBoolean(
CommonConfigurationKeys.IPC_CLIENT_PING_KEY,
CommonConfigurationKeys.IPC_CLIENT_PING_DEFAULT);
@@ -1610,11 +1631,17 @@ public class Client {
public int getMaxRetriesOnSocketTimeouts() {
return maxRetriesOnSocketTimeouts;
}
-
+
+ /** disable nagle's algorithm */
boolean getTcpNoDelay() {
return tcpNoDelay;
}
-
+
+ /** use low-latency QoS bits over TCP */
+ boolean getTcpLowLatency() {
+ return tcpLowLatency;
+ }
+
boolean getDoPing() {
return doPing;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54639c7d/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 74390d8..1d531df 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -978,6 +978,20 @@ for ldap providers in the same way as above does.
</property>
<property>
+ <name>ipc.client.tcpnodelay</name>
+ <value>true</value>
+ <description>Use TCP_NODELAY flag to bypass Nagle's algorithm transmission delays.
+ </description>
+</property>
+
+<property>
+ <name>ipc.client.low-latency</name>
+ <value>false</value>
+ <description>Use low-latency QoS markers for IPC connections.
+ </description>
+</property>
+
+<property>
<name>ipc.server.listen.queue.size</name>
<value>128</value>
<description>Indicates the length of the listen queue for servers accepting