You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2010/02/20 02:23:27 UTC
svn commit: r912056 - in /hadoop/common/trunk: CHANGES.txt
src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
src/java/org/apache/hadoop/ipc/Server.java
src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java
Author: suresh
Date: Sat Feb 20 01:23:26 2010
New Revision: 912056
URL: http://svn.apache.org/viewvc?rev=912056&view=rev
Log:
HADOOP-6577. Add hidden configuration option "ipc.server.max.response.size" to change the default 1 MB, the maximum size when large IPC handler response buffer is reset. Contributed by Suresh Srinivas.
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
hadoop/common/trunk/src/java/org/apache/hadoop/ipc/Server.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java
Modified: hadoop/common/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=912056&r1=912055&r2=912056&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Sat Feb 20 01:23:26 2010
@@ -54,10 +54,14 @@
HADOOP-6510. Adds a way for superusers to impersonate other users
in a secure environment. (Jitendra Nath Pandey via ddas)
- HADOOP-6421 Adds Symbolic links to FileContext, AbstractFileSystem.
+ HADOOP-6421. Adds Symbolic links to FileContext, AbstractFileSystem.
It also adds a limited implementation for the local file system
(RawLocalFs) that allows local symlinks. (Eli Collins via Sanjay Radia)
+ HADOOP-6577. Add hidden configuration option "ipc.server.max.response.size"
+ to change the default 1 MB, the maximum size when large IPC handler
+ response buffer is reset. (suresh)
+
IMPROVEMENTS
HADOOP-6283. Improve the exception messages thrown by
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java?rev=912056&r1=912055&r2=912056&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/fs/CommonConfigurationKeys.java Sat Feb 20 01:23:26 2010
@@ -119,6 +119,10 @@
public static final int IPC_CLIENT_IDLETHRESHOLD_DEFAULT = 4000;
public static final String IPC_SERVER_TCPNODELAY_KEY = "ipc.server.tcpnodelay";
public static final boolean IPC_SERVER_TCPNODELAY_DEFAULT = false;
+ public static final String IPC_SERVER_RPC_MAX_RESPONSE_SIZE_KEY =
+ "ipc.server.max.response.size";
+ public static final int IPC_SERVER_RPC_MAX_RESPONSE_SIZE_DEFAULT =
+ 1024*1024;
public static final String HADOOP_RPC_SOCKET_FACTORY_CLASS_DEFAULT_KEY =
"hadoop.rpc.socket.factory.class.default";
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/ipc/Server.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/ipc/Server.java?rev=912056&r1=912055&r2=912056&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/ipc/Server.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/ipc/Server.java Sat Feb 20 01:23:26 2010
@@ -59,6 +59,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.*;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.ipc.metrics.RpcMetrics;
@@ -104,8 +105,7 @@
* Initial and max size of response buffer
*/
static int INITIAL_RESP_BUF_SIZE = 10240;
- static int MAX_RESP_BUF_SIZE = 1024*1024;
-
+
public static final Log LOG = LogFactory.getLog(Server.class);
private static final ThreadLocal<Server> SERVER = new ThreadLocal<Server>();
@@ -174,6 +174,7 @@
private SecretManager<TokenIdentifier> secretManager;
private int maxQueueSize;
+ private final int maxRespSize;
private int socketSendBufferSize;
private final boolean tcpNoDelay; // if T then disable Nagle's Algorithm
@@ -1207,7 +1208,7 @@
: Status.ERROR, value, errorClass, error);
// Discard the large buf and reset it back to
// smaller size to freeup heap
- if (buf.size() > MAX_RESP_BUF_SIZE) {
+ if (buf.size() > maxRespSize) {
LOG.warn("Large response size " + buf.size() + " for call "
+ call.toString());
buf = new ByteArrayOutputStream(INITIAL_RESP_BUF_SIZE);
@@ -1253,6 +1254,8 @@
this.handlerCount = handlerCount;
this.socketSendBufferSize = 0;
this.maxQueueSize = handlerCount * MAX_QUEUE_SIZE_PER_HANDLER;
+ this.maxRespSize = conf.getInt(IPC_SERVER_RPC_MAX_RESPONSE_SIZE_KEY,
+ IPC_SERVER_RPC_MAX_RESPONSE_SIZE_DEFAULT);
this.callQueue = new LinkedBlockingQueue<Call>(maxQueueSize);
this.maxIdleTime = 2*conf.getInt("ipc.client.connection.maxidletime", 1000);
this.maxConnectionsToNuke = conf.getInt("ipc.client.kill.max", 10);
Modified: hadoop/common/trunk/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java?rev=912056&r1=912055&r2=912056&view=diff
==============================================================================
--- hadoop/common/trunk/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java (original)
+++ hadoop/common/trunk/src/test/core/org/apache/hadoop/ipc/TestIPCServerResponder.java Sat Feb 20 01:23:26 2010
@@ -27,6 +27,7 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.net.NetUtils;
@@ -116,8 +117,10 @@
public void testResponseBuffer() throws Exception {
Server.INITIAL_RESP_BUF_SIZE = 1;
- Server.MAX_RESP_BUF_SIZE = 1;
+ conf.setInt(CommonConfigurationKeys.IPC_SERVER_RPC_MAX_RESPONSE_SIZE_KEY,
+ 1);
testServerResponder(1, true, 1, 1, 5);
+ conf = new Configuration(); // reset configuration
}
public void testServerResponder() throws Exception {