You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by at...@apache.org on 2012/02/24 15:49:40 UTC
svn commit: r1293279 - in
/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common: ./
src/main/docs/ src/main/java/ src/main/java/org/apache/hadoop/ipc/
src/main/java/org/apache/hadoop/net/ src/test/core/
src/test/java/org/apache/hadoop/ipc/
Author: atm
Date: Fri Feb 24 14:49:38 2012
New Revision: 1293279
URL: http://svn.apache.org/viewvc?rev=1293279&view=rev
Log:
Merge trunk into HA branch.
Modified:
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt (contents, props changed)
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/core/ (props changed)
hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1293279&r1=1293278&r2=1293279&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt Fri Feb 24 14:49:38 2012
@@ -89,6 +89,12 @@ Trunk (unreleased changes)
HADOOP-8084. Updates ProtoBufRpc engine to not do an unnecessary copy
for RPC request/response. (ddas)
+ HADOOP-8085. Add RPC metrics to ProtobufRpcEngine. (Hari Mankude via
+ suresh)
+
+ HADOOP-8108. Move method getHostPortString() from NameNode to NetUtils.
+ (Brandon Li via jitendra)
+
BUG FIXES
HADOOP-8018. Hudson auto test for HDFS has started throwing javadoc
Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 24 14:49:38 2012
@@ -1,6 +1,6 @@
/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt:1243206,1291081
/hadoop/common/branches/yahoo-merge/CHANGES.txt:1079157,1079163-1079164,1079167
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161333-1292826
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt:1161333-1293274
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
/hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278
Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 24 14:49:38 2012
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:1152502-1292826
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/docs:1152502-1293274
/hadoop/core/branches/branch-0.19/src/docs:713112
Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 24 14:49:38 2012
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1152502-1292826
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java:1152502-1293274
/hadoop/core/branches/branch-0.19/core/src/java:713112
/hadoop/core/trunk/src/core:776175-785643,785929-786278
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java?rev=1293279&r1=1293278&r2=1293279&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java Fri Feb 24 14:49:38 2012
@@ -379,6 +379,24 @@ public class ProtobufRpcEngine implement
* Protobuf invoker for {@link RpcInvoker}
*/
static class ProtoBufRpcInvoker implements RpcInvoker {
+ private static ProtoClassProtoImpl getProtocolImpl(RPC.Server server,
+ String protoName, long version) throws IOException {
+ ProtoNameVer pv = new ProtoNameVer(protoName, version);
+ ProtoClassProtoImpl impl =
+ server.getProtocolImplMap(RpcKind.RPC_PROTOCOL_BUFFER).get(pv);
+ if (impl == null) { // no match for Protocol AND Version
+ VerProtocolImpl highest =
+ server.getHighestSupportedProtocol(RpcKind.RPC_PROTOCOL_BUFFER,
+ protoName);
+ if (highest == null) {
+ throw new IOException("Unknown protocol: " + protoName);
+ }
+ // protocol supported but not the version that client wants
+ throw new RPC.VersionMismatch(protoName, version,
+ highest.version);
+ }
+ return impl;
+ }
@Override
/**
@@ -409,21 +427,8 @@ public class ProtobufRpcEngine implement
if (server.verbose)
LOG.info("Call: protocol=" + protocol + ", method=" + methodName);
- ProtoNameVer pv = new ProtoNameVer(protoName, clientVersion);
- ProtoClassProtoImpl protocolImpl =
- server.getProtocolImplMap(RpcKind.RPC_PROTOCOL_BUFFER).get(pv);
- if (protocolImpl == null) { // no match for Protocol AND Version
- VerProtocolImpl highest =
- server.getHighestSupportedProtocol(RpcKind.RPC_PROTOCOL_BUFFER,
- protoName);
- if (highest == null) {
- throw new IOException("Unknown protocol: " + protoName);
- }
- // protocol supported but not the version that client wants
- throw new RPC.VersionMismatch(protoName, clientVersion,
- highest.version);
- }
-
+ ProtoClassProtoImpl protocolImpl = getProtocolImpl(server, protoName,
+ clientVersion);
BlockingService service = (BlockingService) protocolImpl.protocolImpl;
MethodDescriptor methodDescriptor = service.getDescriptorForType()
.findMethodByName(methodName);
@@ -438,7 +443,19 @@ public class ProtobufRpcEngine implement
.mergeFrom(rpcRequest.getRequest()).build();
Message result;
try {
+ long startTime = System.currentTimeMillis();
+ server.rpcDetailedMetrics.init(protocolImpl.protocolClass);
result = service.callBlockingMethod(methodDescriptor, null, param);
+ int processingTime = (int) (System.currentTimeMillis() - startTime);
+ int qTime = (int) (startTime - receiveTime);
+ if (LOG.isDebugEnabled()) {
+ LOG.info("Served: " + methodName + " queueTime= " + qTime +
+ " procesingTime= " + processingTime);
+ }
+ server.rpcMetrics.addRpcQueueTime(qTime);
+ server.rpcMetrics.addRpcProcessingTime(processingTime);
+ server.rpcDetailedMetrics.addProcessingTime(methodName,
+ processingTime);
} catch (ServiceException e) {
Throwable cause = e.getCause();
return handleException(cause != null ? cause : e);
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java?rev=1293279&r1=1293278&r2=1293279&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java Fri Feb 24 14:49:38 2012
@@ -317,10 +317,15 @@ public abstract class Server {
* Returns a handle to the rpcMetrics (required in tests)
* @return rpc metrics
*/
+ @VisibleForTesting
public RpcMetrics getRpcMetrics() {
return rpcMetrics;
}
+ @VisibleForTesting
+ public RpcDetailedMetrics getRpcDetailedMetrics() {
+ return rpcDetailedMetrics;
+ }
@VisibleForTesting
Iterable<? extends Thread> getHandlers() {
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java?rev=1293279&r1=1293278&r2=1293279&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java Fri Feb 24 14:49:38 2012
@@ -607,6 +607,13 @@ public class NetUtils {
}
/**
+ * Compose a "host:port" string from the address.
+ */
+ public static String getHostPortString(InetSocketAddress addr) {
+ return addr.getHostName() + ":" + addr.getPort();
+ }
+
+ /**
* Checks if {@code host} is a local host name and return {@link InetAddress}
* corresponding to that address.
*
Propchange: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Feb 24 14:49:38 2012
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:1152502-1292826
+/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/core:1152502-1293274
/hadoop/core/branches/branch-0.19/core/src/test/core:713112
/hadoop/core/trunk/src/test/core:776175-785643,785929-786278
Modified: hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java?rev=1293279&r1=1293278&r2=1293279&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestProtoBufRpc.java Fri Feb 24 14:49:38 2012
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.ipc;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt;
+
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -28,6 +31,7 @@ import org.apache.hadoop.ipc.protobuf.Te
import org.apache.hadoop.ipc.protobuf.TestProtos.EmptyResponseProto;
import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpc2Proto;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.net.NetUtils;
import org.junit.Assert;
import org.junit.Test;
@@ -187,5 +191,14 @@ public class TestProtoBufRpc {
.setMessage("hello").build();
EchoResponseProto echoResponse = client.echo2(null, echoRequest);
Assert.assertEquals(echoResponse.getMessage(), "hello");
+
+ // Ensure RPC metrics are updated
+ MetricsRecordBuilder rpcMetrics = getMetrics(server.getRpcMetrics().name());
+ assertCounterGt("RpcQueueTimeNumOps", 0L, rpcMetrics);
+ assertCounterGt("RpcProcessingTimeNumOps", 0L, rpcMetrics);
+
+ MetricsRecordBuilder rpcDetailedMetrics =
+ getMetrics(server.getRpcDetailedMetrics().name());
+ assertCounterGt("Echo2NumOps", 0L, rpcDetailedMetrics);
}
}
\ No newline at end of file