You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/09/25 11:51:09 UTC
svn commit: r1175359 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/
hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/
hadoop-yarn/hadoop-...
Author: acmurthy
Date: Sun Sep 25 09:51:09 2011
New Revision: 1175359
URL: http://svn.apache.org/viewvc?rev=1175359&view=rev
Log:
Merge -r 1175356:1175357 from trunk to branch-0.23 to fix MAPREDUCE-3053.
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1175359&r1=1175358&r2=1175359&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Sun Sep 25 09:51:09 2011
@@ -1377,6 +1377,12 @@ Release 0.23.0 - Unreleased
MAPREDUCE-2691. Increase threadpool size for launching containers in
MapReduce ApplicationMaster. (vinodkv via acmurthy)
+ MAPREDUCE-2990. Fixed display of NodeHealthStatus. (Subroto Sanyal via
+ acmurthy)
+
+ MAPREDUCE-3053. Better diagnostic message for unknown methods in ProtoBuf
+ RPCs. (vinodkv via acmurthy)
+
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java?rev=1175359&r1=1175358&r2=1175359&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/ProtoOverHadoopRpcEngine.java Sun Sep 25 09:51:09 2011
@@ -329,6 +329,12 @@ public class ProtoOverHadoopRpcEngine im
+ methodName);
MethodDescriptor methodDescriptor = service.getDescriptorForType()
.findMethodByName(methodName);
+ if (methodDescriptor == null) {
+ String msg = "Unknown method " + methodName + " called on "
+ + protocol + " protocol.";
+ LOG.warn(msg);
+ return handleException(new IOException(msg));
+ }
Message prototype = service.getRequestPrototype(methodDescriptor);
Message param = prototype.newBuilderForType()
.mergeFrom(rpcRequest.getRequestProto()).build();
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java?rev=1175359&r1=1175358&r2=1175359&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java Sun Sep 25 09:51:09 2011
@@ -25,9 +25,11 @@ import junit.framework.Assert;
import org.apache.avro.ipc.Server;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
import org.apache.hadoop.yarn.api.ContainerManager;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationIdRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
@@ -47,6 +49,7 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.Records;
import org.junit.Test;
public class TestRPC {
@@ -66,6 +69,35 @@ public class TestRPC {
// }
@Test
+ public void testUnknownCall() {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class
+ .getName());
+ YarnRPC rpc = YarnRPC.create(conf);
+ String bindAddr = "localhost:0";
+ InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
+ Server server = rpc.getServer(ContainerManager.class,
+ new DummyContainerManager(), addr, conf, null, 1);
+ server.start();
+
+ // Any unrelated protocol would do
+ ClientRMProtocol proxy = (ClientRMProtocol) rpc.getProxy(
+ ClientRMProtocol.class, NetUtils.createSocketAddr("localhost:"
+ + server.getPort()), conf);
+
+ try {
+ proxy.getNewApplicationId(Records
+ .newRecord(GetNewApplicationIdRequest.class));
+ Assert.fail("Excepted RPC call to fail with unknown method.");
+ } catch (YarnRemoteException e) {
+ Assert.assertEquals("Unknown method getNewApplicationId called on "
+ + "org.apache.hadoop.yarn.proto.ClientRMProtocol"
+ + "$ClientRMProtocolService$BlockingInterface protocol.", e
+ .getMessage());
+ }
+ }
+
+ @Test
public void testHadoopProtoRPC() throws Exception {
test(HadoopYarnProtoRPC.class.getName());
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java?rev=1175359&r1=1175358&r2=1175359&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java Sun Sep 25 09:51:09 2011
@@ -147,6 +147,7 @@ public class RMNodeImpl implements RMNod
this.httpAddress = hostName + ":" + httpPort;;
this.node = node;
this.nodeHealthStatus.setIsNodeHealthy(true);
+ this.nodeHealthStatus.setHealthReport("Healthy");
this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis());
this.latestHeartBeatResponse.setResponseId(0);
@@ -222,6 +223,18 @@ public class RMNodeImpl implements RMNod
}
}
+ private void setNodeHealthStatus(NodeHealthStatus status)
+ {
+ this.writeLock.lock();
+ try {
+ this.nodeHealthStatus.setHealthReport(status.getHealthReport());
+ this.nodeHealthStatus.setIsNodeHealthy(status.getIsNodeHealthy());
+ this.nodeHealthStatus.setLastHealthReportTime(status.getLastHealthReportTime());
+ } finally {
+ this.writeLock.unlock();
+ }
+ }
+
@Override
public RMNodeState getState() {
this.readLock.lock();
@@ -345,7 +358,10 @@ public class RMNodeImpl implements RMNod
// Switch the last heartbeatresponse.
rmNode.latestHeartBeatResponse = statusEvent.getLatestResponse();
- if (!statusEvent.getNodeHealthStatus().getIsNodeHealthy()) {
+ NodeHealthStatus remoteNodeHealthStatus =
+ statusEvent.getNodeHealthStatus();
+ rmNode.setNodeHealthStatus(remoteNodeHealthStatus);
+ if (!remoteNodeHealthStatus.getIsNodeHealthy()) {
// Inform the scheduler
rmNode.context.getDispatcher().getEventHandler().handle(
new NodeRemovedSchedulerEvent(rmNode));
@@ -392,8 +408,9 @@ public class RMNodeImpl implements RMNod
// Switch the last heartbeatresponse.
rmNode.latestHeartBeatResponse = statusEvent.getLatestResponse();
-
- if (statusEvent.getNodeHealthStatus().getIsNodeHealthy()) {
+ NodeHealthStatus remoteNodeHealthStatus = statusEvent.getNodeHealthStatus();
+ rmNode.setNodeHealthStatus(remoteNodeHealthStatus);
+ if (remoteNodeHealthStatus.getIsNodeHealthy()) {
rmNode.context.getDispatcher().getEventHandler().handle(
new NodeAddedSchedulerEvent(rmNode));
return RMNodeState.RUNNING;
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java?rev=1175359&r1=1175358&r2=1175359&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java Sun Sep 25 09:51:09 2011
@@ -18,12 +18,16 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import static org.junit.Assert.assertNotNull;
+
import java.io.IOException;
+import java.util.Collection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store;
@@ -153,6 +157,23 @@ public class TestResourceManager {
LOG.info("--- END: testResourceAllocation ---");
}
+
+ @Test
+ public void testNodeHealthReportIsNotNull() throws Exception{
+ String host1 = "host1";
+ final int memory = 4 * 1024;
+ org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm1 =
+ registerNode(host1, 1234, 2345, NetworkTopology.DEFAULT_RACK, memory);
+ nm1.heartbeat();
+ nm1.heartbeat();
+ Collection<RMNode> values = resourceManager.getRMContext().getRMNodes().values();
+ for (RMNode ni : values)
+ {
+ NodeHealthStatus nodeHealthStatus = ni.getNodeHealthStatus();
+ String healthReport = nodeHealthStatus.getHealthReport();
+ assertNotNull(healthReport);
+ }
+ }
private void checkResourceUsage(
org.apache.hadoop.yarn.server.resourcemanager.NodeManager... nodes ) {